From 707bb44427cbcf717b8fd973023cb794217f4234 Mon Sep 17 00:00:00 2001 From: Jacob Bohanon Date: Fri, 1 Mar 2024 07:28:05 -0500 Subject: [PATCH 01/37] bump envoy-fork, fix extensions --- bazel/extensions/extensions_build_config.bzl | 20 ++++++++++++++++++++ bazel/repository_locations.bzl | 4 ++-- changelog/v1.29.1-patch1/bump-envoy.yaml | 8 ++++++++ 3 files changed, 30 insertions(+), 2 deletions(-) create mode 100644 changelog/v1.29.1-patch1/bump-envoy.yaml diff --git a/bazel/extensions/extensions_build_config.bzl b/bazel/extensions/extensions_build_config.bzl index 99f179323..f8542194c 100644 --- a/bazel/extensions/extensions_build_config.bzl +++ b/bazel/extensions/extensions_build_config.bzl @@ -124,6 +124,7 @@ EXTENSIONS = { "envoy.filters.http.aws_lambda": "//source/extensions/filters/http/aws_lambda:config", "envoy.filters.http.aws_request_signing": "//source/extensions/filters/http/aws_request_signing:config", "envoy.filters.http.bandwidth_limit": "//source/extensions/filters/http/bandwidth_limit:config", + "envoy.filters.http.basic_auth": "//source/extensions/filters/http/basic_auth:config", "envoy.filters.http.buffer": "//source/extensions/filters/http/buffer:config", "envoy.filters.http.cache": "//source/extensions/filters/http/cache:config", "envoy.filters.http.cdn_loop": "//source/extensions/filters/http/cdn_loop:config", @@ -263,6 +264,19 @@ EXTENSIONS = { "envoy.tracers.skywalking": "//source/extensions/tracers/skywalking:config", "envoy.tracers.opentelemetry": "//source/extensions/tracers/opentelemetry:config", + # + # OpenTelemetry Resource Detectors + # + + "envoy.tracers.opentelemetry.resource_detectors.environment": "//source/extensions/tracers/opentelemetry/resource_detectors/environment:config", + "envoy.tracers.opentelemetry.resource_detectors.dynatrace": "//source/extensions/tracers/opentelemetry/resource_detectors/dynatrace:config", + + # + # OpenTelemetry tracer samplers + # + + "envoy.tracers.opentelemetry.samplers.always_on": "//source/extensions/tracers/opentelemetry/samplers/always_on:config", + # # Transport sockets # @@ -482,6 +496,12 @@ EXTENSIONS = { # Geolocation Provider # # "envoy.geoip_providers.maxmind": "//source/extensions/geoip_providers/maxmind:config", + + # + # cluster specifier plugin + # + + "envoy.router.cluster_specifier_plugin.lua": "//source/extensions/router/cluster_specifiers/lua:config", } # These can be changed to ["//visibility:public"], for downstream builds which diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index 122ba9baa..dc8347a1a 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -1,7 +1,7 @@ REPOSITORY_LOCATIONS = dict( envoy = dict( - # envoy 1.28.1 with backported ext_proc updates - commit = "47d4c36d399b9daae47cd7f6c4d41cf75e7e3ff8", + # envoy 1.29.1 with backported ext_proc updates + commit = "f52f0b37a461975e9053b198375a347628dbbd7d", remote = "https://github.com/solo-io/envoy-fork", ), inja = dict( diff --git a/changelog/v1.29.1-patch1/bump-envoy.yaml b/changelog/v1.29.1-patch1/bump-envoy.yaml new file mode 100644 index 000000000..21f92a5ec --- /dev/null +++ b/changelog/v1.29.1-patch1/bump-envoy.yaml @@ -0,0 +1,8 @@ +changelog: +- type: DEPENDENCY_BUMP + dependencyOwner: solo-io + dependencyRepo: envoy-fork + dependencyTag: v1.29.1 + resolvesIssue: false + description: >- + Update Envoy to latest from forked 1.29.1 From 9bdd8585693afb7182d4c3c01ba779f24c8a0702 Mon Sep 17 00:00:00 2001 From: Jacob Bohanon Date: Fri, 1 Mar 2024 08:07:25 -0500 Subject: [PATCH 02/37] fix FactoryContext from https://github.com/envoyproxy/envoy/pull/31189 --- .../aws_lambda/aws_lambda_filter_config_factory.cc | 12 ++++++------ source/extensions/filters/http/aws_lambda/config.cc | 4 ++-- .../nats_streaming_filter_config_factory.cc | 6 +++--- .../http/transformation/transformation_factory.cc | 4 ++-- .../transformation/transformation_filter_config.cc | 4 ++-- 5 files changed, 15 insertions(+), 15 deletions(-) diff --git a/source/extensions/filters/http/aws_lambda/aws_lambda_filter_config_factory.cc b/source/extensions/filters/http/aws_lambda/aws_lambda_filter_config_factory.cc index f780c052b..fad1dd2d7 100644 --- a/source/extensions/filters/http/aws_lambda/aws_lambda_filter_config_factory.cc +++ b/source/extensions/filters/http/aws_lambda/aws_lambda_filter_config_factory.cc @@ -20,18 +20,18 @@ AWSLambdaFilterConfigFactory::createFilterFactoryFromProtoTyped( auto chain = std::make_unique( - context.api(), Extensions::Common::Aws::Utility::fetchMetadata); - auto sts_factory = StsCredentialsProviderFactory::create(context.api(), - context.clusterManager()); + context.serverFactoryContext().api(), Extensions::Common::Aws::Utility::fetchMetadata); + auto sts_factory = StsCredentialsProviderFactory::create(context.serverFactoryContext().api(), + context.serverFactoryContext().clusterManager()); auto config = std::make_shared(std::move(chain), std::move(sts_factory), - context.mainThreadDispatcher(), context.api(), context.threadLocal(), stats_prefix, - context.scope(), proto_config); + context.serverFactoryContext().mainThreadDispatcher(), context.serverFactoryContext().api(), context.serverFactoryContext().threadLocal(), stats_prefix, + context.serverFactoryContext().scope(), proto_config); return [&context, config] (Http::FilterChainFactoryCallbacks &callbacks) -> void { callbacks.addStreamFilter(std::make_shared( - context.clusterManager(), context.api(), config)); + context.serverFactoryContext().clusterManager(), context.serverFactoryContext().api(), config)); }; } diff --git a/source/extensions/filters/http/aws_lambda/config.cc b/source/extensions/filters/http/aws_lambda/config.cc index 6f8ae3d70..e95d82543 100644 --- a/source/extensions/filters/http/aws_lambda/config.cc +++ b/source/extensions/filters/http/aws_lambda/config.cc @@ -301,13 +301,13 @@ AWSLambdaRouteConfig::AWSLambdaRouteConfig( if (protoconfig.has_transformer_config()) { auto &factory = Config::Utility::getAndCheckFactory(protoconfig.transformer_config()); - auto config = Config::Utility::translateAnyToFactoryConfig(protoconfig.transformer_config().typed_config(), context.messageValidationContext().staticValidationVisitor(), factory); + auto config = Config::Utility::translateAnyToFactoryConfig(protoconfig.transformer_config().typed_config(), context.serverFactoryContext().messageValidationContext().staticValidationVisitor(), factory); transformer_config_ = factory.createTransformer(*config, google::protobuf::BoolValue(), context); } if (protoconfig.has_request_transformer_config()) { auto &request_transformer_factory = Config::Utility::getAndCheckFactory(protoconfig.request_transformer_config()); - auto request_transformer_config = Config::Utility::translateAnyToFactoryConfig(protoconfig.request_transformer_config().typed_config(), context.messageValidationContext().staticValidationVisitor(), request_transformer_factory); + auto request_transformer_config = Config::Utility::translateAnyToFactoryConfig(protoconfig.request_transformer_config().typed_config(), context.serverFactoryContext().messageValidationContext().staticValidationVisitor(), request_transformer_factory); request_transformer_config_ = request_transformer_factory.createTransformer(*request_transformer_config, google::protobuf::BoolValue(), context); } } diff --git a/source/extensions/filters/http/nats/streaming/nats_streaming_filter_config_factory.cc b/source/extensions/filters/http/nats/streaming/nats_streaming_filter_config_factory.cc index 52a4d0cc3..f053e886e 100644 --- a/source/extensions/filters/http/nats/streaming/nats_streaming_filter_config_factory.cc +++ b/source/extensions/filters/http/nats/streaming/nats_streaming_filter_config_factory.cc @@ -24,7 +24,7 @@ NatsStreamingFilterConfigFactory::createFilterFactoryFromProtoTyped( NatsStreamingFilterConfigSharedPtr config = std::make_shared( - NatsStreamingFilterConfig(proto_config, context.clusterManager())); + NatsStreamingFilterConfig(proto_config, context.serverFactoryContext().clusterManager())); Tcp::ConnPoolNats::ClientFactory &client_factory = Tcp::ConnPoolNats::ClientFactoryImpl( - config->cluster(), context.clusterManager(), client_factory, - context.threadLocal(), context.api().randomGenerator(), config->opTimeout()); + config->cluster(), context.serverFactoryContext().clusterManager(), client_factory, + context.serverFactoryContext().threadLocal(), context.serverFactoryContext().api().randomGenerator(), config->opTimeout()); return [config, nats_streaming_client]( Envoy::Http::FilterChainFactoryCallbacks &callbacks) -> void { diff --git a/source/extensions/filters/http/transformation/transformation_factory.cc b/source/extensions/filters/http/transformation/transformation_factory.cc index a869c9377..6b953ea98 100644 --- a/source/extensions/filters/http/transformation/transformation_factory.cc +++ b/source/extensions/filters/http/transformation/transformation_factory.cc @@ -15,14 +15,14 @@ TransformerConstSharedPtr Transformation::getTransformer( switch (transformation.transformation_type_case()) { case envoy::api::v2::filter::http::Transformation::kTransformationTemplate: return std::make_unique( - transformation.transformation_template(), context.api().randomGenerator(), transformation.log_request_response_info(), context.threadLocal()); + transformation.transformation_template(), context.serverFactoryContext().api().randomGenerator(), transformation.log_request_response_info(), context.serverFactoryContext().threadLocal()); case envoy::api::v2::filter::http::Transformation::kHeaderBodyTransform: { const auto& header_body_transform = transformation.header_body_transform(); return std::make_unique(header_body_transform.add_request_metadata(), transformation.log_request_response_info()); } case envoy::api::v2::filter::http::Transformation::kTransformerConfig: { auto &factory = Config::Utility::getAndCheckFactory(transformation.transformer_config()); - auto config = Config::Utility::translateAnyToFactoryConfig(transformation.transformer_config().typed_config(), context.messageValidationContext().staticValidationVisitor(), factory); + auto config = Config::Utility::translateAnyToFactoryConfig(transformation.transformer_config().typed_config(), context.serverFactoryContext().messageValidationContext().staticValidationVisitor(), factory); return factory.createTransformer(*config, transformation.log_request_response_info(), context); } case envoy::api::v2::filter::http::Transformation:: diff --git a/source/extensions/filters/http/transformation/transformation_filter_config.cc b/source/extensions/filters/http/transformation/transformation_filter_config.cc index c96dbacc1..0d5146639 100644 --- a/source/extensions/filters/http/transformation/transformation_filter_config.cc +++ b/source/extensions/filters/http/transformation/transformation_filter_config.cc @@ -38,10 +38,10 @@ void TransformationFilterConfig::addTransformationLegacy( TransformationFilterConfig::TransformationFilterConfig( const TransformationConfigProto &proto_config, const std::string &prefix, Server::Configuration::FactoryContext &context) - : FilterConfig(prefix, context.scope(), proto_config.stage(), + : FilterConfig(prefix, context.serverFactoryContext().scope(), proto_config.stage(), proto_config.log_request_response_info()) { if (proto_config.has_matcher()) { - matcher_ = createTransformationMatcher(proto_config.matcher(), context.getServerFactoryContext()); + matcher_ = createTransformationMatcher(proto_config.matcher(), context.serverFactoryContext()); return; } for (const auto &rule : proto_config.transformations()) { From 0dbab3fddca15ae88f78fcff1f429eee23569968 Mon Sep 17 00:00:00 2001 From: Jacob Bohanon Date: Fri, 1 Mar 2024 08:57:10 -0500 Subject: [PATCH 03/37] more build fixes --- .../http/aws_lambda/aws_lambda_filter_config_factory.cc | 4 +++- source/extensions/filters/http/aws_lambda/config.cc | 4 ++-- .../filters/http/transformation/transformation_factory.cc | 4 ++-- .../http/transformation/transformation_filter_config.cc | 2 +- 4 files changed, 8 insertions(+), 6 deletions(-) diff --git a/source/extensions/filters/http/aws_lambda/aws_lambda_filter_config_factory.cc b/source/extensions/filters/http/aws_lambda/aws_lambda_filter_config_factory.cc index fad1dd2d7..86ae55962 100644 --- a/source/extensions/filters/http/aws_lambda/aws_lambda_filter_config_factory.cc +++ b/source/extensions/filters/http/aws_lambda/aws_lambda_filter_config_factory.cc @@ -20,7 +20,9 @@ AWSLambdaFilterConfigFactory::createFilterFactoryFromProtoTyped( auto chain = std::make_unique( - context.serverFactoryContext().api(), Extensions::Common::Aws::Utility::fetchMetadata); + context.serverFactoryContext().api(), makeOptRef(context.serverFactoryContext()), + proto_config.has_service_account_credentials() ? proto_config.service_account_credentials().region() : "TODO(jbohanon) figure this case out", + Extensions::Common::Aws::Utility::fetchMetadata); auto sts_factory = StsCredentialsProviderFactory::create(context.serverFactoryContext().api(), context.serverFactoryContext().clusterManager()); auto config = std::make_shared(std::move(chain), diff --git a/source/extensions/filters/http/aws_lambda/config.cc b/source/extensions/filters/http/aws_lambda/config.cc index e95d82543..6f8ae3d70 100644 --- a/source/extensions/filters/http/aws_lambda/config.cc +++ b/source/extensions/filters/http/aws_lambda/config.cc @@ -301,13 +301,13 @@ AWSLambdaRouteConfig::AWSLambdaRouteConfig( if (protoconfig.has_transformer_config()) { auto &factory = Config::Utility::getAndCheckFactory(protoconfig.transformer_config()); - auto config = Config::Utility::translateAnyToFactoryConfig(protoconfig.transformer_config().typed_config(), context.serverFactoryContext().messageValidationContext().staticValidationVisitor(), factory); + auto config = Config::Utility::translateAnyToFactoryConfig(protoconfig.transformer_config().typed_config(), context.messageValidationContext().staticValidationVisitor(), factory); transformer_config_ = factory.createTransformer(*config, google::protobuf::BoolValue(), context); } if (protoconfig.has_request_transformer_config()) { auto &request_transformer_factory = Config::Utility::getAndCheckFactory(protoconfig.request_transformer_config()); - auto request_transformer_config = Config::Utility::translateAnyToFactoryConfig(protoconfig.request_transformer_config().typed_config(), context.serverFactoryContext().messageValidationContext().staticValidationVisitor(), request_transformer_factory); + auto request_transformer_config = Config::Utility::translateAnyToFactoryConfig(protoconfig.request_transformer_config().typed_config(), context.messageValidationContext().staticValidationVisitor(), request_transformer_factory); request_transformer_config_ = request_transformer_factory.createTransformer(*request_transformer_config, google::protobuf::BoolValue(), context); } } diff --git a/source/extensions/filters/http/transformation/transformation_factory.cc b/source/extensions/filters/http/transformation/transformation_factory.cc index 6b953ea98..a869c9377 100644 --- a/source/extensions/filters/http/transformation/transformation_factory.cc +++ b/source/extensions/filters/http/transformation/transformation_factory.cc @@ -15,14 +15,14 @@ TransformerConstSharedPtr Transformation::getTransformer( switch (transformation.transformation_type_case()) { case envoy::api::v2::filter::http::Transformation::kTransformationTemplate: return std::make_unique( - transformation.transformation_template(), context.serverFactoryContext().api().randomGenerator(), transformation.log_request_response_info(), context.serverFactoryContext().threadLocal()); + transformation.transformation_template(), context.api().randomGenerator(), transformation.log_request_response_info(), context.threadLocal()); case envoy::api::v2::filter::http::Transformation::kHeaderBodyTransform: { const auto& header_body_transform = transformation.header_body_transform(); return std::make_unique(header_body_transform.add_request_metadata(), transformation.log_request_response_info()); } case envoy::api::v2::filter::http::Transformation::kTransformerConfig: { auto &factory = Config::Utility::getAndCheckFactory(transformation.transformer_config()); - auto config = Config::Utility::translateAnyToFactoryConfig(transformation.transformer_config().typed_config(), context.serverFactoryContext().messageValidationContext().staticValidationVisitor(), factory); + auto config = Config::Utility::translateAnyToFactoryConfig(transformation.transformer_config().typed_config(), context.messageValidationContext().staticValidationVisitor(), factory); return factory.createTransformer(*config, transformation.log_request_response_info(), context); } case envoy::api::v2::filter::http::Transformation:: diff --git a/source/extensions/filters/http/transformation/transformation_filter_config.cc b/source/extensions/filters/http/transformation/transformation_filter_config.cc index 0d5146639..dfac33d88 100644 --- a/source/extensions/filters/http/transformation/transformation_filter_config.cc +++ b/source/extensions/filters/http/transformation/transformation_filter_config.cc @@ -29,7 +29,7 @@ void TransformationFilterConfig::addTransformationLegacy( request_transformation, response_transformation, on_stream_completion_transformation, clear_route_cache); if (rule.has_route_transformations()) { - transformer_pair = createTransformations(rule.route_transformations(), context); + transformer_pair = createTransformations(rule.route_transformations(), context.serverFactoryContext()); } transformer_pairs_.emplace_back(MatcherCopy::Matcher::create(rule.match()), transformer_pair); From ec872f0906292d040772730e36a277e07eabad6f Mon Sep 17 00:00:00 2001 From: Jacob Bohanon Date: Mon, 4 Mar 2024 10:04:31 -0500 Subject: [PATCH 04/37] fix MockFactoryContext --- .../http/aws_lambda/aws_lambda_filter_test.cc | 12 +++---- .../aws_lambda/aws_lambda_transformer_test.cc | 8 ++--- .../filters/http/aws_lambda/config_test.cc | 26 +++++++------- .../aws_lambda/sts_connection_pool_test.cc | 8 ++--- .../sts_credentials_provider_test.cc | 8 ++--- .../http/aws_lambda/sts_fetcher_test.cc | 34 +++++++++---------- .../streaming/nats_streaming_filter_test.cc | 6 ++-- 7 files changed, 51 insertions(+), 51 deletions(-) diff --git a/test/extensions/filters/http/aws_lambda/aws_lambda_filter_test.cc b/test/extensions/filters/http/aws_lambda/aws_lambda_filter_test.cc index 221883460..2e3f498bc 100644 --- a/test/extensions/filters/http/aws_lambda/aws_lambda_filter_test.cc +++ b/test/extensions/filters/http/aws_lambda/aws_lambda_filter_test.cc @@ -83,8 +83,8 @@ class AWSLambdaFilterTest : public testing::Test { void setupRoute(bool sessionToken = false, bool noCredentials = false, bool persistOriginalHeaders = false, bool unwrapAsAlb = false, bool unmanagedCredentials = false, bool unwrapAsApiGateway = false) { - factory_context_.cluster_manager_.initializeClusters({"fake_cluster"}, {}); - factory_context_.cluster_manager_.initializeThreadLocalClusters({"fake_cluster"}); + factory_context_.server_factory_context_.cluster_manager_.initializeClusters({"fake_cluster"}, {}); + factory_context_.server_factory_context_.cluster_manager_.initializeThreadLocalClusters({"fake_cluster"}); routeconfig_.set_name("func"); routeconfig_.set_qualifier("v1"); @@ -131,7 +131,7 @@ class AWSLambdaFilterTest : public testing::Test { filter_config_->propagate_original_routing_=persistOriginalHeaders; ON_CALL( - *factory_context_.cluster_manager_.thread_local_cluster_.cluster_.info_, + *factory_context_.server_factory_context_.cluster_manager_.thread_local_cluster_.cluster_.info_, extensionProtocolOptions(SoloHttpFilterNames::get().AwsLambda)) .WillByDefault( Return(std::make_shared( @@ -139,7 +139,7 @@ class AWSLambdaFilterTest : public testing::Test { filter_ = std::make_unique( - factory_context_.cluster_manager_, factory_context_.api_, + factory_context_.server_factory_context_.cluster_manager_, factory_context_.server_factory_context_.api_, filter_config_); filter_->setDecoderFilterCallbacks(filter_callbacks_); @@ -402,7 +402,7 @@ TEST_F(AWSLambdaFilterTest, SignsDataSetByPreviousFilters) { auto hex_sha1 = auth1.getBodyHexSha(); filter_ = std::make_unique( - factory_context_.cluster_manager_, factory_context_.api_, + factory_context_.server_factory_context_.cluster_manager_, factory_context_.server_factory_context_.api_, filter_config_); filter_->setDecoderFilterCallbacks(filter_callbacks_); @@ -423,7 +423,7 @@ TEST_F(AWSLambdaFilterTest, SignsDataSetByPreviousFilters) { EXPECT_EQ(hex_sha1, hex_sha2); filter_ = std::make_unique( - factory_context_.cluster_manager_, factory_context_.api_, + factory_context_.server_factory_context_.cluster_manager_, factory_context_.server_factory_context_.api_, filter_config_); filter_->setDecoderFilterCallbacks(filter_callbacks_); diff --git a/test/extensions/filters/http/aws_lambda/aws_lambda_transformer_test.cc b/test/extensions/filters/http/aws_lambda/aws_lambda_transformer_test.cc index a8a8a6791..bc129ea3a 100644 --- a/test/extensions/filters/http/aws_lambda/aws_lambda_transformer_test.cc +++ b/test/extensions/filters/http/aws_lambda/aws_lambda_transformer_test.cc @@ -76,8 +76,8 @@ class AWSLambdaTransformerTest : public testing::Test, } void setupRoute(bool unwrapAsApiGateway = false, bool wrapAsApiGateway = false) { - factory_context_.cluster_manager_.initializeClusters({"fake_cluster"}, {}); - factory_context_.cluster_manager_.initializeThreadLocalClusters({"fake_cluster"}); + factory_context_.server_factory_context_.cluster_manager_.initializeClusters({"fake_cluster"}, {}); + factory_context_.server_factory_context_.cluster_manager_.initializeThreadLocalClusters({"fake_cluster"}); routeconfig_.set_name("func"); routeconfig_.set_qualifier("v1"); @@ -109,14 +109,14 @@ class AWSLambdaTransformerTest : public testing::Test, filter_config_->propagate_original_routing_=false; ON_CALL( - *factory_context_.cluster_manager_.thread_local_cluster_.cluster_.info_, + *factory_context_.server_factory_context_.cluster_manager_.thread_local_cluster_.cluster_.info_, extensionProtocolOptions(SoloHttpFilterNames::get().AwsLambda)) .WillByDefault( Return(std::make_shared( protoextconfig))); filter_ = std::make_unique( - factory_context_.cluster_manager_, factory_context_.api_, + factory_context_.server_factory_context_.cluster_manager_, factory_context_.server_factory_context_.api_, filter_config_); filter_->setDecoderFilterCallbacks(filter_callbacks_); } diff --git a/test/extensions/filters/http/aws_lambda/config_test.cc b/test/extensions/filters/http/aws_lambda/config_test.cc index 1a1a648a3..575bbac2c 100644 --- a/test/extensions/filters/http/aws_lambda/config_test.cc +++ b/test/extensions/filters/http/aws_lambda/config_test.cc @@ -49,9 +49,9 @@ class ConfigTest : public testing::Test { NiceMock *prepareTimer() { NiceMock *timer = - new NiceMock(&context_.dispatcher_); + new NiceMock(&context_.server_factory_context_.dispatcher_); protoconfig.mutable_use_default_credentials()->set_value(true); - EXPECT_CALL(context_.thread_local_, allocateSlot()).Times(1); + EXPECT_CALL(context_.server_factory_context_.thread_local_, allocateSlot()).Times(1); return timer; } @@ -84,8 +84,8 @@ TEST_F(ConfigTest, WithUseDefaultCreds) { std::unique_ptr> unique_factory{ sts_factory_}; auto config = std::make_shared( - std::move(cred_provider), std::move(unique_factory), context_.dispatcher_, - context_.api_, context_.thread_local_, "prefix.", *stats_.rootScope(), protoconfig); + std::move(cred_provider), std::move(unique_factory), context_.server_factory_context_.dispatcher_, + context_.server_factory_context_.api_, context_.server_factory_context_.thread_local_, "prefix.", *stats_.rootScope(), protoconfig); NiceMock callbacks_1; @@ -150,8 +150,8 @@ TEST_F(ConfigTest, FailingToRotate) { std::unique_ptr> unique_factory{ sts_factory_}; auto config = std::make_shared( - std::move(cred_provider), std::move(unique_factory), context_.dispatcher_, - context_.api_, context_.thread_local_, "prefix.", *stats_.rootScope(), protoconfig); + std::move(cred_provider), std::move(unique_factory), context_.server_factory_context_.dispatcher_, + context_.server_factory_context_.api_, context_.server_factory_context_.thread_local_, "prefix.", *stats_.rootScope(), protoconfig); std::shared_ptr ext_config_1 = std::make_shared(protoextconfig); @@ -200,8 +200,8 @@ TEST_F(ConfigTest, WithProtocolExtensionCreds) { std::unique_ptr> unique_factory{ sts_factory_}; auto config = std::make_shared( - std::move(cred_provider), std::move(unique_factory), context_.dispatcher_, - context_.api_, context_.thread_local_, "prefix.", *stats_.rootScope(), protoconfig); + std::move(cred_provider), std::move(unique_factory), context_.server_factory_context_.dispatcher_, + context_.server_factory_context_.api_, context_.server_factory_context_.thread_local_, "prefix.", *stats_.rootScope(), protoconfig); NiceMock callbacks_1; @@ -254,10 +254,10 @@ TEST_F(ConfigTest, WithStsCreds) { setenv("AWS_WEB_IDENTITY_TOKEN_FILE", "test", 1); setenv("AWS_ROLE_ARN", "test_arn", 1); - EXPECT_CALL(context_.api_.file_system_, fileExists(_)) + EXPECT_CALL(context_.server_factory_context_.api_.file_system_, fileExists(_)) .Times(1) .WillOnce(Return(true)); - EXPECT_CALL(context_.api_.file_system_, fileReadToEnd(_)) + EXPECT_CALL(context_.server_factory_context_.api_.file_system_, fileReadToEnd(_)) .Times(1) .WillOnce(Return("web_token")); @@ -273,15 +273,15 @@ TEST_F(ConfigTest, WithStsCreds) { })); auto watcher = new Filesystem::MockWatcher(); - EXPECT_CALL(context_.dispatcher_, createFilesystemWatcher_()) + EXPECT_CALL(context_.server_factory_context_.dispatcher_, createFilesystemWatcher_()) .WillOnce(Return(watcher)); EXPECT_CALL(*watcher, addWatch("test", _, _)).Times(1); std::unique_ptr> unique_factory{ sts_factory_}; auto config = std::make_shared( - std::move(cred_provider), std::move(unique_factory), context_.dispatcher_, - context_.api_, context_.thread_local_, "prefix.", *stats_.rootScope(), protoconfig); + std::move(cred_provider), std::move(unique_factory), context_.server_factory_context_.dispatcher_, + context_.server_factory_context_.api_, context_.server_factory_context_.thread_local_, "prefix.", *stats_.rootScope(), protoconfig); NiceMock callbacks; diff --git a/test/extensions/filters/http/aws_lambda/sts_connection_pool_test.cc b/test/extensions/filters/http/aws_lambda/sts_connection_pool_test.cc index d8f13cf40..e5a4a8681 100644 --- a/test/extensions/filters/http/aws_lambda/sts_connection_pool_test.cc +++ b/test/extensions/filters/http/aws_lambda/sts_connection_pool_test.cc @@ -83,7 +83,7 @@ TEST_F(StsConnectionPoolTest, TestSuccessfulCallback) { std::unique_ptr> unique_fetcher{ sts_fetcher_}; auto sts_conn_pool = StsConnectionPool::create( - mock_factory_ctx_.api_, mock_factory_ctx_.dispatcher_, role_arn, role_arn, + mock_factory_ctx_.server_factory_context_.api_, mock_factory_ctx_.server_factory_context_.dispatcher_, role_arn, role_arn, &pool_callbacks, std::move(unique_fetcher)); // Fetch credentials first call as they are not in the cache @@ -133,7 +133,7 @@ TEST_F(StsConnectionPoolTest, TestPostInitAdd) { std::unique_ptr> unique_fetcher{ sts_fetcher_}; auto sts_conn_pool = StsConnectionPool::create( - mock_factory_ctx_.api_, mock_factory_ctx_.dispatcher_, role_arn, role_arn, + mock_factory_ctx_.server_factory_context_.api_, mock_factory_ctx_.server_factory_context_.dispatcher_, role_arn, role_arn, &pool_callbacks, std::move(unique_fetcher)); StsFetcher::Callbacks *lambda_callbacks; @@ -154,7 +154,7 @@ TEST_F(StsConnectionPoolTest, TestPostInitAdd) { auto context_1 = sts_conn_pool->add(&ctx_callbacks); // Expect the context to be removed - EXPECT_CALL(mock_factory_ctx_.dispatcher_, deferredDelete_(_)); + EXPECT_CALL(mock_factory_ctx_.server_factory_context_.dispatcher_, deferredDelete_(_)); context_1->cancel(); @@ -193,7 +193,7 @@ TEST_F(StsConnectionPoolTest, TestFailure) { std::unique_ptr> unique_fetcher{ sts_fetcher_}; auto sts_conn_pool = StsConnectionPool::create( - mock_factory_ctx_.api_, mock_factory_ctx_.dispatcher_, role_arn, role_arn, + mock_factory_ctx_.server_factory_context_.api_, mock_factory_ctx_.server_factory_context_.dispatcher_, role_arn, role_arn, &pool_callbacks, std::move(unique_fetcher)); // Fetch credentials first call as they are not in the cache diff --git a/test/extensions/filters/http/aws_lambda/sts_credentials_provider_test.cc b/test/extensions/filters/http/aws_lambda/sts_credentials_provider_test.cc index f37e8dfc8..dac889d03 100644 --- a/test/extensions/filters/http/aws_lambda/sts_credentials_provider_test.cc +++ b/test/extensions/filters/http/aws_lambda/sts_credentials_provider_test.cc @@ -79,7 +79,7 @@ TEST_F(StsCredentialsProviderTest, TestFullFlow) { std::unique_ptr> factory_ = std::move(sts_connection_pool_factory_); auto* factory = factory_.get(); auto sts_provider = StsCredentialsProvider::create( - config_, mock_factory_ctx_.api_, mock_factory_ctx_.cluster_manager_, + config_, mock_factory_ctx_.server_factory_context_.api_, mock_factory_ctx_.server_factory_context_.cluster_manager_, std::move(factory_), token, role_arn); testing::NiceMock ctx_callbacks_1; @@ -164,7 +164,7 @@ TEST_F(StsCredentialsProviderTest, TestFullChainedFlow) { std::unique_ptr> factory_ = std::move(sts_connection_pool_factory_); auto* factory = factory_.get(); auto sts_provider = StsCredentialsProvider::create( - config_, mock_factory_ctx_.api_, mock_factory_ctx_.cluster_manager_, + config_, mock_factory_ctx_.server_factory_context_.api_, mock_factory_ctx_.server_factory_context_.cluster_manager_, std::move(factory_), token, base_role_arn); testing::NiceMock ctx_callbacks_1; @@ -253,7 +253,7 @@ TEST_F(StsCredentialsProviderTest, TestUnchainedFlow) { auto* factory = factory_.get(); auto sts_provider = StsCredentialsProvider::create( - config_, mock_factory_ctx_.api_, mock_factory_ctx_.cluster_manager_, + config_, mock_factory_ctx_.server_factory_context_.api_, mock_factory_ctx_.server_factory_context_.cluster_manager_, std::move(factory_), token, role_arn); testing::NiceMock ctx_callbacks_1; @@ -317,4 +317,4 @@ TEST_F(StsCredentialsProviderTest, TestUnchainedFlow) { } // namespace AwsLambda } // namespace HttpFilters } // namespace Extensions -} // namespace Envoy \ No newline at end of file +} // namespace Envoy diff --git a/test/extensions/filters/http/aws_lambda/sts_fetcher_test.cc b/test/extensions/filters/http/aws_lambda/sts_fetcher_test.cc index 1d2da68f0..305978fc4 100644 --- a/test/extensions/filters/http/aws_lambda/sts_fetcher_test.cc +++ b/test/extensions/filters/http/aws_lambda/sts_fetcher_test.cc @@ -101,8 +101,8 @@ const std::string web_token = "web_token"; class StsFetcherTest : public testing::Test { public: void SetUp() override { - mock_factory_ctx_.cluster_manager_.initializeClusters({"test"}, {}); - mock_factory_ctx_.cluster_manager_.initializeThreadLocalClusters({"test"}); + mock_factory_ctx_.server_factory_context_.cluster_manager_.initializeClusters({"test"}, {}); + mock_factory_ctx_.server_factory_context_.cluster_manager_.initializeThreadLocalClusters({"test"}); TestUtility::loadFromYaml(service_account_credentials_config, config_); uri_.set_cluster(config_.cluster()); uri_.set_uri(config_.uri()); @@ -119,10 +119,10 @@ class StsFetcherTest : public testing::Test { // Test findByIssuer TEST_F(StsFetcherTest, TestGetSuccess) { // Setup - MockUpstream mock_sts(mock_factory_ctx_.cluster_manager_, "200", + MockUpstream mock_sts(mock_factory_ctx_.server_factory_context_.cluster_manager_, "200", valid_response); std::unique_ptr fetcher(StsFetcher::create( - mock_factory_ctx_.cluster_manager_, mock_factory_ctx_.api_)); + mock_factory_ctx_.server_factory_context_.cluster_manager_, mock_factory_ctx_.server_factory_context_.api_)); EXPECT_TRUE(fetcher != nullptr); testing::NiceMock callbacks; @@ -133,10 +133,10 @@ TEST_F(StsFetcherTest, TestGetSuccess) { TEST_F(StsFetcherTest, TestChainedSts) { // Setup - MockUpstream mock_sts(mock_factory_ctx_.cluster_manager_, "200", + MockUpstream mock_sts(mock_factory_ctx_.server_factory_context_.cluster_manager_, "200", valid_chained_response); std::unique_ptr fetcher(StsFetcher::create( - mock_factory_ctx_.cluster_manager_, mock_factory_ctx_.api_)); + mock_factory_ctx_.server_factory_context_.cluster_manager_, mock_factory_ctx_.server_factory_context_.api_)); EXPECT_TRUE(fetcher != nullptr); testing::NiceMock callbacks; @@ -157,9 +157,9 @@ TEST_F(StsFetcherTest, TestChainedSts) { TEST_F(StsFetcherTest, TestGet503) { // Setup - MockUpstream mock_sts(mock_factory_ctx_.cluster_manager_, "503", "invalid"); + MockUpstream mock_sts(mock_factory_ctx_.server_factory_context_.cluster_manager_, "503", "invalid"); std::unique_ptr fetcher(StsFetcher::create( - mock_factory_ctx_.cluster_manager_, mock_factory_ctx_.api_)); + mock_factory_ctx_.server_factory_context_.cluster_manager_, mock_factory_ctx_.server_factory_context_.api_)); EXPECT_TRUE(fetcher != nullptr); testing::NiceMock callbacks; @@ -171,10 +171,10 @@ TEST_F(StsFetcherTest, TestGet503) { TEST_F(StsFetcherTest, TestCredentialsExpired) { // Setup - MockUpstream mock_sts(mock_factory_ctx_.cluster_manager_, "401", + MockUpstream mock_sts(mock_factory_ctx_.server_factory_context_.cluster_manager_, "401", expired_token_response); std::unique_ptr fetcher(StsFetcher::create( - mock_factory_ctx_.cluster_manager_, mock_factory_ctx_.api_)); + mock_factory_ctx_.server_factory_context_.cluster_manager_, mock_factory_ctx_.server_factory_context_.api_)); EXPECT_TRUE(fetcher != nullptr); testing::NiceMock callbacks; @@ -187,10 +187,10 @@ TEST_F(StsFetcherTest, TestCredentialsExpired) { TEST_F(StsFetcherTest, TestCredentialScopeMismatch) { // Setup - MockUpstream mock_sts(mock_factory_ctx_.cluster_manager_, "401", + MockUpstream mock_sts(mock_factory_ctx_.server_factory_context_.cluster_manager_, "401", credential_scope_mismatch); std::unique_ptr fetcher(StsFetcher::create( - mock_factory_ctx_.cluster_manager_, mock_factory_ctx_.api_)); + mock_factory_ctx_.server_factory_context_.cluster_manager_, mock_factory_ctx_.server_factory_context_.api_)); EXPECT_TRUE(fetcher != nullptr); testing::NiceMock callbacks; @@ -203,10 +203,10 @@ TEST_F(StsFetcherTest, TestCredentialScopeMismatch) { TEST_F(StsFetcherTest, TestHttpFailure) { // Setup - MockUpstream mock_sts(mock_factory_ctx_.cluster_manager_, + MockUpstream mock_sts(mock_factory_ctx_.server_factory_context_.cluster_manager_, Http::AsyncClient::FailureReason::Reset); std::unique_ptr fetcher(StsFetcher::create( - mock_factory_ctx_.cluster_manager_, mock_factory_ctx_.api_)); + mock_factory_ctx_.server_factory_context_.cluster_manager_, mock_factory_ctx_.server_factory_context_.api_)); EXPECT_TRUE(fetcher != nullptr); testing::NiceMock callbacks; @@ -219,10 +219,10 @@ TEST_F(StsFetcherTest, TestHttpFailure) { TEST_F(StsFetcherTest, TestCancel) { // Setup Http::MockAsyncClientRequest request( - &(mock_factory_ctx_.cluster_manager_.thread_local_cluster_.async_client_)); - MockUpstream mock_sts(mock_factory_ctx_.cluster_manager_, &request); + &(mock_factory_ctx_.server_factory_context_.cluster_manager_.thread_local_cluster_.async_client_)); + MockUpstream mock_sts(mock_factory_ctx_.server_factory_context_.cluster_manager_, &request); std::unique_ptr fetcher(StsFetcher::create( - mock_factory_ctx_.cluster_manager_, mock_factory_ctx_.api_)); + mock_factory_ctx_.server_factory_context_.cluster_manager_, mock_factory_ctx_.server_factory_context_.api_)); EXPECT_TRUE(fetcher != nullptr); EXPECT_CALL(request, cancel()).Times(1); diff --git a/test/extensions/filters/http/nats/streaming/nats_streaming_filter_test.cc b/test/extensions/filters/http/nats/streaming/nats_streaming_filter_test.cc index d9a8a735a..e61fba950 100644 --- a/test/extensions/filters/http/nats/streaming/nats_streaming_filter_test.cc +++ b/test/extensions/filters/http/nats/streaming/nats_streaming_filter_test.cc @@ -33,11 +33,11 @@ class NatsStreamingFilterTest : public testing::Test { proto_config.mutable_op_timeout()->set_nanos(17 * 1000000); proto_config.set_max_connections(1); proto_config.set_cluster("cluster"); - factory_context_.cluster_manager_.initializeClusters({"cluster"}, {}); - factory_context_.cluster_manager_.initializeThreadLocalClusters({"cluster"}); + factory_context_.server_factory_context_.cluster_manager_.initializeClusters({"cluster"}, {}); + factory_context_.server_factory_context_.cluster_manager_.initializeThreadLocalClusters({"cluster"}); config_.reset(new NatsStreamingFilterConfig( - proto_config, factory_context_.clusterManager())); + proto_config, factory_context_.server_factory_context_.clusterManager())); nats_streaming_client_.reset( new NiceMock); filter_.reset(new NatsStreamingFilter(config_, nats_streaming_client_)); From be9722bc5af601d72736e979128b5c2039aed483 Mon Sep 17 00:00:00 2001 From: Jacob Bohanon Date: Mon, 4 Mar 2024 11:35:37 -0500 Subject: [PATCH 05/37] use xds pkg for unified matcher instead of udpa --- api/envoy/config/filter/http/transformation/v2/BUILD | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/envoy/config/filter/http/transformation/v2/BUILD b/api/envoy/config/filter/http/transformation/v2/BUILD index 32a78b320..649334859 100644 --- a/api/envoy/config/filter/http/transformation/v2/BUILD +++ b/api/envoy/config/filter/http/transformation/v2/BUILD @@ -16,7 +16,7 @@ api_proto_package( "@envoy_api//envoy/type:pkg", "@envoy_api//envoy/type/matcher/v3:pkg", "@envoy_api//envoy/type/matcher:pkg", - "@com_github_cncf_udpa//xds/type/matcher/v3:pkg", + "@com_github_cncf_xds//xds/type/matcher/v3:pkg", ], visibility = ["//visibility:public"], ) From 6526e4b934613efffdfdbe18464cb75b769924c3 Mon Sep 17 00:00:00 2001 From: Jacob Bohanon Date: Mon, 4 Mar 2024 13:21:04 -0500 Subject: [PATCH 06/37] export the build options --- ci/do_ci.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ci/do_ci.sh b/ci/do_ci.sh index 56be03f3c..2890082a1 100755 --- a/ci/do_ci.sh +++ b/ci/do_ci.sh @@ -42,9 +42,11 @@ export BAZEL_EXTRA_TEST_OPTIONS="--test_env=ENVOY_IP_TEST_VERSIONS=v4only --test export ENVOY_CONTRIB_BUILD_TARGET="//source/exe:envoy-static" export ENVOY_CONTRIB_BUILD_DEBUG_INFORMATION="//source/exe:envoy-static.dwp" +export BAZEL_BUILD_EXTRA_OPTIONS BAZEL_BUILD_EXTRA_OPTIONS+=" --remote_cache=${BAZEL_REMOTE_CACHE}" -export GCP_SERVICE_ACCOUNT_KEY_PATH=$(mktemp -t gcp_service_account.XXXXXX.json) +export GCP_SERVICE_ACCOUNT_KEY_PATH +GCP_SERVICE_ACCOUNT_KEY_PATH=$(mktemp -t gcp_service_account.XXXXXX.json) echo "${GCP_SERVICE_ACCOUNT_KEY}" | base64 --decode > "${GCP_SERVICE_ACCOUNT_KEY_PATH}" BAZEL_BUILD_EXTRA_OPTIONS+=" --google_credentials=${GCP_SERVICE_ACCOUNT_KEY_PATH}" From 2b0389ddb2a70b77a252360c0c9b1c532bf0a49d Mon Sep 17 00:00:00 2001 From: Jacob Bohanon Date: Mon, 4 Mar 2024 14:55:11 -0500 Subject: [PATCH 07/37] another ServerFactoryContext --- .../http/transformation/transformation_filter_config_test.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/extensions/filters/http/transformation/transformation_filter_config_test.cc b/test/extensions/filters/http/transformation/transformation_filter_config_test.cc index ee68ae030..975e4accb 100644 --- a/test/extensions/filters/http/transformation/transformation_filter_config_test.cc +++ b/test/extensions/filters/http/transformation/transformation_filter_config_test.cc @@ -48,7 +48,7 @@ TEST(Transformation, TestGetTransformer) { factoryConfig->set_name("io.solo.transformer.fake"); auto any = factoryConfig->mutable_typed_config(); any->set_type_url("type.googleapis.com/envoy.test.extensions.transformation.FakeTransformer"); - auto transformer = t.getTransformer(transformation, factory_context_); + auto transformer = t.getTransformer(transformation, factory_context_.server_factory_context_); auto fakeTransformer = dynamic_cast(transformer.get()); // if transformer is not fake transformer type, will return nullptr EXPECT_NE(fakeTransformer, nullptr); @@ -57,4 +57,4 @@ TEST(Transformation, TestGetTransformer) { } } } -} \ No newline at end of file +} From ccbeb1831ece43a1e38a2f5e117898de9c3aebc3 Mon Sep 17 00:00:00 2001 From: Jacob Bohanon Date: Mon, 4 Mar 2024 16:41:25 -0500 Subject: [PATCH 08/37] use typed_config for lambda filter test --- test/integration/aws_lambda_filter_integration_test.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/integration/aws_lambda_filter_integration_test.cc b/test/integration/aws_lambda_filter_integration_test.cc index fb498dc68..0bd016024 100644 --- a/test/integration/aws_lambda_filter_integration_test.cc +++ b/test/integration/aws_lambda_filter_integration_test.cc @@ -13,6 +13,8 @@ namespace Envoy { const std::string DEFAULT_LAMBDA_FILTER = R"EOF( name: io.solo.aws_lambda +typed_config: + "@type": type.googleapis.com/envoy.config.filter.http.aws_lambda.v2.AWSLambdaConfig )EOF"; const std::string USE_CHAIN_LAMBDA_FILTER = From a7aedaf065dacdbf1f004d2ae72e7b8cd14eb1fc Mon Sep 17 00:00:00 2001 From: Jacob Bohanon Date: Tue, 5 Mar 2024 09:50:39 -0500 Subject: [PATCH 09/37] runtime flag...? --- test/integration/BUILD | 1 + test/integration/aws_lambda_filter_integration_test.cc | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/test/integration/BUILD b/test/integration/BUILD index a5894b8be..a904a3266 100644 --- a/test/integration/BUILD +++ b/test/integration/BUILD @@ -24,6 +24,7 @@ envoy_gloo_cc_test( "//source/extensions/filters/http/transformation:transformation_filter_config_lib", "@envoy//test/integration:http_integration_lib", "@envoy//test/integration:integration_lib", + "@envoy//test/test_common:test_runtime_lib", ], ) diff --git a/test/integration/aws_lambda_filter_integration_test.cc b/test/integration/aws_lambda_filter_integration_test.cc index 0bd016024..9abe7a9de 100644 --- a/test/integration/aws_lambda_filter_integration_test.cc +++ b/test/integration/aws_lambda_filter_integration_test.cc @@ -5,6 +5,7 @@ #include "test/integration/http_integration.h" #include "test/integration/integration.h" #include "test/integration/utility.h" +#include "test/test_common/test_runtime.h" #include "api/envoy/config/filter/http/aws_lambda/v2/aws_lambda.pb.validate.h" @@ -74,6 +75,8 @@ class AWSLambdaFilterIntegrationTest * Initializer for an individual integration test. */ void initialize() override { + scoped_runtime_.mergeValues( + {{"envoy.reloadable_features.use_libcurl_to_fetch_aws_credentials", true}}); if (add_transform_) { // not sure why but checking the "authorization" in the test succeeds. @@ -189,6 +192,7 @@ name: io.solo.aws_lambda enum class CredMode{DEFAULT, CHAIN, STS}; CredMode cred_mode_{}; bool add_transform_{}; + TestScopedRuntime scoped_runtime_; }; INSTANTIATE_TEST_SUITE_P( From d7e2a4bbe37e846522bd0212166abc74d7ae1821 Mon Sep 17 00:00:00 2001 From: Jacob Bohanon Date: Tue, 5 Mar 2024 10:06:18 -0500 Subject: [PATCH 10/37] runtime flag...as a string...? --- test/integration/aws_lambda_filter_integration_test.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/aws_lambda_filter_integration_test.cc b/test/integration/aws_lambda_filter_integration_test.cc index 9abe7a9de..2ab917610 100644 --- a/test/integration/aws_lambda_filter_integration_test.cc +++ b/test/integration/aws_lambda_filter_integration_test.cc @@ -76,7 +76,7 @@ class AWSLambdaFilterIntegrationTest */ void initialize() override { scoped_runtime_.mergeValues( - {{"envoy.reloadable_features.use_libcurl_to_fetch_aws_credentials", true}}); + {{"envoy.reloadable_features.use_libcurl_to_fetch_aws_credentials", "true"}}); if (add_transform_) { // not sure why but checking the "authorization" in the test succeeds. From 4de8a2f55a703f4ebed235fe9639aaaf3c54e002 Mon Sep 17 00:00:00 2001 From: Jacob Bohanon Date: Tue, 5 Mar 2024 10:58:33 -0500 Subject: [PATCH 11/37] use nullopt instead of server factory context. why? not sure yet --- .../http/aws_lambda/aws_lambda_filter_config_factory.cc | 2 +- test/integration/aws_lambda_filter_integration_test.cc | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/source/extensions/filters/http/aws_lambda/aws_lambda_filter_config_factory.cc b/source/extensions/filters/http/aws_lambda/aws_lambda_filter_config_factory.cc index 86ae55962..480992115 100644 --- a/source/extensions/filters/http/aws_lambda/aws_lambda_filter_config_factory.cc +++ b/source/extensions/filters/http/aws_lambda/aws_lambda_filter_config_factory.cc @@ -20,7 +20,7 @@ AWSLambdaFilterConfigFactory::createFilterFactoryFromProtoTyped( auto chain = std::make_unique( - context.serverFactoryContext().api(), makeOptRef(context.serverFactoryContext()), + context.serverFactoryContext().api(), absl::nullopt, proto_config.has_service_account_credentials() ? proto_config.service_account_credentials().region() : "TODO(jbohanon) figure this case out", Extensions::Common::Aws::Utility::fetchMetadata); auto sts_factory = StsCredentialsProviderFactory::create(context.serverFactoryContext().api(), diff --git a/test/integration/aws_lambda_filter_integration_test.cc b/test/integration/aws_lambda_filter_integration_test.cc index 2ab917610..9314d5184 100644 --- a/test/integration/aws_lambda_filter_integration_test.cc +++ b/test/integration/aws_lambda_filter_integration_test.cc @@ -75,8 +75,8 @@ class AWSLambdaFilterIntegrationTest * Initializer for an individual integration test. */ void initialize() override { - scoped_runtime_.mergeValues( - {{"envoy.reloadable_features.use_libcurl_to_fetch_aws_credentials", "true"}}); + /* scoped_runtime_.mergeValues( */ + /* {{"envoy.reloadable_features.use_libcurl_to_fetch_aws_credentials", "true"}}); */ if (add_transform_) { // not sure why but checking the "authorization" in the test succeeds. From aa63816221af275899e374de6370a3ffc19e8e1c Mon Sep 17 00:00:00 2001 From: Jacob Bohanon Date: Wed, 6 Mar 2024 10:46:04 -0500 Subject: [PATCH 12/37] put the server factory context back and try a different runtime flag per https://github.com/envoyproxy/envoy/pull/31135 --- .../http/aws_lambda/aws_lambda_filter_config_factory.cc | 2 +- test/integration/aws_lambda_filter_integration_test.cc | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/source/extensions/filters/http/aws_lambda/aws_lambda_filter_config_factory.cc b/source/extensions/filters/http/aws_lambda/aws_lambda_filter_config_factory.cc index 480992115..86ae55962 100644 --- a/source/extensions/filters/http/aws_lambda/aws_lambda_filter_config_factory.cc +++ b/source/extensions/filters/http/aws_lambda/aws_lambda_filter_config_factory.cc @@ -20,7 +20,7 @@ AWSLambdaFilterConfigFactory::createFilterFactoryFromProtoTyped( auto chain = std::make_unique( - context.serverFactoryContext().api(), absl::nullopt, + context.serverFactoryContext().api(), makeOptRef(context.serverFactoryContext()), proto_config.has_service_account_credentials() ? proto_config.service_account_credentials().region() : "TODO(jbohanon) figure this case out", Extensions::Common::Aws::Utility::fetchMetadata); auto sts_factory = StsCredentialsProviderFactory::create(context.serverFactoryContext().api(), diff --git a/test/integration/aws_lambda_filter_integration_test.cc b/test/integration/aws_lambda_filter_integration_test.cc index 9314d5184..d253cf146 100644 --- a/test/integration/aws_lambda_filter_integration_test.cc +++ b/test/integration/aws_lambda_filter_integration_test.cc @@ -75,8 +75,8 @@ class AWSLambdaFilterIntegrationTest * Initializer for an individual integration test. */ void initialize() override { - /* scoped_runtime_.mergeValues( */ - /* {{"envoy.reloadable_features.use_libcurl_to_fetch_aws_credentials", "true"}}); */ + scoped_runtime_.mergeValues( + {{"envoy.reloadable_features.use_http_client_to_fetch_aws_credentials", "true"}}); if (add_transform_) { // not sure why but checking the "authorization" in the test succeeds. From 941ed364f2d271192af237e39643f99c02b2b97b Mon Sep 17 00:00:00 2001 From: Jacob Bohanon Date: Wed, 6 Mar 2024 11:05:57 -0500 Subject: [PATCH 13/37] nuke le asan --- cloudbuild.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cloudbuild.yaml b/cloudbuild.yaml index 5797547c0..463a029b3 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -3,8 +3,8 @@ steps: - name: 'gcr.io/cloud-builders/gcloud' args: ['builds','submit','--config=ci/cloudbuild.yaml','--substitutions','TAG_NAME=$TAG_NAME,COMMIT_SHA=$COMMIT_SHA,_BUILD_TYPE='] -- name: 'gcr.io/cloud-builders/gcloud' - args: ['builds','submit','--config=ci/cloudbuild.yaml','--substitutions','COMMIT_SHA=$COMMIT_SHA,_BUILD_TYPE=clang-asan'] - waitFor: ['-'] +# - name: 'gcr.io/cloud-builders/gcloud' +# args: ['builds','submit','--config=ci/cloudbuild.yaml','--substitutions','COMMIT_SHA=$COMMIT_SHA,_BUILD_TYPE=clang-asan'] +# waitFor: ['-'] timeout: 20000s From d36e0e17b4131b010c232467d5f68b05763627cb Mon Sep 17 00:00:00 2001 From: Jacob Bohanon Date: Wed, 6 Mar 2024 11:41:25 -0500 Subject: [PATCH 14/37] uhh --- .../filters/http/aws_lambda/sts_credentials_provider.h | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/source/extensions/filters/http/aws_lambda/sts_credentials_provider.h b/source/extensions/filters/http/aws_lambda/sts_credentials_provider.h index 3ce82252d..41899e83a 100644 --- a/source/extensions/filters/http/aws_lambda/sts_credentials_provider.h +++ b/source/extensions/filters/http/aws_lambda/sts_credentials_provider.h @@ -62,6 +62,16 @@ class StsCredentialsProviderFactory { static StsCredentialsProviderFactoryPtr create(Api::Api &api, Upstream::ClusterManager &cm); + + Envoy::Extensions::Common::Aws::CredentialsProviderSharedPtr createWebIdentityCredentialsProvider( + Api::Api& api, Server::Configuration::ServerFactoryContext context, + const Envoy::Extensions::Common::Aws::MetadataCredentialsProviderBase::CurlMetadataFetcher& fetch_metadata_using_curl, + Envoy::Extensions::Common::Aws::CreateMetadataFetcherCb create_metadata_fetcher_cb, absl::string_view cluster_name, + absl::string_view token_file_path, absl::string_view sts_endpoint, absl::string_view role_arn, + absl::string_view role_session_name) const override { + std::cout << "we have hit createWebIdentityCredentialsProvider" < Date: Thu, 7 Mar 2024 14:49:03 -0500 Subject: [PATCH 15/37] use libcurl, patch out virtual call to ServerFactoryContext::clusterManager() --- bazel/repositories.bzl | 4 +++- .../aws_lambda_filter_config_factory.cc | 16 ++++++++-------- .../http/aws_lambda/sts_credentials_provider.h | 9 --------- .../aws_lambda_filter_integration_test.cc | 2 +- 4 files changed, 12 insertions(+), 19 deletions(-) diff --git a/bazel/repositories.bzl b/bazel/repositories.bzl index c477ce41c..a5215d96b 100644 --- a/bazel/repositories.bzl +++ b/bazel/repositories.bzl @@ -70,6 +70,8 @@ def envoy_gloo_dependencies(): # passthrough and only need to be backported onto envoy v1.25.x # these should be removed when moving to v1.26.x since this code exists in # upstream at that point. - _repository_impl("envoy") + _repository_impl("envoy", patches = [ + "@envoy_gloo//bazel/foreign_cc:aws-optref.patch", + ]) _repository_impl("json", build_file = "@envoy_gloo//bazel/external:json.BUILD") _repository_impl("inja", build_file = "@envoy_gloo//bazel/external:inja.BUILD") diff --git a/source/extensions/filters/http/aws_lambda/aws_lambda_filter_config_factory.cc b/source/extensions/filters/http/aws_lambda/aws_lambda_filter_config_factory.cc index 86ae55962..50e8f5f67 100644 --- a/source/extensions/filters/http/aws_lambda/aws_lambda_filter_config_factory.cc +++ b/source/extensions/filters/http/aws_lambda/aws_lambda_filter_config_factory.cc @@ -17,23 +17,23 @@ AWSLambdaFilterConfigFactory::createFilterFactoryFromProtoTyped( &proto_config, const std::string &stats_prefix, Server::Configuration::FactoryContext &context) { - + auto& server_context = context.serverFactoryContext(); auto chain = std::make_unique( - context.serverFactoryContext().api(), makeOptRef(context.serverFactoryContext()), + server_context.api(), makeOptRef(server_context), proto_config.has_service_account_credentials() ? proto_config.service_account_credentials().region() : "TODO(jbohanon) figure this case out", Extensions::Common::Aws::Utility::fetchMetadata); - auto sts_factory = StsCredentialsProviderFactory::create(context.serverFactoryContext().api(), - context.serverFactoryContext().clusterManager()); + auto sts_factory = StsCredentialsProviderFactory::create(server_context.api(), + server_context.clusterManager()); auto config = std::make_shared(std::move(chain), std::move(sts_factory), - context.serverFactoryContext().mainThreadDispatcher(), context.serverFactoryContext().api(), context.serverFactoryContext().threadLocal(), stats_prefix, - context.serverFactoryContext().scope(), proto_config); + server_context.mainThreadDispatcher(), server_context.api(), server_context.threadLocal(), stats_prefix, + server_context.scope(), proto_config); return - [&context, config] + [&server_context, config] (Http::FilterChainFactoryCallbacks &callbacks) -> void { callbacks.addStreamFilter(std::make_shared( - context.serverFactoryContext().clusterManager(), context.serverFactoryContext().api(), config)); + server_context.clusterManager(), server_context.api(), config)); }; } diff --git a/source/extensions/filters/http/aws_lambda/sts_credentials_provider.h b/source/extensions/filters/http/aws_lambda/sts_credentials_provider.h index 41899e83a..8ee3dad80 100644 --- a/source/extensions/filters/http/aws_lambda/sts_credentials_provider.h +++ b/source/extensions/filters/http/aws_lambda/sts_credentials_provider.h @@ -63,15 +63,6 @@ class StsCredentialsProviderFactory { static StsCredentialsProviderFactoryPtr create(Api::Api &api, Upstream::ClusterManager &cm); - Envoy::Extensions::Common::Aws::CredentialsProviderSharedPtr createWebIdentityCredentialsProvider( - Api::Api& api, Server::Configuration::ServerFactoryContext context, - const Envoy::Extensions::Common::Aws::MetadataCredentialsProviderBase::CurlMetadataFetcher& fetch_metadata_using_curl, - Envoy::Extensions::Common::Aws::CreateMetadataFetcherCb create_metadata_fetcher_cb, absl::string_view cluster_name, - absl::string_view token_file_path, absl::string_view sts_endpoint, absl::string_view role_arn, - absl::string_view role_session_name) const override { - std::cout << "we have hit createWebIdentityCredentialsProvider" < Date: Thu, 7 Mar 2024 14:54:29 -0500 Subject: [PATCH 16/37] forgot the patch d'oh --- bazel/foreign_cc/aws-optref.patch | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 bazel/foreign_cc/aws-optref.patch diff --git a/bazel/foreign_cc/aws-optref.patch b/bazel/foreign_cc/aws-optref.patch new file mode 100644 index 000000000..d34bea917 --- /dev/null +++ b/bazel/foreign_cc/aws-optref.patch @@ -0,0 +1,13 @@ +diff --git a/source/extensions/common/aws/credentials_provider_impl.cc b/source/extensions/common/aws/credentials_provider_impl.cc +index 7b9744a87a..619d23a65c 100644 +--- source/extensions/common/aws/credentials_provider_impl.cc ++++ source/extensions/common/aws/credentials_provider_impl.cc +@@ -108,7 +108,7 @@ MetadataCredentialsProviderBase::MetadataCredentialsProviderBase( + cluster_name_(std::string(cluster_name)), cluster_type_(cluster_type), uri_(std::string(uri)), + cache_duration_(getCacheDuration()), + debug_name_(absl::StrCat("Fetching aws credentials from cluster=", cluster_name)) { +- if (context_) { ++ if (false) { + context_->mainThreadDispatcher().post([this]() { + if (!Utility::addInternalClusterStatic(context_->clusterManager(), cluster_name_, + cluster_type_, uri_)) { From c5922ea9211400a5e241102bacee52e3868c42e3 Mon Sep 17 00:00:00 2001 From: Jacob Bohanon Date: Thu, 7 Mar 2024 17:46:26 -0500 Subject: [PATCH 17/37] ls on the out dirs --- .bazelversion | 2 +- ci/do_ci.sh | 16 ++++++++++------ 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/.bazelversion b/.bazelversion index dfda3e0b4..91e4a9f26 100644 --- a/.bazelversion +++ b/.bazelversion @@ -1 +1 @@ -6.1.0 +6.3.2 diff --git a/ci/do_ci.sh b/ci/do_ci.sh index 2890082a1..c9196bb93 100755 --- a/ci/do_ci.sh +++ b/ci/do_ci.sh @@ -42,13 +42,13 @@ export BAZEL_EXTRA_TEST_OPTIONS="--test_env=ENVOY_IP_TEST_VERSIONS=v4only --test export ENVOY_CONTRIB_BUILD_TARGET="//source/exe:envoy-static" export ENVOY_CONTRIB_BUILD_DEBUG_INFORMATION="//source/exe:envoy-static.dwp" -export BAZEL_BUILD_EXTRA_OPTIONS -BAZEL_BUILD_EXTRA_OPTIONS+=" --remote_cache=${BAZEL_REMOTE_CACHE}" +# export BAZEL_BUILD_EXTRA_OPTIONS +# BAZEL_BUILD_EXTRA_OPTIONS+=" --remote_cache=${BAZEL_REMOTE_CACHE}" -export GCP_SERVICE_ACCOUNT_KEY_PATH -GCP_SERVICE_ACCOUNT_KEY_PATH=$(mktemp -t gcp_service_account.XXXXXX.json) -echo "${GCP_SERVICE_ACCOUNT_KEY}" | base64 --decode > "${GCP_SERVICE_ACCOUNT_KEY_PATH}" -BAZEL_BUILD_EXTRA_OPTIONS+=" --google_credentials=${GCP_SERVICE_ACCOUNT_KEY_PATH}" +# export GCP_SERVICE_ACCOUNT_KEY_PATH +# GCP_SERVICE_ACCOUNT_KEY_PATH=$(mktemp -t gcp_service_account.XXXXXX.json) +# echo "${GCP_SERVICE_ACCOUNT_KEY}" | base64 --decode > "${GCP_SERVICE_ACCOUNT_KEY_PATH}" +# BAZEL_BUILD_EXTRA_OPTIONS+=" --google_credentials=${GCP_SERVICE_ACCOUNT_KEY_PATH}" if [ "${BUILD_TYPE:-}" != "" ] ; then BUILD_CONFIG="--config=$BUILD_TYPE" @@ -63,6 +63,10 @@ bash -x "$UPSTREAM_ENVOY_SRCDIR/ci/do_ci.sh" "$@" echo Extracting release binaries ENVOY_GLOO_BIN_DIR='linux/amd64/build_envoy_release' mkdir -p "$ENVOY_GLOO_BIN_DIR" +ls -al /build/ +ls -al /build/envoy/ +ls -al /build/envoy/x64/ +ls -al /build/envoy/x64/bin/ bazel run @envoy//tools/zstd:zstd -- --stdout -d /build/envoy/x64/bin/release.tar.zst \ | tar xfO - envoy > "$ENVOY_GLOO_BIN_DIR/envoy" From 2f8250dd6143178f442790287eab8fe5f2915979 Mon Sep 17 00:00:00 2001 From: Jacob Bohanon Date: Thu, 7 Mar 2024 17:53:35 -0500 Subject: [PATCH 18/37] undo commenting creds --- ci/do_ci.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/ci/do_ci.sh b/ci/do_ci.sh index c9196bb93..894a7fd48 100755 --- a/ci/do_ci.sh +++ b/ci/do_ci.sh @@ -42,13 +42,13 @@ export BAZEL_EXTRA_TEST_OPTIONS="--test_env=ENVOY_IP_TEST_VERSIONS=v4only --test export ENVOY_CONTRIB_BUILD_TARGET="//source/exe:envoy-static" export ENVOY_CONTRIB_BUILD_DEBUG_INFORMATION="//source/exe:envoy-static.dwp" -# export BAZEL_BUILD_EXTRA_OPTIONS -# BAZEL_BUILD_EXTRA_OPTIONS+=" --remote_cache=${BAZEL_REMOTE_CACHE}" +export BAZEL_BUILD_EXTRA_OPTIONS +BAZEL_BUILD_EXTRA_OPTIONS+=" --remote_cache=${BAZEL_REMOTE_CACHE}" -# export GCP_SERVICE_ACCOUNT_KEY_PATH -# GCP_SERVICE_ACCOUNT_KEY_PATH=$(mktemp -t gcp_service_account.XXXXXX.json) -# echo "${GCP_SERVICE_ACCOUNT_KEY}" | base64 --decode > "${GCP_SERVICE_ACCOUNT_KEY_PATH}" -# BAZEL_BUILD_EXTRA_OPTIONS+=" --google_credentials=${GCP_SERVICE_ACCOUNT_KEY_PATH}" +export GCP_SERVICE_ACCOUNT_KEY_PATH +GCP_SERVICE_ACCOUNT_KEY_PATH=$(mktemp -t gcp_service_account.XXXXXX.json) +echo "${GCP_SERVICE_ACCOUNT_KEY}" | base64 --decode > "${GCP_SERVICE_ACCOUNT_KEY_PATH}" +BAZEL_BUILD_EXTRA_OPTIONS+=" --google_credentials=${GCP_SERVICE_ACCOUNT_KEY_PATH}" if [ "${BUILD_TYPE:-}" != "" ] ; then BUILD_CONFIG="--config=$BUILD_TYPE" From 7e8e577b207516db62315889e4f16326a3eec230 Mon Sep 17 00:00:00 2001 From: Jacob Bohanon Date: Thu, 7 Mar 2024 18:19:07 -0500 Subject: [PATCH 19/37] add a build dir --- ci/do_ci.sh | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/ci/do_ci.sh b/ci/do_ci.sh index 894a7fd48..194c78f61 100755 --- a/ci/do_ci.sh +++ b/ci/do_ci.sh @@ -2,10 +2,9 @@ set -e if [ -n "$ENVOY_DOCKER_BUILD_DIR" ]; then - rm -rf "$ENVOY_DOCKER_BUILD_DIR/envoy/x64/bin/" -else - rm -rf "/build/envoy/x64/bin/" + export ENVOY_DOCKER_BUILD_DIR=/build fi +rm -rf "$ENVOY_DOCKER_BUILD_DIR/envoy/x64/bin/" bazel fetch //source/exe:envoy-static From efdb1d2ae19bc89b47f1b984bc4a2d8cbc90f57f Mon Sep 17 00:00:00 2001 From: Jacob Bohanon Date: Thu, 7 Mar 2024 18:29:34 -0500 Subject: [PATCH 20/37] derp --- ci/do_ci.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ci/do_ci.sh b/ci/do_ci.sh index 194c78f61..d39313408 100755 --- a/ci/do_ci.sh +++ b/ci/do_ci.sh @@ -1,9 +1,12 @@ #!/bin/bash set -e -if [ -n "$ENVOY_DOCKER_BUILD_DIR" ]; then +if [ -z "$ENVOY_DOCKER_BUILD_DIR" ]; then export ENVOY_DOCKER_BUILD_DIR=/build fi +if [ -z "$ENVOY_BUILD_DIR" ]; then + export ENVOY_BUILD_DIR=/build +fi rm -rf "$ENVOY_DOCKER_BUILD_DIR/envoy/x64/bin/" bazel fetch //source/exe:envoy-static From d49d1929f11d3602729bf647d46462e357d3d7ed Mon Sep 17 00:00:00 2001 From: Jacob Bohanon Date: Thu, 7 Mar 2024 18:44:27 -0500 Subject: [PATCH 21/37] what the heck, respect my vars bro --- ci/do_ci.sh | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/ci/do_ci.sh b/ci/do_ci.sh index d39313408..e314821fc 100755 --- a/ci/do_ci.sh +++ b/ci/do_ci.sh @@ -3,11 +3,18 @@ set -e if [ -z "$ENVOY_DOCKER_BUILD_DIR" ]; then export ENVOY_DOCKER_BUILD_DIR=/build + echo "using docker build dir: ${ENVOY_DOCKER_BUILD_DIR}" +else + echo "using pre-defined docker build dir: ${ENVOY_DOCKER_BUILD_DIR}" fi if [ -z "$ENVOY_BUILD_DIR" ]; then - export ENVOY_BUILD_DIR=/build + export ENVOY_BUILD_DIR=/build/envoy/x64 + echo "using build dir: ${ENVOY_BUILD_DIR}" +else + echo "using pre-defined build dir: ${ENVOY_BUILD_DIR}" fi rm -rf "$ENVOY_DOCKER_BUILD_DIR/envoy/x64/bin/" +mkdir -p ${ENVOY_BUILD_DIR} bazel fetch //source/exe:envoy-static From 6f70edc2122e0c65bf06b9969c103c20d1d6cd16 Mon Sep 17 00:00:00 2001 From: Jacob Bohanon Date: Thu, 7 Mar 2024 18:59:08 -0500 Subject: [PATCH 22/37] WHY AREN'T WE USING THE RIGHT ENV VARS --- ci/do_ci.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ci/do_ci.sh b/ci/do_ci.sh index e314821fc..20b9df50a 100755 --- a/ci/do_ci.sh +++ b/ci/do_ci.sh @@ -1,14 +1,16 @@ #!/bin/bash set -e +export ENVOY_DOCKER_BUILD_DIR +export ENVOY_BUILD_DIR if [ -z "$ENVOY_DOCKER_BUILD_DIR" ]; then - export ENVOY_DOCKER_BUILD_DIR=/build + ENVOY_DOCKER_BUILD_DIR=/build echo "using docker build dir: ${ENVOY_DOCKER_BUILD_DIR}" else echo "using pre-defined docker build dir: ${ENVOY_DOCKER_BUILD_DIR}" fi if [ -z "$ENVOY_BUILD_DIR" ]; then - export ENVOY_BUILD_DIR=/build/envoy/x64 + ENVOY_BUILD_DIR=/build/envoy/x64 echo "using build dir: ${ENVOY_BUILD_DIR}" else echo "using pre-defined build dir: ${ENVOY_BUILD_DIR}" From 33f54b68db8d646c4f7659205b4130f63ab2d3c6 Mon Sep 17 00:00:00 2001 From: Jacob Bohanon Date: Thu, 7 Mar 2024 19:01:16 -0500 Subject: [PATCH 23/37] look at me. i am the captain now. --- bazel/foreign_cc/aws-optref.patch | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/bazel/foreign_cc/aws-optref.patch b/bazel/foreign_cc/aws-optref.patch index d34bea917..234ef194a 100644 --- a/bazel/foreign_cc/aws-optref.patch +++ b/bazel/foreign_cc/aws-optref.patch @@ -1,3 +1,18 @@ +diff --git a/ci/do_ci.sh b/ci/do_ci.sh +index e500dd8792..bfda75e998 100755 +--- ci/do_ci.sh ++++ ci/do_ci.sh +@@ -820,6 +820,10 @@ case $CI_TARGET in + fi + fi + setup_clang_toolchain ++ echo "before: ${ENVOY_BUILD_DIR}" ++ echo "hardcoding ENVOY_BUILD_DIR" ++ ENVOY_BUILD_DIR=/build ++ echo "after: ${ENVOY_BUILD_DIR}" + ENVOY_BINARY_DIR="${ENVOY_BUILD_DIR}/bin" + if [[ -e "${ENVOY_BINARY_DIR}" ]]; then + echo "Existing output directory found (${ENVOY_BINARY_DIR}), removing ..." diff --git a/source/extensions/common/aws/credentials_provider_impl.cc b/source/extensions/common/aws/credentials_provider_impl.cc index 7b9744a87a..619d23a65c 100644 --- source/extensions/common/aws/credentials_provider_impl.cc @@ -7,7 +22,7 @@ index 7b9744a87a..619d23a65c 100644 cache_duration_(getCacheDuration()), debug_name_(absl::StrCat("Fetching aws credentials from cluster=", cluster_name)) { - if (context_) { -+ if (false) { ++ if (context_.has_value()) { context_->mainThreadDispatcher().post([this]() { if (!Utility::addInternalClusterStatic(context_->clusterManager(), cluster_name_, cluster_type_, uri_)) { From bdb84e93aa57844a876be934b65da49e604b29be Mon Sep 17 00:00:00 2001 From: Jacob Bohanon Date: Thu, 7 Mar 2024 19:07:54 -0500 Subject: [PATCH 24/37] shhh, it can't see you if you don't move --- bazel/foreign_cc/aws-optref.patch | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bazel/foreign_cc/aws-optref.patch b/bazel/foreign_cc/aws-optref.patch index 234ef194a..10bf37eae 100644 --- a/bazel/foreign_cc/aws-optref.patch +++ b/bazel/foreign_cc/aws-optref.patch @@ -22,7 +22,7 @@ index 7b9744a87a..619d23a65c 100644 cache_duration_(getCacheDuration()), debug_name_(absl::StrCat("Fetching aws credentials from cluster=", cluster_name)) { - if (context_) { -+ if (context_.has_value()) { ++ if (false) { context_->mainThreadDispatcher().post([this]() { if (!Utility::addInternalClusterStatic(context_->clusterManager(), cluster_name_, cluster_type_, uri_)) { From fa8200adf722027b9b62c3517353aac9e3ba927f Mon Sep 17 00:00:00 2001 From: Jacob Bohanon Date: Thu, 7 Mar 2024 19:17:30 -0500 Subject: [PATCH 25/37] well this is nasty but we're getting somewhere --- bazel/foreign_cc/aws-optref.patch | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bazel/foreign_cc/aws-optref.patch b/bazel/foreign_cc/aws-optref.patch index 10bf37eae..5e5e4c1ae 100644 --- a/bazel/foreign_cc/aws-optref.patch +++ b/bazel/foreign_cc/aws-optref.patch @@ -8,7 +8,7 @@ index e500dd8792..bfda75e998 100755 setup_clang_toolchain + echo "before: ${ENVOY_BUILD_DIR}" + echo "hardcoding ENVOY_BUILD_DIR" -+ ENVOY_BUILD_DIR=/build ++ ENVOY_BUILD_DIR=/build/envoy/x64" + echo "after: ${ENVOY_BUILD_DIR}" ENVOY_BINARY_DIR="${ENVOY_BUILD_DIR}/bin" if [[ -e "${ENVOY_BINARY_DIR}" ]]; then From 591b36c58490069a54cecf0f5e595c337bae24ba Mon Sep 17 00:00:00 2001 From: Jacob Bohanon Date: Thu, 7 Mar 2024 19:21:48 -0500 Subject: [PATCH 26/37] alt+f4 --- bazel/foreign_cc/aws-optref.patch | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bazel/foreign_cc/aws-optref.patch b/bazel/foreign_cc/aws-optref.patch index 5e5e4c1ae..627343a7b 100644 --- a/bazel/foreign_cc/aws-optref.patch +++ b/bazel/foreign_cc/aws-optref.patch @@ -8,7 +8,7 @@ index e500dd8792..bfda75e998 100755 setup_clang_toolchain + echo "before: ${ENVOY_BUILD_DIR}" + echo "hardcoding ENVOY_BUILD_DIR" -+ ENVOY_BUILD_DIR=/build/envoy/x64" ++ ENVOY_BUILD_DIR=/build/envoy/x64 + echo "after: ${ENVOY_BUILD_DIR}" ENVOY_BINARY_DIR="${ENVOY_BUILD_DIR}/bin" if [[ -e "${ENVOY_BINARY_DIR}" ]]; then From d5871c4d00868a4a05f3380d8b693af65982b2cf Mon Sep 17 00:00:00 2001 From: Jacob Bohanon Date: Thu, 7 Mar 2024 19:37:24 -0500 Subject: [PATCH 27/37] asan --- cloudbuild.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cloudbuild.yaml b/cloudbuild.yaml index 463a029b3..5797547c0 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -3,8 +3,8 @@ steps: - name: 'gcr.io/cloud-builders/gcloud' args: ['builds','submit','--config=ci/cloudbuild.yaml','--substitutions','TAG_NAME=$TAG_NAME,COMMIT_SHA=$COMMIT_SHA,_BUILD_TYPE='] -# - name: 'gcr.io/cloud-builders/gcloud' -# args: ['builds','submit','--config=ci/cloudbuild.yaml','--substitutions','COMMIT_SHA=$COMMIT_SHA,_BUILD_TYPE=clang-asan'] -# waitFor: ['-'] +- name: 'gcr.io/cloud-builders/gcloud' + args: ['builds','submit','--config=ci/cloudbuild.yaml','--substitutions','COMMIT_SHA=$COMMIT_SHA,_BUILD_TYPE=clang-asan'] + waitFor: ['-'] timeout: 20000s From 053f304574fe97ca8bde7545a1b5af2a810a6359 Mon Sep 17 00:00:00 2001 From: Jacob Bohanon Date: Thu, 7 Mar 2024 20:13:47 -0500 Subject: [PATCH 28/37] lower jobs oom-killed --- ci/do_ci.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/do_ci.sh b/ci/do_ci.sh index 20b9df50a..4e27422d5 100755 --- a/ci/do_ci.sh +++ b/ci/do_ci.sh @@ -43,7 +43,7 @@ fi export ENVOY_SRCDIR=$SOURCE_DIR # google cloud build times out when using full throttle. -export NUM_CPUS=10 +export NUM_CPUS=5 # google cloud build doesn't like ipv6 export BAZEL_EXTRA_TEST_OPTIONS="--test_env=ENVOY_IP_TEST_VERSIONS=v4only --test_output=errors --jobs=${NUM_CPUS}" From 813325c9385abd3c5c28bd56ca888e257de9c028 Mon Sep 17 00:00:00 2001 From: Jacob Bohanon Date: Thu, 7 Mar 2024 20:47:16 -0500 Subject: [PATCH 29/37] try some more goofiness with the optref --- bazel/foreign_cc/aws-optref.patch | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/bazel/foreign_cc/aws-optref.patch b/bazel/foreign_cc/aws-optref.patch index 627343a7b..756fba984 100644 --- a/bazel/foreign_cc/aws-optref.patch +++ b/bazel/foreign_cc/aws-optref.patch @@ -22,7 +22,8 @@ index 7b9744a87a..619d23a65c 100644 cache_duration_(getCacheDuration()), debug_name_(absl::StrCat("Fetching aws credentials from cluster=", cluster_name)) { - if (context_) { -+ if (false) { ++ if (context_.has_value()) { context_->mainThreadDispatcher().post([this]() { - if (!Utility::addInternalClusterStatic(context_->clusterManager(), cluster_name_, +- if (!Utility::addInternalClusterStatic(context_->clusterManager(), cluster_name_, ++ if (!Utility::addInternalClusterStatic(context_.ref().clusterManager(), cluster_name_, cluster_type_, uri_)) { From 8a44c494b01e4d50c97b366e735a70c16440207a Mon Sep 17 00:00:00 2001 From: Jacob Bohanon Date: Fri, 8 Mar 2024 08:07:47 -0500 Subject: [PATCH 30/37] ci shenanigans --- bazel/foreign_cc/aws-optref.patch | 44 +++++++++++-------------------- ci/do_ci.sh | 2 +- 2 files changed, 16 insertions(+), 30 deletions(-) diff --git a/bazel/foreign_cc/aws-optref.patch b/bazel/foreign_cc/aws-optref.patch index 756fba984..7d3604d20 100644 --- a/bazel/foreign_cc/aws-optref.patch +++ b/bazel/foreign_cc/aws-optref.patch @@ -1,29 +1,15 @@ -diff --git a/ci/do_ci.sh b/ci/do_ci.sh -index e500dd8792..bfda75e998 100755 ---- ci/do_ci.sh -+++ ci/do_ci.sh -@@ -820,6 +820,10 @@ case $CI_TARGET in - fi - fi - setup_clang_toolchain -+ echo "before: ${ENVOY_BUILD_DIR}" -+ echo "hardcoding ENVOY_BUILD_DIR" -+ ENVOY_BUILD_DIR=/build/envoy/x64 -+ echo "after: ${ENVOY_BUILD_DIR}" - ENVOY_BINARY_DIR="${ENVOY_BUILD_DIR}/bin" - if [[ -e "${ENVOY_BINARY_DIR}" ]]; then - echo "Existing output directory found (${ENVOY_BINARY_DIR}), removing ..." -diff --git a/source/extensions/common/aws/credentials_provider_impl.cc b/source/extensions/common/aws/credentials_provider_impl.cc -index 7b9744a87a..619d23a65c 100644 ---- source/extensions/common/aws/credentials_provider_impl.cc -+++ source/extensions/common/aws/credentials_provider_impl.cc -@@ -108,7 +108,7 @@ MetadataCredentialsProviderBase::MetadataCredentialsProviderBase( - cluster_name_(std::string(cluster_name)), cluster_type_(cluster_type), uri_(std::string(uri)), - cache_duration_(getCacheDuration()), - debug_name_(absl::StrCat("Fetching aws credentials from cluster=", cluster_name)) { -- if (context_) { -+ if (context_.has_value()) { - context_->mainThreadDispatcher().post([this]() { -- if (!Utility::addInternalClusterStatic(context_->clusterManager(), cluster_name_, -+ if (!Utility::addInternalClusterStatic(context_.ref().clusterManager(), cluster_name_, - cluster_type_, uri_)) { +# diff --git a/ci/do_ci.sh b/ci/do_ci.sh +# index e500dd8792..bfda75e998 100755 +# --- ci/do_ci.sh +# +++ ci/do_ci.sh +# @@ -820,6 +820,10 @@ case $CI_TARGET in +# fi +# fi +# setup_clang_toolchain +# + echo "before: ${ENVOY_BUILD_DIR}" +# + echo "hardcoding ENVOY_BUILD_DIR" +# + ENVOY_BUILD_DIR=/build/envoy/x64 +# + echo "after: ${ENVOY_BUILD_DIR}" +# ENVOY_BINARY_DIR="${ENVOY_BUILD_DIR}/bin" +# if [[ -e "${ENVOY_BINARY_DIR}" ]]; then +# echo "Existing output directory found (${ENVOY_BINARY_DIR}), removing ..." diff --git a/ci/do_ci.sh b/ci/do_ci.sh index 4e27422d5..27742b0da 100755 --- a/ci/do_ci.sh +++ b/ci/do_ci.sh @@ -69,7 +69,7 @@ echo "BUILD_CONFIG is ${BUILD_CONFIG}" echo "test $BUILD_CONFIG" >> "${SOURCE_DIR}/test.bazelrc" echo Building -bash -x "$UPSTREAM_ENVOY_SRCDIR/ci/do_ci.sh" "$@" +bash -x "ENVOY_BUILD_DIR=${ENVOY_BUILD_DIR}" "$UPSTREAM_ENVOY_SRCDIR/ci/do_ci.sh" "$@" echo Extracting release binaries ENVOY_GLOO_BIN_DIR='linux/amd64/build_envoy_release' From 63d0631fdece56e42c4e55b5b5a589b040794708 Mon Sep 17 00:00:00 2001 From: Jacob Bohanon Date: Fri, 8 Mar 2024 08:08:09 -0500 Subject: [PATCH 31/37] server factory context as nullopt --- .../aws_lambda_filter_config_factory.cc | 5 +- .../aws_lambda_filter_integration_test.cc | 173 +++++++++++------- 2 files changed, 109 insertions(+), 69 deletions(-) diff --git a/source/extensions/filters/http/aws_lambda/aws_lambda_filter_config_factory.cc b/source/extensions/filters/http/aws_lambda/aws_lambda_filter_config_factory.cc index 50e8f5f67..8fd264359 100644 --- a/source/extensions/filters/http/aws_lambda/aws_lambda_filter_config_factory.cc +++ b/source/extensions/filters/http/aws_lambda/aws_lambda_filter_config_factory.cc @@ -19,8 +19,11 @@ AWSLambdaFilterConfigFactory::createFilterFactoryFromProtoTyped( Server::Configuration::FactoryContext &context) { auto& server_context = context.serverFactoryContext(); + // ServerFactoryContext::clusterManager() is not available during server initialization + // therefore we need to pass absl::nullopt in lieu of the server_context to prevent + // the upstream code from attempting to access the method. https://github.com/envoyproxy/envoy/issues/26653 auto chain = std::make_unique( - server_context.api(), makeOptRef(server_context), + server_context.api(), absl::nullopt /* ServerFactoryContextOptRef context */, proto_config.has_service_account_credentials() ? proto_config.service_account_credentials().region() : "TODO(jbohanon) figure this case out", Extensions::Common::Aws::Utility::fetchMetadata); auto sts_factory = StsCredentialsProviderFactory::create(server_context.api(), diff --git a/test/integration/aws_lambda_filter_integration_test.cc b/test/integration/aws_lambda_filter_integration_test.cc index d1e32825f..05890cbc7 100644 --- a/test/integration/aws_lambda_filter_integration_test.cc +++ b/test/integration/aws_lambda_filter_integration_test.cc @@ -76,7 +76,8 @@ class AWSLambdaFilterIntegrationTest */ void initialize() override { scoped_runtime_.mergeValues( - {{"envoy.reloadable_features.use_http_client_to_fetch_aws_credentials", "false"}}); + /* {{"envoy.reloadable_features.use_http_client_to_fetch_aws_credentials", "false"}}); */ + {{"envoy.reloadable_features.use_http_client_to_fetch_aws_credentials", "true"}}); if (add_transform_) { // not sure why but checking the "authorization" in the test succeeds. @@ -189,6 +190,88 @@ name: io.solo.aws_lambda makeHttpConnection(makeClientConnection((lookupPort("http")))); } + void testWithConfig() { + initialize(); + Http::TestRequestHeaderMapImpl request_headers{ + {":method", "POST"}, {":authority", "www.solo.io"}, {":path", "/"}}; + + sendRequestAndWaitForResponse(request_headers, 10, default_response_headers_, + 10); + + EXPECT_NE(0, upstream_request_->headers() + .get(Http::LowerCaseString("authorization"))[0] + ->value() + .size()); + } + + void testWithChain() { + cred_mode_ = CredMode::CHAIN; + initialize(); + Http::TestRequestHeaderMapImpl request_headers{ + {":method", "POST"}, {":authority", "www.solo.io"}, {":path", "/"}}; + + sendRequestAndWaitForResponse(request_headers, 10, default_response_headers_, + 10); + + EXPECT_NE(0, upstream_request_->headers() + .get(Http::LowerCaseString("authorization"))[0] + ->value() + .size()); + EXPECT_EQ(1UL, + test_server_->gauge("http.config_test.aws_lambda.current_state") + ->value()); + EXPECT_EQ(1UL, + test_server_->counter("http.config_test.aws_lambda.creds_rotated") + ->value()); + EXPECT_EQ(0UL, + test_server_->counter("http.config_test.aws_lambda.fetch_failed") + ->value()); + } + + void testWithSTS() { + cred_mode_ = CredMode::STS; + add_transform_ = true; + initialize(); + Http::TestRequestHeaderMapImpl request_headers{ + {":method", "GET"}, {":authority", "www.solo.io"}, {":path", "/"}}; + + IntegrationStreamDecoderPtr response = codec_client_->makeHeaderOnlyRequest(request_headers); + + auto timeout = TestUtility::DefaultTimeout; + + // first request is sts request; return sts response. + waitForNextUpstreamRequest(0, timeout); + upstream_request_->encodeHeaders(default_response_headers_, false); + upstream_request_->encodeData(VALID_CHAINED_RESPONSE, true); + + // second upstream request is the "lambda" request. + waitForNextUpstreamRequest(0, timeout); + + // make sure we have a body (i.e. make sure transformation filter worked). + std::string body = upstream_request_->body().toString(); + EXPECT_EQ(body, "abc /"); + + // make sure that the transformation filter after the lambda was called and observed the authorization header: + // ** THIS IS THE MANIFESTATION OF THE BUG ** + const auto& auth_header = upstream_request_->headers() + .get(Http::LowerCaseString("x-authorization"))[0] + ->value(); + EXPECT_NE(0, auth_header.size()); + + // wrap up the test nicely: + upstream_request_->encodeHeaders(default_response_headers_, false); + upstream_request_->encodeData(10, true); + + // Wait for the response to be read by the codec client. + RELEASE_ASSERT(response->waitForEndStream(timeout), "unexpected timeout"); + } + + void setUseHttpClient(bool use) { + std::string useStr = use ? "true" : "false"; + scoped_runtime_.mergeValues( + {{"envoy.reloadable_features.use_http_client_to_fetch_aws_credentials", useStr}}); + } + enum class CredMode{DEFAULT, CHAIN, STS}; CredMode cred_mode_{}; bool add_transform_{}; @@ -199,80 +282,34 @@ INSTANTIATE_TEST_SUITE_P( IpVersions, AWSLambdaFilterIntegrationTest, testing::ValuesIn(TestEnvironment::getIpVersionsForTest())); -TEST_P(AWSLambdaFilterIntegrationTest, TestWithConfig) { - initialize(); - Http::TestRequestHeaderMapImpl request_headers{ - {":method", "POST"}, {":authority", "www.solo.io"}, {":path", "/"}}; - - sendRequestAndWaitForResponse(request_headers, 10, default_response_headers_, - 10); - - EXPECT_NE(0, upstream_request_->headers() - .get(Http::LowerCaseString("authorization"))[0] - ->value() - .size()); +TEST_P(AWSLambdaFilterIntegrationTest, TestWithConfigLibcurl) { + setUseHttpClient(false); + testWithConfig(); } -TEST_P(AWSLambdaFilterIntegrationTest, TestWithChain) { - cred_mode_ = CredMode::CHAIN; - initialize(); - Http::TestRequestHeaderMapImpl request_headers{ - {":method", "POST"}, {":authority", "www.solo.io"}, {":path", "/"}}; - - sendRequestAndWaitForResponse(request_headers, 10, default_response_headers_, - 10); - - EXPECT_NE(0, upstream_request_->headers() - .get(Http::LowerCaseString("authorization"))[0] - ->value() - .size()); - EXPECT_EQ(1UL, - test_server_->gauge("http.config_test.aws_lambda.current_state") - ->value()); - EXPECT_EQ(1UL, - test_server_->counter("http.config_test.aws_lambda.creds_rotated") - ->value()); - EXPECT_EQ(0UL, - test_server_->counter("http.config_test.aws_lambda.fetch_failed") - ->value()); +TEST_P(AWSLambdaFilterIntegrationTest, TestWithConfigHttpClient) { + setUseHttpClient(true); + testWithConfig(); } -TEST_P(AWSLambdaFilterIntegrationTest, TestWithSTS) { - cred_mode_ = CredMode::STS; - add_transform_ = true; - initialize(); - Http::TestRequestHeaderMapImpl request_headers{ - {":method", "GET"}, {":authority", "www.solo.io"}, {":path", "/"}}; - - IntegrationStreamDecoderPtr response = codec_client_->makeHeaderOnlyRequest(request_headers); - - auto timeout = TestUtility::DefaultTimeout; - - // first request is sts request; return sts response. - waitForNextUpstreamRequest(0, timeout); - upstream_request_->encodeHeaders(default_response_headers_, false); - upstream_request_->encodeData(VALID_CHAINED_RESPONSE, true); - - // second upstream request is the "lambda" request. - waitForNextUpstreamRequest(0, timeout); - - // make sure we have a body (i.e. make sure transformation filter worked). - std::string body = upstream_request_->body().toString(); - EXPECT_EQ(body, "abc /"); +TEST_P(AWSLambdaFilterIntegrationTest, TestWithChainLibcurl) { + setUseHttpClient(false); + testWithChain(); +} - // make sure that the transformation filter after the lambda was called and observed the authorization header: - // ** THIS IS THE MANIFESTATION OF THE BUG ** - const auto& auth_header = upstream_request_->headers() - .get(Http::LowerCaseString("x-authorization"))[0] - ->value(); - EXPECT_NE(0, auth_header.size()); +TEST_P(AWSLambdaFilterIntegrationTest, TestWithChainHttpClient) { + setUseHttpClient(true); + testWithChain(); +} - // wrap up the test nicely: - upstream_request_->encodeHeaders(default_response_headers_, false); - upstream_request_->encodeData(10, true); +TEST_P(AWSLambdaFilterIntegrationTest, TestWithSTSLibcurl) { + setUseHttpClient(false); + testWithSTS(); +} - // Wait for the response to be read by the codec client. - RELEASE_ASSERT(response->waitForEndStream(timeout), "unexpected timeout"); +TEST_P(AWSLambdaFilterIntegrationTest, TestWithSTSHttpClient) { + setUseHttpClient(true); + testWithSTS(); } From 97346fd1d962626cc0faefd708cb2311686aa231 Mon Sep 17 00:00:00 2001 From: Jacob Bohanon Date: Fri, 8 Mar 2024 08:12:52 -0500 Subject: [PATCH 32/37] ci nonsense --- ci/do_ci.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/do_ci.sh b/ci/do_ci.sh index 27742b0da..9227162dc 100755 --- a/ci/do_ci.sh +++ b/ci/do_ci.sh @@ -69,7 +69,7 @@ echo "BUILD_CONFIG is ${BUILD_CONFIG}" echo "test $BUILD_CONFIG" >> "${SOURCE_DIR}/test.bazelrc" echo Building -bash -x "ENVOY_BUILD_DIR=${ENVOY_BUILD_DIR}" "$UPSTREAM_ENVOY_SRCDIR/ci/do_ci.sh" "$@" +bash -xc "ENVOY_BUILD_DIR=${ENVOY_BUILD_DIR} $UPSTREAM_ENVOY_SRCDIR/ci/do_ci.sh $@" echo Extracting release binaries ENVOY_GLOO_BIN_DIR='linux/amd64/build_envoy_release' From 46d8ac215fd0d9eb426c5f97945ccf55427ac8d3 Mon Sep 17 00:00:00 2001 From: Jacob Bohanon Date: Fri, 8 Mar 2024 08:21:21 -0500 Subject: [PATCH 33/37] getting closer --- ci/do_ci.sh | 21 ++++++--------------- 1 file changed, 6 insertions(+), 15 deletions(-) diff --git a/ci/do_ci.sh b/ci/do_ci.sh index 9227162dc..144e0bd38 100755 --- a/ci/do_ci.sh +++ b/ci/do_ci.sh @@ -1,22 +1,13 @@ #!/bin/bash set -e -export ENVOY_DOCKER_BUILD_DIR -export ENVOY_BUILD_DIR -if [ -z "$ENVOY_DOCKER_BUILD_DIR" ]; then - ENVOY_DOCKER_BUILD_DIR=/build - echo "using docker build dir: ${ENVOY_DOCKER_BUILD_DIR}" +if [ -z "$BUILD_DIR" ]; then + BUILD_DIR=/build + echo "using build dir: ${BUILD_DIR}" else - echo "using pre-defined docker build dir: ${ENVOY_DOCKER_BUILD_DIR}" + echo "using pre-defined build dir: ${BUILD_DIR}" fi -if [ -z "$ENVOY_BUILD_DIR" ]; then - ENVOY_BUILD_DIR=/build/envoy/x64 - echo "using build dir: ${ENVOY_BUILD_DIR}" -else - echo "using pre-defined build dir: ${ENVOY_BUILD_DIR}" -fi -rm -rf "$ENVOY_DOCKER_BUILD_DIR/envoy/x64/bin/" -mkdir -p ${ENVOY_BUILD_DIR} +export BUILD_DIR bazel fetch //source/exe:envoy-static @@ -69,7 +60,7 @@ echo "BUILD_CONFIG is ${BUILD_CONFIG}" echo "test $BUILD_CONFIG" >> "${SOURCE_DIR}/test.bazelrc" echo Building -bash -xc "ENVOY_BUILD_DIR=${ENVOY_BUILD_DIR} $UPSTREAM_ENVOY_SRCDIR/ci/do_ci.sh $@" +bash -x "$UPSTREAM_ENVOY_SRCDIR/ci/do_ci.sh" "$@" echo Extracting release binaries ENVOY_GLOO_BIN_DIR='linux/amd64/build_envoy_release' From 7d2d775f0f45fd2ff53ffb365017eefedf92a055 Mon Sep 17 00:00:00 2001 From: Jacob Bohanon Date: Fri, 8 Mar 2024 10:14:52 -0500 Subject: [PATCH 34/37] cleanup --- bazel/foreign_cc/aws-optref.patch | 15 --------------- bazel/repositories.bzl | 8 +------- ci/do_ci.sh | 4 ---- 3 files changed, 1 insertion(+), 26 deletions(-) delete mode 100644 bazel/foreign_cc/aws-optref.patch diff --git a/bazel/foreign_cc/aws-optref.patch b/bazel/foreign_cc/aws-optref.patch deleted file mode 100644 index 7d3604d20..000000000 --- a/bazel/foreign_cc/aws-optref.patch +++ /dev/null @@ -1,15 +0,0 @@ -# diff --git a/ci/do_ci.sh b/ci/do_ci.sh -# index e500dd8792..bfda75e998 100755 -# --- ci/do_ci.sh -# +++ ci/do_ci.sh -# @@ -820,6 +820,10 @@ case $CI_TARGET in -# fi -# fi -# setup_clang_toolchain -# + echo "before: ${ENVOY_BUILD_DIR}" -# + echo "hardcoding ENVOY_BUILD_DIR" -# + ENVOY_BUILD_DIR=/build/envoy/x64 -# + echo "after: ${ENVOY_BUILD_DIR}" -# ENVOY_BINARY_DIR="${ENVOY_BUILD_DIR}/bin" -# if [[ -e "${ENVOY_BINARY_DIR}" ]]; then -# echo "Existing output directory found (${ENVOY_BINARY_DIR}), removing ..." diff --git a/bazel/repositories.bzl b/bazel/repositories.bzl index a5215d96b..94d406888 100644 --- a/bazel/repositories.bzl +++ b/bazel/repositories.bzl @@ -66,12 +66,6 @@ def _repository_impl(name, **kwargs): ) def envoy_gloo_dependencies(): - # the following 2 patches are needed to support the deprecated cipher - # passthrough and only need to be backported onto envoy v1.25.x - # these should be removed when moving to v1.26.x since this code exists in - # upstream at that point. - _repository_impl("envoy", patches = [ - "@envoy_gloo//bazel/foreign_cc:aws-optref.patch", - ]) + _repository_impl("envoy") _repository_impl("json", build_file = "@envoy_gloo//bazel/external:json.BUILD") _repository_impl("inja", build_file = "@envoy_gloo//bazel/external:inja.BUILD") diff --git a/ci/do_ci.sh b/ci/do_ci.sh index 144e0bd38..ab5d16955 100755 --- a/ci/do_ci.sh +++ b/ci/do_ci.sh @@ -65,10 +65,6 @@ bash -x "$UPSTREAM_ENVOY_SRCDIR/ci/do_ci.sh" "$@" echo Extracting release binaries ENVOY_GLOO_BIN_DIR='linux/amd64/build_envoy_release' mkdir -p "$ENVOY_GLOO_BIN_DIR" -ls -al /build/ -ls -al /build/envoy/ -ls -al /build/envoy/x64/ -ls -al /build/envoy/x64/bin/ bazel run @envoy//tools/zstd:zstd -- --stdout -d /build/envoy/x64/bin/release.tar.zst \ | tar xfO - envoy > "$ENVOY_GLOO_BIN_DIR/envoy" From 0cc9bb25a35f3fde4feb8305bcb1e1d3ce5f793e Mon Sep 17 00:00:00 2001 From: Jacob Bohanon Date: Fri, 8 Mar 2024 10:33:39 -0500 Subject: [PATCH 35/37] remove hard-coding override in initialize() --- test/integration/aws_lambda_filter_integration_test.cc | 4 ---- 1 file changed, 4 deletions(-) diff --git a/test/integration/aws_lambda_filter_integration_test.cc b/test/integration/aws_lambda_filter_integration_test.cc index 05890cbc7..a7ea8eb36 100644 --- a/test/integration/aws_lambda_filter_integration_test.cc +++ b/test/integration/aws_lambda_filter_integration_test.cc @@ -75,10 +75,6 @@ class AWSLambdaFilterIntegrationTest * Initializer for an individual integration test. */ void initialize() override { - scoped_runtime_.mergeValues( - /* {{"envoy.reloadable_features.use_http_client_to_fetch_aws_credentials", "false"}}); */ - {{"envoy.reloadable_features.use_http_client_to_fetch_aws_credentials", "true"}}); - if (add_transform_) { // not sure why but checking the "authorization" in the test succeeds. // what i really want to test is that the filter that follows the aws filter gets the auth header From 6bfe2a58a4405a0657451593854a67a41c81a326 Mon Sep 17 00:00:00 2001 From: Jacob Bohanon Date: Fri, 8 Mar 2024 12:39:11 -0500 Subject: [PATCH 36/37] bump to 1.29.2 --- bazel/repository_locations.bzl | 4 ++-- .../http/aws_lambda/aws_lambda_filter_config_factory.cc | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index dc8347a1a..bae85e423 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -1,7 +1,7 @@ REPOSITORY_LOCATIONS = dict( envoy = dict( - # envoy 1.29.1 with backported ext_proc updates - commit = "f52f0b37a461975e9053b198375a347628dbbd7d", + # envoy 1.29.2 with backported ext_proc updates + commit = "18eb48c482dce77f55de274a2435d6f098237511", remote = "https://github.com/solo-io/envoy-fork", ), inja = dict( diff --git a/source/extensions/filters/http/aws_lambda/aws_lambda_filter_config_factory.cc b/source/extensions/filters/http/aws_lambda/aws_lambda_filter_config_factory.cc index 8fd264359..d256661fe 100644 --- a/source/extensions/filters/http/aws_lambda/aws_lambda_filter_config_factory.cc +++ b/source/extensions/filters/http/aws_lambda/aws_lambda_filter_config_factory.cc @@ -24,7 +24,8 @@ AWSLambdaFilterConfigFactory::createFilterFactoryFromProtoTyped( // the upstream code from attempting to access the method. https://github.com/envoyproxy/envoy/issues/26653 auto chain = std::make_unique( server_context.api(), absl::nullopt /* ServerFactoryContextOptRef context */, - proto_config.has_service_account_credentials() ? proto_config.service_account_credentials().region() : "TODO(jbohanon) figure this case out", + // We pass an empty string if we don't have a region + proto_config.has_service_account_credentials() ? proto_config.service_account_credentials().region() : "", Extensions::Common::Aws::Utility::fetchMetadata); auto sts_factory = StsCredentialsProviderFactory::create(server_context.api(), server_context.clusterManager()); From f3ba6327fb491947999a05ca5c072a84a6f651c3 Mon Sep 17 00:00:00 2001 From: Jacob Bohanon Date: Tue, 12 Mar 2024 14:54:46 -0400 Subject: [PATCH 37/37] bump to merged --- bazel/repository_locations.bzl | 2 +- changelog/{v1.29.1-patch1 => v1.29.2-patch1}/bump-envoy.yaml | 4 ++-- ci/do_ci.sh | 3 ++- 3 files changed, 5 insertions(+), 4 deletions(-) rename changelog/{v1.29.1-patch1 => v1.29.2-patch1}/bump-envoy.yaml (65%) diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index bae85e423..1eb6e0425 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -1,7 +1,7 @@ REPOSITORY_LOCATIONS = dict( envoy = dict( # envoy 1.29.2 with backported ext_proc updates - commit = "18eb48c482dce77f55de274a2435d6f098237511", + commit = "b5c77750dc0f9de19478c89776f580fb9336fc99", remote = "https://github.com/solo-io/envoy-fork", ), inja = dict( diff --git a/changelog/v1.29.1-patch1/bump-envoy.yaml b/changelog/v1.29.2-patch1/bump-envoy.yaml similarity index 65% rename from changelog/v1.29.1-patch1/bump-envoy.yaml rename to changelog/v1.29.2-patch1/bump-envoy.yaml index 21f92a5ec..8e04fbe89 100644 --- a/changelog/v1.29.1-patch1/bump-envoy.yaml +++ b/changelog/v1.29.2-patch1/bump-envoy.yaml @@ -2,7 +2,7 @@ changelog: - type: DEPENDENCY_BUMP dependencyOwner: solo-io dependencyRepo: envoy-fork - dependencyTag: v1.29.1 + dependencyTag: v1.29.2 resolvesIssue: false description: >- - Update Envoy to latest from forked 1.29.1 + Update Envoy to latest from forked 1.29.2 diff --git a/ci/do_ci.sh b/ci/do_ci.sh index ab5d16955..63a15bf4f 100755 --- a/ci/do_ci.sh +++ b/ci/do_ci.sh @@ -34,7 +34,8 @@ fi export ENVOY_SRCDIR=$SOURCE_DIR # google cloud build times out when using full throttle. -export NUM_CPUS=5 +# additionally, we see builds killed due to OOM at high concurrency. +export NUM_CPUS=10 # google cloud build doesn't like ipv6 export BAZEL_EXTRA_TEST_OPTIONS="--test_env=ENVOY_IP_TEST_VERSIONS=v4only --test_output=errors --jobs=${NUM_CPUS}"