diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 64ff05d8f4a8..748cc2119c1a 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -617,6 +617,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Libbeat: report beat version to monitoring. {pull}26214[26214] - Ensure common proxy settings support in HTTP clients: proxy_disabled, proxy_url, proxy_headers and typical environment variables HTTP_PROXY, HTTPS_PROXY, NOPROXY. {pull}25219[25219] - `add_process_metadata` processor enrich process information with owner name and id. {issue}21068[21068] {pull}21111[21111] +- Add proxy support for AWS functions. {pull}26832[26832] *Auditbeat* diff --git a/filebeat/docs/modules/aws.asciidoc b/filebeat/docs/modules/aws.asciidoc index eadfabf30552..bb8269933a4c 100644 --- a/filebeat/docs/modules/aws.asciidoc +++ b/filebeat/docs/modules/aws.asciidoc @@ -53,6 +53,7 @@ Example config: #var.api_timeout: 120s #var.endpoint: amazonaws.com #var.role_arn: arn:aws:iam::123456789012:role/test-mb + #var.proxy_url: http://proxy:8080 cloudwatch: enabled: false @@ -66,6 +67,7 @@ Example config: #var.api_timeout: 120s #var.endpoint: amazonaws.com #var.role_arn: arn:aws:iam::123456789012:role/test-mb + #var.proxy_url: http://proxy:8080 ec2: enabled: false @@ -79,6 +81,7 @@ Example config: #var.api_timeout: 120s #var.endpoint: amazonaws.com #var.role_arn: arn:aws:iam::123456789012:role/test-mb + #var.proxy_url: http://proxy:8080 elb: enabled: false @@ -92,6 +95,7 @@ Example config: #var.api_timeout: 120s #var.endpoint: amazonaws.com #var.role_arn: arn:aws:iam::123456789012:role/test-mb + #var.proxy_url: http://proxy:8080 s3access: enabled: false @@ -105,6 +109,7 @@ Example config: #var.api_timeout: 120s #var.endpoint: amazonaws.com #var.role_arn: arn:aws:iam::123456789012:role/test-mb + #var.proxy_url: http://proxy:8080 vpcflow: enabled: false @@ -118,6 +123,7 @@ Example config: #var.api_timeout: 120s #var.endpoint: amazonaws.com #var.role_arn: arn:aws:iam::123456789012:role/test-mb + #var.proxy_url: http://proxy:8080 ---- *`var.queue_url`*:: diff --git a/libbeat/common/transport/httpcommon/proxy.go b/libbeat/common/transport/httpcommon/proxy.go index 40275d04af55..c63b11716c2e 100644 --- a/libbeat/common/transport/httpcommon/proxy.go +++ b/libbeat/common/transport/httpcommon/proxy.go @@ -28,7 +28,7 @@ import ( // // Proxy usage will be disabled in general if Disable is set. // If URL is not set, the proxy configuration will default -// to HTTP_PROXY, HTTPS_PPROXY, and NO_PROXY. +// to HTTP_PROXY, HTTPS_PROXY, and NO_PROXY. // // The default (and zero) value of HTTPClientProxySettings has Proxy support // enabled, and will select the proxy per URL based on the environment variables. diff --git a/x-pack/filebeat/input/awscloudwatch/input.go b/x-pack/filebeat/input/awscloudwatch/input.go index 105b0e99beb6..ca8ff57c770d 100644 --- a/x-pack/filebeat/input/awscloudwatch/input.go +++ b/x-pack/filebeat/input/awscloudwatch/input.go @@ -103,9 +103,9 @@ func NewInput(cfg *common.Config, connector channel.Connector, context input.Con config.RegionName = regionName } - awsConfig, err := awscommon.GetAWSCredentials(config.AwsConfig) + awsConfig, err := awscommon.InitializeAWSConfig(config.AwsConfig) if err != nil { - return nil, errors.Wrap(err, "getAWSCredentials failed") + return nil, errors.Wrap(err, "InitializeAWSConfig failed") } awsConfig.Region = config.RegionName diff --git a/x-pack/filebeat/input/awss3/input.go b/x-pack/filebeat/input/awss3/input.go index 1c1572c7d9ee..3d09a1527cc9 100644 --- a/x-pack/filebeat/input/awss3/input.go +++ b/x-pack/filebeat/input/awss3/input.go @@ -54,9 +54,9 @@ func newInput(config config) (*s3Input, error) { func (in *s3Input) Name() string { return inputName } func (in *s3Input) Test(ctx v2.TestContext) error { - _, err := awscommon.GetAWSCredentials(in.config.AWSConfig) + _, err := awscommon.InitializeAWSConfig(in.config.AWSConfig) if err != nil { - return fmt.Errorf("getAWSCredentials failed: %w", err) + return fmt.Errorf("InitializeAWSConfig failed: %w", err) } return nil } @@ -98,9 +98,9 @@ func (in *s3Input) createCollector(ctx v2.Context, pipeline beat.Pipeline) (*s3C log = log.With("region", regionName) } - awsConfig, err := awscommon.GetAWSCredentials(in.config.AWSConfig) + awsConfig, err := awscommon.InitializeAWSConfig(in.config.AWSConfig) if err != nil { - return nil, fmt.Errorf("getAWSCredentials failed: %w", err) + return nil, fmt.Errorf("InitializeAWSConfig failed: %w", err) } awsConfig.Region = regionName diff --git a/x-pack/filebeat/input/awss3/s3_integration_test.go b/x-pack/filebeat/input/awss3/s3_integration_test.go index 4966bed60841..59c3f1e19481 100644 --- a/x-pack/filebeat/input/awss3/s3_integration_test.go +++ b/x-pack/filebeat/input/awss3/s3_integration_test.go @@ -138,9 +138,9 @@ func setupCollector(t *testing.T, cfg *common.Config, mock bool) (*s3Collector, } config := getConfigForTest(t) - awsConfig, err := awscommon.GetAWSCredentials(config.AWSConfig) + awsConfig, err := awscommon.InitializeAWSConfig(config.AWSConfig) if err != nil { - t.Fatal("failed GetAWSCredentials with AWS Config: ", err) + t.Fatal("failed InitializeAWSConfig with AWS Config: ", err) } s3BucketRegion := os.Getenv("S3_BUCKET_REGION") diff --git a/x-pack/filebeat/module/aws/_meta/docs.asciidoc b/x-pack/filebeat/module/aws/_meta/docs.asciidoc index 04e0ff33cfe8..4cd9482486a1 100644 --- a/x-pack/filebeat/module/aws/_meta/docs.asciidoc +++ b/x-pack/filebeat/module/aws/_meta/docs.asciidoc @@ -48,6 +48,7 @@ Example config: #var.api_timeout: 120s #var.endpoint: amazonaws.com #var.role_arn: arn:aws:iam::123456789012:role/test-mb + #var.proxy_url: http://proxy:8080 cloudwatch: enabled: false @@ -61,6 +62,7 @@ Example config: #var.api_timeout: 120s #var.endpoint: amazonaws.com #var.role_arn: arn:aws:iam::123456789012:role/test-mb + #var.proxy_url: http://proxy:8080 ec2: enabled: false @@ -74,6 +76,7 @@ Example config: #var.api_timeout: 120s #var.endpoint: amazonaws.com #var.role_arn: arn:aws:iam::123456789012:role/test-mb + #var.proxy_url: http://proxy:8080 elb: enabled: false @@ -87,6 +90,7 @@ Example config: #var.api_timeout: 120s #var.endpoint: amazonaws.com #var.role_arn: arn:aws:iam::123456789012:role/test-mb + #var.proxy_url: http://proxy:8080 s3access: enabled: false @@ -100,6 +104,7 @@ Example config: #var.api_timeout: 120s #var.endpoint: amazonaws.com #var.role_arn: arn:aws:iam::123456789012:role/test-mb + #var.proxy_url: http://proxy:8080 vpcflow: enabled: false @@ -113,6 +118,7 @@ Example config: #var.api_timeout: 120s #var.endpoint: amazonaws.com #var.role_arn: arn:aws:iam::123456789012:role/test-mb + #var.proxy_url: http://proxy:8080 ---- *`var.queue_url`*:: diff --git a/x-pack/filebeat/module/aws/cloudtrail/config/aws-s3.yml b/x-pack/filebeat/module/aws/cloudtrail/config/aws-s3.yml index fb1054d9863f..4daf262994bf 100644 --- a/x-pack/filebeat/module/aws/cloudtrail/config/aws-s3.yml +++ b/x-pack/filebeat/module/aws/cloudtrail/config/aws-s3.yml @@ -59,6 +59,10 @@ fips_enabled: {{ .fips_enabled }} max_number_of_messages: {{ .max_number_of_messages }} {{ end }} +{{ if .proxy_url }} +proxy_url: {{ .proxy_url }} +{{ end }} + tags: {{.tags | tojson}} publisher_pipeline.disable_host: {{ inList .tags "forwarded" }} diff --git a/x-pack/filebeat/module/aws/cloudtrail/manifest.yml b/x-pack/filebeat/module/aws/cloudtrail/manifest.yml index bad63e1224b6..1903ee34f251 100644 --- a/x-pack/filebeat/module/aws/cloudtrail/manifest.yml +++ b/x-pack/filebeat/module/aws/cloudtrail/manifest.yml @@ -22,6 +22,7 @@ var: - name: process_insight_logs default: true - name: fips_enabled + - name: proxy_url - name: max_number_of_messages ingest_pipeline: ingest/pipeline.yml diff --git a/x-pack/filebeat/module/aws/cloudwatch/config/aws-s3.yml b/x-pack/filebeat/module/aws/cloudwatch/config/aws-s3.yml index 9d0605877da0..b0fb5feed0c5 100644 --- a/x-pack/filebeat/module/aws/cloudwatch/config/aws-s3.yml +++ b/x-pack/filebeat/module/aws/cloudwatch/config/aws-s3.yml @@ -45,6 +45,10 @@ fips_enabled: {{ .fips_enabled }} max_number_of_messages: {{ .max_number_of_messages }} {{ end }} +{{ if .proxy_url }} +proxy_url: {{ .proxy_url }} +{{ end }} + tags: {{.tags | tojson}} publisher_pipeline.disable_host: {{ inList .tags "forwarded" }} diff --git a/x-pack/filebeat/module/aws/cloudwatch/manifest.yml b/x-pack/filebeat/module/aws/cloudwatch/manifest.yml index ca3a74dadd37..84f672107c63 100644 --- a/x-pack/filebeat/module/aws/cloudwatch/manifest.yml +++ b/x-pack/filebeat/module/aws/cloudwatch/manifest.yml @@ -16,6 +16,7 @@ var: - name: tags default: [forwarded] - name: fips_enabled + - name: proxy_url - name: max_number_of_messages ingest_pipeline: ingest/pipeline.yml diff --git a/x-pack/filebeat/module/aws/ec2/config/aws-s3.yml b/x-pack/filebeat/module/aws/ec2/config/aws-s3.yml index 9d0605877da0..b0fb5feed0c5 100644 --- a/x-pack/filebeat/module/aws/ec2/config/aws-s3.yml +++ b/x-pack/filebeat/module/aws/ec2/config/aws-s3.yml @@ -45,6 +45,10 @@ fips_enabled: {{ .fips_enabled }} max_number_of_messages: {{ .max_number_of_messages }} {{ end }} +{{ if .proxy_url }} +proxy_url: {{ .proxy_url }} +{{ end }} + tags: {{.tags | tojson}} publisher_pipeline.disable_host: {{ inList .tags "forwarded" }} diff --git a/x-pack/filebeat/module/aws/ec2/manifest.yml b/x-pack/filebeat/module/aws/ec2/manifest.yml index ca3a74dadd37..84f672107c63 100644 --- a/x-pack/filebeat/module/aws/ec2/manifest.yml +++ b/x-pack/filebeat/module/aws/ec2/manifest.yml @@ -16,6 +16,7 @@ var: - name: tags default: [forwarded] - name: fips_enabled + - name: proxy_url - name: max_number_of_messages ingest_pipeline: ingest/pipeline.yml diff --git a/x-pack/filebeat/module/aws/elb/config/aws-s3.yml b/x-pack/filebeat/module/aws/elb/config/aws-s3.yml index 9d0605877da0..b0fb5feed0c5 100644 --- a/x-pack/filebeat/module/aws/elb/config/aws-s3.yml +++ b/x-pack/filebeat/module/aws/elb/config/aws-s3.yml @@ -45,6 +45,10 @@ fips_enabled: {{ .fips_enabled }} max_number_of_messages: {{ .max_number_of_messages }} {{ end }} +{{ if .proxy_url }} +proxy_url: {{ .proxy_url }} +{{ end }} + tags: {{.tags | tojson}} publisher_pipeline.disable_host: {{ inList .tags "forwarded" }} diff --git a/x-pack/filebeat/module/aws/elb/manifest.yml b/x-pack/filebeat/module/aws/elb/manifest.yml index 54fa469d7015..735591632349 100644 --- a/x-pack/filebeat/module/aws/elb/manifest.yml +++ b/x-pack/filebeat/module/aws/elb/manifest.yml @@ -16,6 +16,7 @@ var: - name: tags default: [forwarded] - name: fips_enabled + - name: proxy_url - name: max_number_of_messages ingest_pipeline: ingest/pipeline.yml diff --git a/x-pack/filebeat/module/aws/s3access/config/aws-s3.yml b/x-pack/filebeat/module/aws/s3access/config/aws-s3.yml index 9d0605877da0..b0fb5feed0c5 100644 --- a/x-pack/filebeat/module/aws/s3access/config/aws-s3.yml +++ b/x-pack/filebeat/module/aws/s3access/config/aws-s3.yml @@ -45,6 +45,10 @@ fips_enabled: {{ .fips_enabled }} max_number_of_messages: {{ .max_number_of_messages }} {{ end }} +{{ if .proxy_url }} +proxy_url: {{ .proxy_url }} +{{ end }} + tags: {{.tags | tojson}} publisher_pipeline.disable_host: {{ inList .tags "forwarded" }} diff --git a/x-pack/filebeat/module/aws/s3access/manifest.yml b/x-pack/filebeat/module/aws/s3access/manifest.yml index ca3a74dadd37..84f672107c63 100644 --- a/x-pack/filebeat/module/aws/s3access/manifest.yml +++ b/x-pack/filebeat/module/aws/s3access/manifest.yml @@ -16,6 +16,7 @@ var: - name: tags default: [forwarded] - name: fips_enabled + - name: proxy_url - name: max_number_of_messages ingest_pipeline: ingest/pipeline.yml diff --git a/x-pack/filebeat/module/aws/vpcflow/config/input.yml b/x-pack/filebeat/module/aws/vpcflow/config/input.yml index 54b45591f791..8fb86aee8725 100644 --- a/x-pack/filebeat/module/aws/vpcflow/config/input.yml +++ b/x-pack/filebeat/module/aws/vpcflow/config/input.yml @@ -47,6 +47,10 @@ fips_enabled: {{ .fips_enabled }} max_number_of_messages: {{ .max_number_of_messages }} {{ end }} +{{ if .proxy_url }} +proxy_url: {{ .proxy_url }} +{{ end }} + {{ else if eq .input "file" }} type: log diff --git a/x-pack/filebeat/module/aws/vpcflow/manifest.yml b/x-pack/filebeat/module/aws/vpcflow/manifest.yml index d084692d5c45..0c2ec0f7e1b4 100644 --- a/x-pack/filebeat/module/aws/vpcflow/manifest.yml +++ b/x-pack/filebeat/module/aws/vpcflow/manifest.yml @@ -16,6 +16,7 @@ var: - name: tags default: [forwarded] - name: fips_enabled + - name: proxy_url - name: max_number_of_messages ingest_pipeline: ingest/pipeline.yml diff --git a/x-pack/functionbeat/manager/aws/cli_manager.go b/x-pack/functionbeat/manager/aws/cli_manager.go index a19cb22823db..a697c703ff02 100644 --- a/x-pack/functionbeat/manager/aws/cli_manager.go +++ b/x-pack/functionbeat/manager/aws/cli_manager.go @@ -214,7 +214,7 @@ func NewCLI( if err := cfg.Unpack(config); err != nil { return nil, err } - awsCfg, err := awscommon.GetAWSCredentials(config.Credentials) + awsCfg, err := awscommon.InitializeAWSConfig(config.Credentials) if err != nil { return nil, fmt.Errorf("failed to get aws credentials, please check AWS credential in config: %+v", err) } diff --git a/x-pack/libbeat/autodiscover/providers/aws/ec2/provider.go b/x-pack/libbeat/autodiscover/providers/aws/ec2/provider.go index 029a54d5403b..19d8c8a9784f 100644 --- a/x-pack/libbeat/autodiscover/providers/aws/ec2/provider.go +++ b/x-pack/libbeat/autodiscover/providers/aws/ec2/provider.go @@ -52,7 +52,7 @@ func AutodiscoverBuilder( return nil, err } - awsCfg, err := awscommon.GetAWSCredentials( + awsCfg, err := awscommon.InitializeAWSConfig( awscommon.ConfigAWS{ AccessKeyID: config.AWSConfig.AccessKeyID, SecretAccessKey: config.AWSConfig.SecretAccessKey, diff --git a/x-pack/libbeat/autodiscover/providers/aws/elb/provider.go b/x-pack/libbeat/autodiscover/providers/aws/elb/provider.go index 03d76c2e7f32..39313f368710 100644 --- a/x-pack/libbeat/autodiscover/providers/aws/elb/provider.go +++ b/x-pack/libbeat/autodiscover/providers/aws/elb/provider.go @@ -54,7 +54,7 @@ func AutodiscoverBuilder( return nil, err } - awsCfg, err := awscommon.GetAWSCredentials(awscommon.ConfigAWS{ + awsCfg, err := awscommon.InitializeAWSConfig(awscommon.ConfigAWS{ AccessKeyID: config.AWSConfig.AccessKeyID, SecretAccessKey: config.AWSConfig.SecretAccessKey, SessionToken: config.AWSConfig.SessionToken, @@ -76,7 +76,7 @@ func AutodiscoverBuilder( var clients []elasticloadbalancingv2iface.ClientAPI for _, region := range config.Regions { - awsCfg, err := awscommon.GetAWSCredentials(awscommon.ConfigAWS{ + awsCfg, err := awscommon.InitializeAWSConfig(awscommon.ConfigAWS{ AccessKeyID: config.AWSConfig.AccessKeyID, SecretAccessKey: config.AWSConfig.SecretAccessKey, SessionToken: config.AWSConfig.SessionToken, diff --git a/x-pack/libbeat/common/aws/credentials.go b/x-pack/libbeat/common/aws/credentials.go index 6c7e61c0f632..cae5cb1b6b51 100644 --- a/x-pack/libbeat/common/aws/credentials.go +++ b/x-pack/libbeat/common/aws/credentials.go @@ -5,6 +5,9 @@ package aws import ( + "net/http" + "net/url" + awssdk "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/aws/defaults" "github.com/aws/aws-sdk-go-v2/aws/external" @@ -18,14 +21,29 @@ import ( // ConfigAWS is a structure defined for AWS credentials type ConfigAWS struct { - AccessKeyID string `config:"access_key_id"` - SecretAccessKey string `config:"secret_access_key"` - SessionToken string `config:"session_token"` - ProfileName string `config:"credential_profile_name"` - SharedCredentialFile string `config:"shared_credential_file"` - Endpoint string `config:"endpoint"` - RoleArn string `config:"role_arn"` - AWSPartition string `config:"aws_partition"` // Deprecated. + AccessKeyID string `config:"access_key_id"` + SecretAccessKey string `config:"secret_access_key"` + SessionToken string `config:"session_token"` + ProfileName string `config:"credential_profile_name"` + SharedCredentialFile string `config:"shared_credential_file"` + Endpoint string `config:"endpoint"` + RoleArn string `config:"role_arn"` + AWSPartition string `config:"aws_partition"` // Deprecated. + ProxyUrl *url.URL `config:"proxy_url"` +} + +// InitializeAWSConfig function creates the awssdk.Config object from the provided config +func InitializeAWSConfig(config ConfigAWS) (awssdk.Config, error) { + AWSConfig, _ := GetAWSCredentials(config) + if config.ProxyUrl != nil { + httpClient := &http.Client{ + Transport: &http.Transport{ + Proxy: http.ProxyURL(config.ProxyUrl), + }, + } + AWSConfig.HTTPClient = httpClient + } + return AWSConfig, nil } // GetAWSCredentials function gets aws credentials from the config. diff --git a/x-pack/libbeat/docs/aws-credentials-config.asciidoc b/x-pack/libbeat/docs/aws-credentials-config.asciidoc index c5b94016114e..e56d5b18816d 100644 --- a/x-pack/libbeat/docs/aws-credentials-config.asciidoc +++ b/x-pack/libbeat/docs/aws-credentials-config.asciidoc @@ -17,6 +17,7 @@ Some services, such as IAM, do not support regions. The endpoints for these services do not include a region. In `aws` module, `endpoint` config is to set the `endpoint-code` part, such as `amazonaws.com`, `amazonaws.com.cn`, `c2s.ic.gov`, `sc2s.sgov.gov`. +* *proxy_url*: URL of the proxy to use to connect to AWS web services. The syntax is http(s)://: [float] ==== Supported Formats diff --git a/x-pack/metricbeat/module/aws/aws.go b/x-pack/metricbeat/module/aws/aws.go index f3a7caf6cc8b..f22a1b5ffe1d 100644 --- a/x-pack/metricbeat/module/aws/aws.go +++ b/x-pack/metricbeat/module/aws/aws.go @@ -76,7 +76,7 @@ func NewMetricSet(base mb.BaseMetricSet) (*MetricSet, error) { return nil, err } - awsConfig, err := awscommon.GetAWSCredentials(config.AWSConfig) + awsConfig, err := awscommon.InitializeAWSConfig(config.AWSConfig) if err != nil { return nil, fmt.Errorf("failed to get aws credentials, please check AWS credential in config: %w", err) }