From 21dbabcfffb262dca9624eaa9416836d70b3e55c Mon Sep 17 00:00:00 2001 From: Kyle Pollich Date: Tue, 14 May 2024 15:34:07 -0400 Subject: [PATCH 01/14] Add endpoint var to AWS integration + data streams --- .../_dev/benchmark/rally/billing-benchmark/config.yml | 4 ++-- .../_dev/benchmark/rally/billing-benchmark/fields.yml | 2 +- .../aws/_dev/benchmark/rally/ec2_metrics-benchmark.yml | 2 +- packages/aws/_dev/benchmark/rally/sqs-benchmark.yml | 2 +- packages/aws/changelog.yml | 5 +++++ .../agent/stream/aws-cloudwatch.yml.hbs | 3 +++ .../apigateway_logs/agent/stream/aws-s3.yml.hbs | 5 ++++- .../aws/data_stream/apigateway_logs/fields/agent.yml | 1 + .../apigateway_metrics/agent/stream/stream.yml.hbs | 5 ++++- .../data_stream/billing/agent/stream/stream.yml.hbs | 3 +++ packages/aws/data_stream/billing/manifest.yml | 2 ++ .../cloudfront_logs/agent/stream/aws-s3.yml.hbs | 5 ++++- .../aws/data_stream/cloudfront_logs/fields/agent.yml | 1 + .../cloudtrail/agent/stream/aws-cloudwatch.yml.hbs | 3 +++ .../data_stream/cloudtrail/agent/stream/aws-s3.yml.hbs | 6 ++++-- packages/aws/data_stream/cloudtrail/fields/agent.yml | 1 + .../agent/stream/aws-cloudwatch.yml.hbs | 3 +++ .../cloudwatch_metrics/agent/stream/stream.yml.hbs | 3 +++ .../cloudwatch_metrics/fields/base-fields.yml | 2 +- .../data_stream/dynamodb/agent/stream/stream.yml.hbs | 5 ++++- .../aws/data_stream/ebs/agent/stream/stream.yml.hbs | 5 ++++- .../ec2_logs/agent/stream/aws-cloudwatch.yml.hbs | 3 +++ .../data_stream/ec2_logs/agent/stream/aws-s3.yml.hbs | 5 ++++- packages/aws/data_stream/ec2_logs/fields/agent.yml | 1 + .../ec2_metrics/agent/stream/stream.yml.hbs | 3 +++ .../ecs_metrics/agent/stream/stream.yml.hbs | 6 ++++++ .../elb_logs/agent/stream/aws-cloudwatch.yml.hbs | 6 ++++++ .../data_stream/elb_logs/agent/stream/aws-s3.yml.hbs | 8 +++++++- packages/aws/data_stream/elb_logs/fields/agent.yml | 1 + .../elb_metrics/agent/stream/stream.yml.hbs | 5 ++++- .../emr_logs/agent/stream/aws-cloudwatch.yml.hbs | 3 +++ .../data_stream/emr_logs/agent/stream/aws-s3.yml.hbs | 6 ++++-- packages/aws/data_stream/emr_logs/fields/agent.yml | 1 + .../emr_metrics/agent/stream/stream.yml.hbs | 3 +++ .../firewall_logs/agent/stream/aws-cloudwatch.yml.hbs | 3 +++ .../firewall_logs/agent/stream/aws-s3.yml.hbs | 5 ++++- .../aws/data_stream/firewall_logs/fields/agent.yml | 1 + .../firewall_metrics/agent/stream/stream.yml.hbs | 3 +++ .../kafka_metrics/agent/stream/stream.yml.hbs | 3 +++ .../data_stream/kinesis/agent/stream/stream.yml.hbs | 3 +++ .../aws/data_stream/lambda/agent/stream/stream.yml.hbs | 3 +++ .../data_stream/natgateway/agent/stream/stream.yml.hbs | 3 +++ .../aws/data_stream/rds/agent/stream/stream.yml.hbs | 5 ++++- .../data_stream/redshift/agent/stream/stream.yml.hbs | 6 ++++++ .../agent/stream/aws-cloudwatch.yml.hbs | 3 +++ .../agent/stream/aws-cloudwatch.yml.hbs | 3 +++ .../route53_resolver_logs/agent/stream/aws-s3.yml.hbs | 5 ++++- .../data_stream/route53_resolver_logs/fields/agent.yml | 1 + .../s3_daily_storage/agent/stream/stream.yml.hbs | 3 +++ .../data_stream/s3_request/agent/stream/stream.yml.hbs | 3 +++ .../s3_storage_lens/agent/stream/stream.yml.hbs | 5 ++++- .../data_stream/s3access/agent/stream/aws-s3.yml.hbs | 5 ++++- packages/aws/data_stream/s3access/fields/agent.yml | 1 + .../aws/data_stream/sns/agent/stream/stream.yml.hbs | 3 +++ .../aws/data_stream/sqs/agent/stream/stream.yml.hbs | 3 +++ .../transitgateway/agent/stream/stream.yml.hbs | 3 +++ .../aws/data_stream/usage/agent/stream/stream.yml.hbs | 5 ++++- .../vpcflow/agent/stream/aws-cloudwatch.yml.hbs | 3 +++ .../data_stream/vpcflow/agent/stream/aws-s3.yml.hbs | 5 ++++- packages/aws/data_stream/vpcflow/fields/agent.yml | 1 + .../aws/data_stream/vpn/agent/stream/stream.yml.hbs | 3 +++ .../waf/agent/stream/aws-cloudwatch.yml.hbs | 3 +++ .../aws/data_stream/waf/agent/stream/aws-s3.yml.hbs | 5 ++++- packages/aws/data_stream/waf/fields/agent.yml | 1 + packages/aws/manifest.yml | 10 +++++++++- 65 files changed, 201 insertions(+), 27 deletions(-) diff --git a/packages/aws/_dev/benchmark/rally/billing-benchmark/config.yml b/packages/aws/_dev/benchmark/rally/billing-benchmark/config.yml index d8c08c13dc3..04482d3bfd9 100644 --- a/packages/aws/_dev/benchmark/rally/billing-benchmark/config.yml +++ b/packages/aws/_dev/benchmark/rally/billing-benchmark/config.yml @@ -49,6 +49,6 @@ fields: - name: aws.billing.group_definition.type value: "DIMENSION" - name: aws.billing.group_by.INSTANCE_TYPE - enum: ["NoInstanceType", "a1.large", "c5.2xlarge", "c5.xlarge", "c6i.2xlarge", "db.r6g.2xlarge", "db.t2.micro", "dc2.large", "m5.large", "t1.micro", "t2.medium", "t2.micro", "t2.small", "t2.xlarge", "t3.2xlarge", "t3.medium", "t3.xlarge","t3.xlarge"] + enum: ["NoInstanceType", "a1.large", "c5.2xlarge", "c5.xlarge", "c6i.2xlarge", "db.r6g.2xlarge", "db.t2.micro", "dc2.large", "m5.large", "t1.micro", "t2.medium", "t2.micro", "t2.small", "t2.xlarge", "t3.2xlarge", "t3.medium", "t3.xlarge", "t3.xlarge"] - name: aws.billing.group_by.SERVICE - enum: ["Amazon Simple Storage Service", "Amazon Elastic Compute Cloud - Compute", "EC2 - Other", "Amazon Kinesis", "Amazon Relational Database Service", "Amazon Elastic Load Balancing", "AmazonCloudWatch", "AWS CloudTrail", "AWS Config", "AWS Key Management Service", "AWS Lambda", "AWS Secrets Manager", "AWS Service Catalog", "Amazon API Gateway", "Amazon DynamoDB", "Amazon EC2 Container Registry (ECR)", "Amazon Elastic Container Service for Kubernetes", "Amazon Kinesis Firehose", "Amazon Redshift", "Amazon Simple Notification Service", "Amazon Simple Queue Service", "Amazon Virtual Private Cloud"] \ No newline at end of file + enum: ["Amazon Simple Storage Service", "Amazon Elastic Compute Cloud - Compute", "EC2 - Other", "Amazon Kinesis", "Amazon Relational Database Service", "Amazon Elastic Load Balancing", "AmazonCloudWatch", "AWS CloudTrail", "AWS Config", "AWS Key Management Service", "AWS Lambda", "AWS Secrets Manager", "AWS Service Catalog", "Amazon API Gateway", "Amazon DynamoDB", "Amazon EC2 Container Registry (ECR)", "Amazon Elastic Container Service for Kubernetes", "Amazon Kinesis Firehose", "Amazon Redshift", "Amazon Simple Notification Service", "Amazon Simple Queue Service", "Amazon Virtual Private Cloud"] diff --git a/packages/aws/_dev/benchmark/rally/billing-benchmark/fields.yml b/packages/aws/_dev/benchmark/rally/billing-benchmark/fields.yml index d82177e1a45..c784220ad5a 100644 --- a/packages/aws/_dev/benchmark/rally/billing-benchmark/fields.yml +++ b/packages/aws/_dev/benchmark/rally/billing-benchmark/fields.yml @@ -48,4 +48,4 @@ - name: aws.billing.group_by.INSTANCE_TYPE type: keyword - name: aws.billing.group_by.SERVICE - type: keyword \ No newline at end of file + type: keyword diff --git a/packages/aws/_dev/benchmark/rally/ec2_metrics-benchmark.yml b/packages/aws/_dev/benchmark/rally/ec2_metrics-benchmark.yml index cfe99dfae65..ea0faa02699 100644 --- a/packages/aws/_dev/benchmark/rally/ec2_metrics-benchmark.yml +++ b/packages/aws/_dev/benchmark/rally/ec2_metrics-benchmark.yml @@ -11,4 +11,4 @@ corpora: config: path: ./ec2metrics-benchmark/config.yml fields: - path: ./ec2metrics-benchmark/fields.yml \ No newline at end of file + path: ./ec2metrics-benchmark/fields.yml diff --git a/packages/aws/_dev/benchmark/rally/sqs-benchmark.yml b/packages/aws/_dev/benchmark/rally/sqs-benchmark.yml index bbfd29b6022..d663df3be37 100644 --- a/packages/aws/_dev/benchmark/rally/sqs-benchmark.yml +++ b/packages/aws/_dev/benchmark/rally/sqs-benchmark.yml @@ -11,4 +11,4 @@ corpora: config: path: ./sqs-benchmark/config.yml fields: - path: ./sqs-benchmark/fields.yml \ No newline at end of file + path: ./sqs-benchmark/fields.yml diff --git a/packages/aws/changelog.yml b/packages/aws/changelog.yml index 4bd9010311d..84d910067e2 100644 --- a/packages/aws/changelog.yml +++ b/packages/aws/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "2.15.2-preview01" + changes: + - description: Port endpoint package-level variable from AWS Custom Logs integration to AWS integration + type: enhancement + link: https://github.com/elastic/integrations/pull/9705 - version: "2.15.1" changes: - description: Don't index empty AWS Security Hub responses. diff --git a/packages/aws/data_stream/apigateway_logs/agent/stream/aws-cloudwatch.yml.hbs b/packages/aws/data_stream/apigateway_logs/agent/stream/aws-cloudwatch.yml.hbs index 934f97a44c3..8cc06c88c22 100644 --- a/packages/aws/data_stream/apigateway_logs/agent/stream/aws-cloudwatch.yml.hbs +++ b/packages/aws/data_stream/apigateway_logs/agent/stream/aws-cloudwatch.yml.hbs @@ -66,6 +66,9 @@ shared_credential_file: {{shared_credential_file}} {{#if api_timeout}} api_timeout: {{api_timeout}} {{/if}} +{{#if endpoint}} +endpoint: {{endpoint}} +{{/if}} {{#if default_region}} default_region: {{default_region}} {{/if}} diff --git a/packages/aws/data_stream/apigateway_logs/agent/stream/aws-s3.yml.hbs b/packages/aws/data_stream/apigateway_logs/agent/stream/aws-s3.yml.hbs index dfdf91c9e58..7c66e7a31ee 100644 --- a/packages/aws/data_stream/apigateway_logs/agent/stream/aws-s3.yml.hbs +++ b/packages/aws/data_stream/apigateway_logs/agent/stream/aws-s3.yml.hbs @@ -35,6 +35,9 @@ credential_profile_name: {{credential_profile_name}} {{#if shared_credential_file}} shared_credential_file: {{shared_credential_file}} {{/if}} +{{#if endpoint}} +endpoint: {{endpoint}} +{{/if}} {{#if default_region}} default_region: {{default_region}} {{/if}} @@ -69,4 +72,4 @@ publisher_pipeline.disable_host: true {{#if processors}} processors: {{processors}} -{{/if}} \ No newline at end of file +{{/if}} diff --git a/packages/aws/data_stream/apigateway_logs/fields/agent.yml b/packages/aws/data_stream/apigateway_logs/fields/agent.yml index 196042e493f..b688ceac1b4 100644 --- a/packages/aws/data_stream/apigateway_logs/fields/agent.yml +++ b/packages/aws/data_stream/apigateway_logs/fields/agent.yml @@ -32,6 +32,7 @@ example: "stretch" description: > OS codename, if any. + - name: log.offset type: long description: Log offset diff --git a/packages/aws/data_stream/apigateway_metrics/agent/stream/stream.yml.hbs b/packages/aws/data_stream/apigateway_metrics/agent/stream/stream.yml.hbs index a16d15846a2..321ba873a5a 100644 --- a/packages/aws/data_stream/apigateway_metrics/agent/stream/stream.yml.hbs +++ b/packages/aws/data_stream/apigateway_metrics/agent/stream/stream.yml.hbs @@ -24,6 +24,9 @@ shared_credential_file: {{shared_credential_file}} {{#if role_arn}} role_arn: {{role_arn}} {{/if}} +{{#if endpoint}} +endpoint: {{endpoint}} +{{/if}} {{#if default_region}} default_region: {{default_region}} {{/if}} @@ -63,4 +66,4 @@ metrics: name: - Latency - IntegrationLatency - - DataProcessed \ No newline at end of file + - DataProcessed diff --git a/packages/aws/data_stream/billing/agent/stream/stream.yml.hbs b/packages/aws/data_stream/billing/agent/stream/stream.yml.hbs index fbbfd6733a7..8e0cce0ca62 100644 --- a/packages/aws/data_stream/billing/agent/stream/stream.yml.hbs +++ b/packages/aws/data_stream/billing/agent/stream/stream.yml.hbs @@ -24,6 +24,9 @@ shared_credential_file: {{shared_credential_file}} {{#if role_arn}} role_arn: {{role_arn}} {{/if}} +{{#if endpoint}} +endpoint: {{endpoint}} +{{/if}} {{#if default_region}} default_region: {{default_region}} {{/if}} diff --git a/packages/aws/data_stream/billing/manifest.yml b/packages/aws/data_stream/billing/manifest.yml index 3fda2894036..97d2c4a444a 100644 --- a/packages/aws/data_stream/billing/manifest.yml +++ b/packages/aws/data_stream/billing/manifest.yml @@ -58,6 +58,7 @@ streams: default: false description: > Enable leaderelection between a set of Elastic Agents running on Kubernetes. + - name: processors type: yaml title: Processors @@ -66,5 +67,6 @@ streams: show_user: false description: > Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the events are shipped. See [Processors](https://www.elastic.co/guide/en/fleet/current/elastic-agent-processor-configuration.html) for details. + title: AWS Billing Metrics description: Collect billing metrics from Amazon Web Services with Elastic Agent. diff --git a/packages/aws/data_stream/cloudfront_logs/agent/stream/aws-s3.yml.hbs b/packages/aws/data_stream/cloudfront_logs/agent/stream/aws-s3.yml.hbs index 1d0bbc280fc..8b130a4c568 100644 --- a/packages/aws/data_stream/cloudfront_logs/agent/stream/aws-s3.yml.hbs +++ b/packages/aws/data_stream/cloudfront_logs/agent/stream/aws-s3.yml.hbs @@ -14,6 +14,9 @@ api_timeout: {{api_timeout}} {{#if max_number_of_messages}} max_number_of_messages: {{max_number_of_messages}} {{/if}} +{{#if endpoint}} +endpoint: {{endpoint}} +{{/if}} {{#if default_region}} default_region: {{default_region}} {{/if}} @@ -52,4 +55,4 @@ processors: message: "^#.*" {{#if processors}} {{processors}} -{{/if}} \ No newline at end of file +{{/if}} diff --git a/packages/aws/data_stream/cloudfront_logs/fields/agent.yml b/packages/aws/data_stream/cloudfront_logs/fields/agent.yml index 1e09834aad6..2ddef01a27b 100644 --- a/packages/aws/data_stream/cloudfront_logs/fields/agent.yml +++ b/packages/aws/data_stream/cloudfront_logs/fields/agent.yml @@ -32,6 +32,7 @@ example: "stretch" description: > OS codename, if any. + - name: log.offset type: long description: Log offset diff --git a/packages/aws/data_stream/cloudtrail/agent/stream/aws-cloudwatch.yml.hbs b/packages/aws/data_stream/cloudtrail/agent/stream/aws-cloudwatch.yml.hbs index 934f97a44c3..8cc06c88c22 100644 --- a/packages/aws/data_stream/cloudtrail/agent/stream/aws-cloudwatch.yml.hbs +++ b/packages/aws/data_stream/cloudtrail/agent/stream/aws-cloudwatch.yml.hbs @@ -66,6 +66,9 @@ shared_credential_file: {{shared_credential_file}} {{#if api_timeout}} api_timeout: {{api_timeout}} {{/if}} +{{#if endpoint}} +endpoint: {{endpoint}} +{{/if}} {{#if default_region}} default_region: {{default_region}} {{/if}} diff --git a/packages/aws/data_stream/cloudtrail/agent/stream/aws-s3.yml.hbs b/packages/aws/data_stream/cloudtrail/agent/stream/aws-s3.yml.hbs index b54c36bb530..038d3b5f5e8 100644 --- a/packages/aws/data_stream/cloudtrail/agent/stream/aws-s3.yml.hbs +++ b/packages/aws/data_stream/cloudtrail/agent/stream/aws-s3.yml.hbs @@ -50,7 +50,9 @@ credential_profile_name: {{credential_profile_name}} shared_credential_file: {{shared_credential_file}} {{/if}} - +{{#if endpoint}} +endpoint: {{endpoint}} +{{/if}} {{#if default_region}} default_region: {{default_region}} {{/if}} @@ -86,4 +88,4 @@ publisher_pipeline.disable_host: true {{#if processors}} processors: {{processors}} -{{/if}} \ No newline at end of file +{{/if}} diff --git a/packages/aws/data_stream/cloudtrail/fields/agent.yml b/packages/aws/data_stream/cloudtrail/fields/agent.yml index 196042e493f..b688ceac1b4 100644 --- a/packages/aws/data_stream/cloudtrail/fields/agent.yml +++ b/packages/aws/data_stream/cloudtrail/fields/agent.yml @@ -32,6 +32,7 @@ example: "stretch" description: > OS codename, if any. + - name: log.offset type: long description: Log offset diff --git a/packages/aws/data_stream/cloudwatch_logs/agent/stream/aws-cloudwatch.yml.hbs b/packages/aws/data_stream/cloudwatch_logs/agent/stream/aws-cloudwatch.yml.hbs index 9588557d158..0f789a733b2 100644 --- a/packages/aws/data_stream/cloudwatch_logs/agent/stream/aws-cloudwatch.yml.hbs +++ b/packages/aws/data_stream/cloudwatch_logs/agent/stream/aws-cloudwatch.yml.hbs @@ -69,6 +69,9 @@ shared_credential_file: {{shared_credential_file}} {{#if api_timeout}} api_timeout: {{api_timeout}} {{/if}} +{{#if endpoint}} +endpoint: {{endpoint}} +{{/if}} {{#if default_region}} default_region: {{default_region}} {{/if}} diff --git a/packages/aws/data_stream/cloudwatch_metrics/agent/stream/stream.yml.hbs b/packages/aws/data_stream/cloudwatch_metrics/agent/stream/stream.yml.hbs index fbf98ab3a0a..9bccc83fb15 100644 --- a/packages/aws/data_stream/cloudwatch_metrics/agent/stream/stream.yml.hbs +++ b/packages/aws/data_stream/cloudwatch_metrics/agent/stream/stream.yml.hbs @@ -24,6 +24,9 @@ shared_credential_file: {{shared_credential_file}} {{#if role_arn}} role_arn: {{role_arn}} {{/if}} +{{#if endpoint}} +endpoint: {{endpoint}} +{{/if}} {{#if default_region}} default_region: {{default_region}} {{/if}} diff --git a/packages/aws/data_stream/cloudwatch_metrics/fields/base-fields.yml b/packages/aws/data_stream/cloudwatch_metrics/fields/base-fields.yml index e13227ae1d1..2b34ac77b17 100644 --- a/packages/aws/data_stream/cloudwatch_metrics/fields/base-fields.yml +++ b/packages/aws/data_stream/cloudwatch_metrics/fields/base-fields.yml @@ -13,4 +13,4 @@ - name: event.module type: constant_keyword description: Event module - value: aws \ No newline at end of file + value: aws diff --git a/packages/aws/data_stream/dynamodb/agent/stream/stream.yml.hbs b/packages/aws/data_stream/dynamodb/agent/stream/stream.yml.hbs index 74b1ae36bfc..8121c7b3555 100644 --- a/packages/aws/data_stream/dynamodb/agent/stream/stream.yml.hbs +++ b/packages/aws/data_stream/dynamodb/agent/stream/stream.yml.hbs @@ -24,6 +24,9 @@ shared_credential_file: {{shared_credential_file}} {{#if role_arn}} role_arn: {{role_arn}} {{/if}} +{{#if endpoint}} +endpoint: {{endpoint}} +{{/if}} {{#if default_region}} default_region: {{default_region}} {{/if}} @@ -81,4 +84,4 @@ metrics: - AccountMaxTableLevelWrites - AccountMaxWrites - MaxProvisionedTableReadCapacityUtilization - - MaxProvisionedTableWriteCapacityUtilization \ No newline at end of file + - MaxProvisionedTableWriteCapacityUtilization diff --git a/packages/aws/data_stream/ebs/agent/stream/stream.yml.hbs b/packages/aws/data_stream/ebs/agent/stream/stream.yml.hbs index ab300cc6746..6c8340e738a 100644 --- a/packages/aws/data_stream/ebs/agent/stream/stream.yml.hbs +++ b/packages/aws/data_stream/ebs/agent/stream/stream.yml.hbs @@ -24,6 +24,9 @@ shared_credential_file: {{shared_credential_file}} {{#if role_arn}} role_arn: {{role_arn}} {{/if}} +{{#if endpoint}} +endpoint: {{endpoint}} +{{/if}} {{#if default_region}} default_region: {{default_region}} {{/if}} @@ -50,4 +53,4 @@ metrics: - namespace: AWS/EBS statistic: ["Sum"] name: ["VolumeTotalReadTime", "VolumeTotalWriteTime", "VolumeIdleTime"] - resource_type: ec2 \ No newline at end of file + resource_type: ec2 diff --git a/packages/aws/data_stream/ec2_logs/agent/stream/aws-cloudwatch.yml.hbs b/packages/aws/data_stream/ec2_logs/agent/stream/aws-cloudwatch.yml.hbs index 934f97a44c3..8cc06c88c22 100644 --- a/packages/aws/data_stream/ec2_logs/agent/stream/aws-cloudwatch.yml.hbs +++ b/packages/aws/data_stream/ec2_logs/agent/stream/aws-cloudwatch.yml.hbs @@ -66,6 +66,9 @@ shared_credential_file: {{shared_credential_file}} {{#if api_timeout}} api_timeout: {{api_timeout}} {{/if}} +{{#if endpoint}} +endpoint: {{endpoint}} +{{/if}} {{#if default_region}} default_region: {{default_region}} {{/if}} diff --git a/packages/aws/data_stream/ec2_logs/agent/stream/aws-s3.yml.hbs b/packages/aws/data_stream/ec2_logs/agent/stream/aws-s3.yml.hbs index 77fae5d0dc8..dfdeeb91a28 100644 --- a/packages/aws/data_stream/ec2_logs/agent/stream/aws-s3.yml.hbs +++ b/packages/aws/data_stream/ec2_logs/agent/stream/aws-s3.yml.hbs @@ -14,6 +14,9 @@ api_timeout: {{api_timeout}} {{#if max_number_of_messages}} max_number_of_messages: {{max_number_of_messages}} {{/if}} +{{#if endpoint}} +endpoint: {{endpoint}} +{{/if}} {{#if default_region}} default_region: {{default_region}} {{/if}} @@ -48,4 +51,4 @@ publisher_pipeline.disable_host: true {{#if processors}} processors: {{processors}} -{{/if}} \ No newline at end of file +{{/if}} diff --git a/packages/aws/data_stream/ec2_logs/fields/agent.yml b/packages/aws/data_stream/ec2_logs/fields/agent.yml index 1e09834aad6..2ddef01a27b 100644 --- a/packages/aws/data_stream/ec2_logs/fields/agent.yml +++ b/packages/aws/data_stream/ec2_logs/fields/agent.yml @@ -32,6 +32,7 @@ example: "stretch" description: > OS codename, if any. + - name: log.offset type: long description: Log offset diff --git a/packages/aws/data_stream/ec2_metrics/agent/stream/stream.yml.hbs b/packages/aws/data_stream/ec2_metrics/agent/stream/stream.yml.hbs index 12d3f6848f3..02dbe07a92a 100644 --- a/packages/aws/data_stream/ec2_metrics/agent/stream/stream.yml.hbs +++ b/packages/aws/data_stream/ec2_metrics/agent/stream/stream.yml.hbs @@ -24,6 +24,9 @@ shared_credential_file: {{shared_credential_file}} {{#if role_arn}} role_arn: {{role_arn}} {{/if}} +{{#if endpoint}} +endpoint: {{endpoint}} +{{/if}} {{#if default_region}} default_region: {{default_region}} {{/if}} diff --git a/packages/aws/data_stream/ecs_metrics/agent/stream/stream.yml.hbs b/packages/aws/data_stream/ecs_metrics/agent/stream/stream.yml.hbs index d8b2cf6475f..67788f32081 100644 --- a/packages/aws/data_stream/ecs_metrics/agent/stream/stream.yml.hbs +++ b/packages/aws/data_stream/ecs_metrics/agent/stream/stream.yml.hbs @@ -24,6 +24,12 @@ shared_credential_file: {{shared_credential_file}} {{#if role_arn}} role_arn: {{role_arn}} {{/if}} +{{#if endpoint}} +endpoint: {{endpoint}} +{{/if}} +{{#if endpoint}} +endpoint: {{endpoint}} +{{/if}} {{#if default_region}} default_region: {{default_region}} {{/if}} diff --git a/packages/aws/data_stream/elb_logs/agent/stream/aws-cloudwatch.yml.hbs b/packages/aws/data_stream/elb_logs/agent/stream/aws-cloudwatch.yml.hbs index 934f97a44c3..59974851e5f 100644 --- a/packages/aws/data_stream/elb_logs/agent/stream/aws-cloudwatch.yml.hbs +++ b/packages/aws/data_stream/elb_logs/agent/stream/aws-cloudwatch.yml.hbs @@ -66,6 +66,12 @@ shared_credential_file: {{shared_credential_file}} {{#if api_timeout}} api_timeout: {{api_timeout}} {{/if}} +{{#if endpoint}} +endpoint: {{endpoint}} +{{/if}} +{{#if endpoint}} +endpoint: {{endpoint}} +{{/if}} {{#if default_region}} default_region: {{default_region}} {{/if}} diff --git a/packages/aws/data_stream/elb_logs/agent/stream/aws-s3.yml.hbs b/packages/aws/data_stream/elb_logs/agent/stream/aws-s3.yml.hbs index 77fae5d0dc8..d4fdba80f0e 100644 --- a/packages/aws/data_stream/elb_logs/agent/stream/aws-s3.yml.hbs +++ b/packages/aws/data_stream/elb_logs/agent/stream/aws-s3.yml.hbs @@ -14,6 +14,12 @@ api_timeout: {{api_timeout}} {{#if max_number_of_messages}} max_number_of_messages: {{max_number_of_messages}} {{/if}} +{{#if endpoint}} +endpoint: {{endpoint}} +{{/if}} +{{#if endpoint}} +endpoint: {{endpoint}} +{{/if}} {{#if default_region}} default_region: {{default_region}} {{/if}} @@ -48,4 +54,4 @@ publisher_pipeline.disable_host: true {{#if processors}} processors: {{processors}} -{{/if}} \ No newline at end of file +{{/if}} diff --git a/packages/aws/data_stream/elb_logs/fields/agent.yml b/packages/aws/data_stream/elb_logs/fields/agent.yml index 1e09834aad6..2ddef01a27b 100644 --- a/packages/aws/data_stream/elb_logs/fields/agent.yml +++ b/packages/aws/data_stream/elb_logs/fields/agent.yml @@ -32,6 +32,7 @@ example: "stretch" description: > OS codename, if any. + - name: log.offset type: long description: Log offset diff --git a/packages/aws/data_stream/elb_metrics/agent/stream/stream.yml.hbs b/packages/aws/data_stream/elb_metrics/agent/stream/stream.yml.hbs index 52f762fcc89..cf3d2ff7917 100644 --- a/packages/aws/data_stream/elb_metrics/agent/stream/stream.yml.hbs +++ b/packages/aws/data_stream/elb_metrics/agent/stream/stream.yml.hbs @@ -24,6 +24,9 @@ shared_credential_file: {{shared_credential_file}} {{#if role_arn}} role_arn: {{role_arn}} {{/if}} +{{#if endpoint}} +endpoint: {{endpoint}} +{{/if}} {{#if default_region}} default_region: {{default_region}} {{/if}} @@ -140,4 +143,4 @@ metrics: statistic: ["Maximum"] name: - HealthyHostCount - - UnHealthyHostCount \ No newline at end of file + - UnHealthyHostCount diff --git a/packages/aws/data_stream/emr_logs/agent/stream/aws-cloudwatch.yml.hbs b/packages/aws/data_stream/emr_logs/agent/stream/aws-cloudwatch.yml.hbs index 934f97a44c3..8cc06c88c22 100644 --- a/packages/aws/data_stream/emr_logs/agent/stream/aws-cloudwatch.yml.hbs +++ b/packages/aws/data_stream/emr_logs/agent/stream/aws-cloudwatch.yml.hbs @@ -66,6 +66,9 @@ shared_credential_file: {{shared_credential_file}} {{#if api_timeout}} api_timeout: {{api_timeout}} {{/if}} +{{#if endpoint}} +endpoint: {{endpoint}} +{{/if}} {{#if default_region}} default_region: {{default_region}} {{/if}} diff --git a/packages/aws/data_stream/emr_logs/agent/stream/aws-s3.yml.hbs b/packages/aws/data_stream/emr_logs/agent/stream/aws-s3.yml.hbs index c479d4b6dd0..9e59987afdb 100644 --- a/packages/aws/data_stream/emr_logs/agent/stream/aws-s3.yml.hbs +++ b/packages/aws/data_stream/emr_logs/agent/stream/aws-s3.yml.hbs @@ -34,7 +34,9 @@ file_selectors: {{/if}} - +{{#if endpoint}} +endpoint: {{endpoint}} +{{/if}} {{#if default_region}} default_region: {{default_region}} {{/if}} @@ -75,4 +77,4 @@ publisher_pipeline.disable_host: true {{#if processors}} processors: {{processors}} -{{/if}} \ No newline at end of file +{{/if}} diff --git a/packages/aws/data_stream/emr_logs/fields/agent.yml b/packages/aws/data_stream/emr_logs/fields/agent.yml index 196042e493f..b688ceac1b4 100644 --- a/packages/aws/data_stream/emr_logs/fields/agent.yml +++ b/packages/aws/data_stream/emr_logs/fields/agent.yml @@ -32,6 +32,7 @@ example: "stretch" description: > OS codename, if any. + - name: log.offset type: long description: Log offset diff --git a/packages/aws/data_stream/emr_metrics/agent/stream/stream.yml.hbs b/packages/aws/data_stream/emr_metrics/agent/stream/stream.yml.hbs index 00fcc848016..67fd238e8c9 100644 --- a/packages/aws/data_stream/emr_metrics/agent/stream/stream.yml.hbs +++ b/packages/aws/data_stream/emr_metrics/agent/stream/stream.yml.hbs @@ -24,6 +24,9 @@ shared_credential_file: {{shared_credential_file}} {{#if role_arn}} role_arn: {{role_arn}} {{/if}} +{{#if endpoint}} +endpoint: {{endpoint}} +{{/if}} {{#if default_region}} default_region: {{default_region}} {{/if}} diff --git a/packages/aws/data_stream/firewall_logs/agent/stream/aws-cloudwatch.yml.hbs b/packages/aws/data_stream/firewall_logs/agent/stream/aws-cloudwatch.yml.hbs index 934f97a44c3..8cc06c88c22 100644 --- a/packages/aws/data_stream/firewall_logs/agent/stream/aws-cloudwatch.yml.hbs +++ b/packages/aws/data_stream/firewall_logs/agent/stream/aws-cloudwatch.yml.hbs @@ -66,6 +66,9 @@ shared_credential_file: {{shared_credential_file}} {{#if api_timeout}} api_timeout: {{api_timeout}} {{/if}} +{{#if endpoint}} +endpoint: {{endpoint}} +{{/if}} {{#if default_region}} default_region: {{default_region}} {{/if}} diff --git a/packages/aws/data_stream/firewall_logs/agent/stream/aws-s3.yml.hbs b/packages/aws/data_stream/firewall_logs/agent/stream/aws-s3.yml.hbs index 6edb2d8079b..196dda8a63e 100644 --- a/packages/aws/data_stream/firewall_logs/agent/stream/aws-s3.yml.hbs +++ b/packages/aws/data_stream/firewall_logs/agent/stream/aws-s3.yml.hbs @@ -14,6 +14,9 @@ api_timeout: {{api_timeout}} {{#if max_number_of_messages}} max_number_of_messages: {{max_number_of_messages}} {{/if}} +{{#if endpoint}} +endpoint: {{endpoint}} +{{/if}} {{#if default_region}} default_region: {{default_region}} {{/if}} @@ -49,4 +52,4 @@ publisher_pipeline.disable_host: true {{#if processors}} processors: {{processors}} -{{/if}} \ No newline at end of file +{{/if}} diff --git a/packages/aws/data_stream/firewall_logs/fields/agent.yml b/packages/aws/data_stream/firewall_logs/fields/agent.yml index 1e09834aad6..2ddef01a27b 100644 --- a/packages/aws/data_stream/firewall_logs/fields/agent.yml +++ b/packages/aws/data_stream/firewall_logs/fields/agent.yml @@ -32,6 +32,7 @@ example: "stretch" description: > OS codename, if any. + - name: log.offset type: long description: Log offset diff --git a/packages/aws/data_stream/firewall_metrics/agent/stream/stream.yml.hbs b/packages/aws/data_stream/firewall_metrics/agent/stream/stream.yml.hbs index fbf98ab3a0a..9bccc83fb15 100644 --- a/packages/aws/data_stream/firewall_metrics/agent/stream/stream.yml.hbs +++ b/packages/aws/data_stream/firewall_metrics/agent/stream/stream.yml.hbs @@ -24,6 +24,9 @@ shared_credential_file: {{shared_credential_file}} {{#if role_arn}} role_arn: {{role_arn}} {{/if}} +{{#if endpoint}} +endpoint: {{endpoint}} +{{/if}} {{#if default_region}} default_region: {{default_region}} {{/if}} diff --git a/packages/aws/data_stream/kafka_metrics/agent/stream/stream.yml.hbs b/packages/aws/data_stream/kafka_metrics/agent/stream/stream.yml.hbs index 395ce09518d..b3194ecf505 100644 --- a/packages/aws/data_stream/kafka_metrics/agent/stream/stream.yml.hbs +++ b/packages/aws/data_stream/kafka_metrics/agent/stream/stream.yml.hbs @@ -24,6 +24,9 @@ shared_credential_file: {{shared_credential_file}} {{#if role_arn}} role_arn: {{role_arn}} {{/if}} +{{#if endpoint}} +endpoint: {{endpoint}} +{{/if}} {{#if default_region}} default_region: {{default_region}} {{/if}} diff --git a/packages/aws/data_stream/kinesis/agent/stream/stream.yml.hbs b/packages/aws/data_stream/kinesis/agent/stream/stream.yml.hbs index e0ec3daefc3..0d75c9dcc5d 100644 --- a/packages/aws/data_stream/kinesis/agent/stream/stream.yml.hbs +++ b/packages/aws/data_stream/kinesis/agent/stream/stream.yml.hbs @@ -24,6 +24,9 @@ shared_credential_file: {{shared_credential_file}} {{#if role_arn}} role_arn: {{role_arn}} {{/if}} +{{#if endpoint}} +endpoint: {{endpoint}} +{{/if}} {{#if default_region}} default_region: {{default_region}} {{/if}} diff --git a/packages/aws/data_stream/lambda/agent/stream/stream.yml.hbs b/packages/aws/data_stream/lambda/agent/stream/stream.yml.hbs index 633fc3f698f..7a43c14e13f 100644 --- a/packages/aws/data_stream/lambda/agent/stream/stream.yml.hbs +++ b/packages/aws/data_stream/lambda/agent/stream/stream.yml.hbs @@ -24,6 +24,9 @@ shared_credential_file: {{shared_credential_file}} {{#if role_arn}} role_arn: {{role_arn}} {{/if}} +{{#if endpoint}} +endpoint: {{endpoint}} +{{/if}} {{#if default_region}} default_region: {{default_region}} {{/if}} diff --git a/packages/aws/data_stream/natgateway/agent/stream/stream.yml.hbs b/packages/aws/data_stream/natgateway/agent/stream/stream.yml.hbs index c260b3d0aa7..268edb1bdf0 100644 --- a/packages/aws/data_stream/natgateway/agent/stream/stream.yml.hbs +++ b/packages/aws/data_stream/natgateway/agent/stream/stream.yml.hbs @@ -24,6 +24,9 @@ shared_credential_file: {{shared_credential_file}} {{#if role_arn}} role_arn: {{role_arn}} {{/if}} +{{#if endpoint}} +endpoint: {{endpoint}} +{{/if}} {{#if default_region}} default_region: {{default_region}} {{/if}} diff --git a/packages/aws/data_stream/rds/agent/stream/stream.yml.hbs b/packages/aws/data_stream/rds/agent/stream/stream.yml.hbs index 4c5b7abca05..fc4c0739092 100644 --- a/packages/aws/data_stream/rds/agent/stream/stream.yml.hbs +++ b/packages/aws/data_stream/rds/agent/stream/stream.yml.hbs @@ -24,6 +24,9 @@ shared_credential_file: {{shared_credential_file}} {{#if role_arn}} role_arn: {{role_arn}} {{/if}} +{{#if endpoint}} +endpoint: {{endpoint}} +{{/if}} {{#if default_region}} default_region: {{default_region}} {{/if}} @@ -45,4 +48,4 @@ proxy_url: {{proxy_url}} metrics: - namespace: AWS/RDS resource_type: rds - statistic: ["Average"] \ No newline at end of file + statistic: ["Average"] diff --git a/packages/aws/data_stream/redshift/agent/stream/stream.yml.hbs b/packages/aws/data_stream/redshift/agent/stream/stream.yml.hbs index d3d0eff8669..5e430cedcfa 100644 --- a/packages/aws/data_stream/redshift/agent/stream/stream.yml.hbs +++ b/packages/aws/data_stream/redshift/agent/stream/stream.yml.hbs @@ -24,6 +24,12 @@ shared_credential_file: {{shared_credential_file}} {{#if role_arn}} role_arn: {{role_arn}} {{/if}} +{{#if endpoint}} +endpoint: {{endpoint}} +{{/if}} +{{#if endpoint}} +endpoint: {{endpoint}} +{{/if}} {{#if default_region}} default_region: {{default_region}} {{/if}} diff --git a/packages/aws/data_stream/route53_public_logs/agent/stream/aws-cloudwatch.yml.hbs b/packages/aws/data_stream/route53_public_logs/agent/stream/aws-cloudwatch.yml.hbs index 934f97a44c3..8cc06c88c22 100644 --- a/packages/aws/data_stream/route53_public_logs/agent/stream/aws-cloudwatch.yml.hbs +++ b/packages/aws/data_stream/route53_public_logs/agent/stream/aws-cloudwatch.yml.hbs @@ -66,6 +66,9 @@ shared_credential_file: {{shared_credential_file}} {{#if api_timeout}} api_timeout: {{api_timeout}} {{/if}} +{{#if endpoint}} +endpoint: {{endpoint}} +{{/if}} {{#if default_region}} default_region: {{default_region}} {{/if}} diff --git a/packages/aws/data_stream/route53_resolver_logs/agent/stream/aws-cloudwatch.yml.hbs b/packages/aws/data_stream/route53_resolver_logs/agent/stream/aws-cloudwatch.yml.hbs index 934f97a44c3..8cc06c88c22 100644 --- a/packages/aws/data_stream/route53_resolver_logs/agent/stream/aws-cloudwatch.yml.hbs +++ b/packages/aws/data_stream/route53_resolver_logs/agent/stream/aws-cloudwatch.yml.hbs @@ -66,6 +66,9 @@ shared_credential_file: {{shared_credential_file}} {{#if api_timeout}} api_timeout: {{api_timeout}} {{/if}} +{{#if endpoint}} +endpoint: {{endpoint}} +{{/if}} {{#if default_region}} default_region: {{default_region}} {{/if}} diff --git a/packages/aws/data_stream/route53_resolver_logs/agent/stream/aws-s3.yml.hbs b/packages/aws/data_stream/route53_resolver_logs/agent/stream/aws-s3.yml.hbs index 6edb2d8079b..196dda8a63e 100644 --- a/packages/aws/data_stream/route53_resolver_logs/agent/stream/aws-s3.yml.hbs +++ b/packages/aws/data_stream/route53_resolver_logs/agent/stream/aws-s3.yml.hbs @@ -14,6 +14,9 @@ api_timeout: {{api_timeout}} {{#if max_number_of_messages}} max_number_of_messages: {{max_number_of_messages}} {{/if}} +{{#if endpoint}} +endpoint: {{endpoint}} +{{/if}} {{#if default_region}} default_region: {{default_region}} {{/if}} @@ -49,4 +52,4 @@ publisher_pipeline.disable_host: true {{#if processors}} processors: {{processors}} -{{/if}} \ No newline at end of file +{{/if}} diff --git a/packages/aws/data_stream/route53_resolver_logs/fields/agent.yml b/packages/aws/data_stream/route53_resolver_logs/fields/agent.yml index 1e09834aad6..2ddef01a27b 100644 --- a/packages/aws/data_stream/route53_resolver_logs/fields/agent.yml +++ b/packages/aws/data_stream/route53_resolver_logs/fields/agent.yml @@ -32,6 +32,7 @@ example: "stretch" description: > OS codename, if any. + - name: log.offset type: long description: Log offset diff --git a/packages/aws/data_stream/s3_daily_storage/agent/stream/stream.yml.hbs b/packages/aws/data_stream/s3_daily_storage/agent/stream/stream.yml.hbs index b072f249870..531cb9d47e2 100644 --- a/packages/aws/data_stream/s3_daily_storage/agent/stream/stream.yml.hbs +++ b/packages/aws/data_stream/s3_daily_storage/agent/stream/stream.yml.hbs @@ -24,6 +24,9 @@ shared_credential_file: {{shared_credential_file}} {{#if role_arn}} role_arn: {{role_arn}} {{/if}} +{{#if endpoint}} +endpoint: {{endpoint}} +{{/if}} {{#if default_region}} default_region: {{default_region}} {{/if}} diff --git a/packages/aws/data_stream/s3_request/agent/stream/stream.yml.hbs b/packages/aws/data_stream/s3_request/agent/stream/stream.yml.hbs index 3cf5faaac4d..9e83fe9ea92 100644 --- a/packages/aws/data_stream/s3_request/agent/stream/stream.yml.hbs +++ b/packages/aws/data_stream/s3_request/agent/stream/stream.yml.hbs @@ -24,6 +24,9 @@ shared_credential_file: {{shared_credential_file}} {{#if role_arn}} role_arn: {{role_arn}} {{/if}} +{{#if endpoint}} +endpoint: {{endpoint}} +{{/if}} {{#if default_region}} default_region: {{default_region}} {{/if}} diff --git a/packages/aws/data_stream/s3_storage_lens/agent/stream/stream.yml.hbs b/packages/aws/data_stream/s3_storage_lens/agent/stream/stream.yml.hbs index 08fabbf30b3..ca6eb412848 100644 --- a/packages/aws/data_stream/s3_storage_lens/agent/stream/stream.yml.hbs +++ b/packages/aws/data_stream/s3_storage_lens/agent/stream/stream.yml.hbs @@ -24,6 +24,9 @@ shared_credential_file: {{shared_credential_file}} {{#if role_arn}} role_arn: {{role_arn}} {{/if}} +{{#if endpoint}} +endpoint: {{endpoint}} +{{/if}} {{#if default_region}} default_region: {{default_region}} {{/if}} @@ -107,4 +110,4 @@ processors: - drop_fields: ignore_missing: true fields: - - "aws.storage-lens" \ No newline at end of file + - "aws.storage-lens" diff --git a/packages/aws/data_stream/s3access/agent/stream/aws-s3.yml.hbs b/packages/aws/data_stream/s3access/agent/stream/aws-s3.yml.hbs index 77fae5d0dc8..dfdeeb91a28 100644 --- a/packages/aws/data_stream/s3access/agent/stream/aws-s3.yml.hbs +++ b/packages/aws/data_stream/s3access/agent/stream/aws-s3.yml.hbs @@ -14,6 +14,9 @@ api_timeout: {{api_timeout}} {{#if max_number_of_messages}} max_number_of_messages: {{max_number_of_messages}} {{/if}} +{{#if endpoint}} +endpoint: {{endpoint}} +{{/if}} {{#if default_region}} default_region: {{default_region}} {{/if}} @@ -48,4 +51,4 @@ publisher_pipeline.disable_host: true {{#if processors}} processors: {{processors}} -{{/if}} \ No newline at end of file +{{/if}} diff --git a/packages/aws/data_stream/s3access/fields/agent.yml b/packages/aws/data_stream/s3access/fields/agent.yml index 1e09834aad6..2ddef01a27b 100644 --- a/packages/aws/data_stream/s3access/fields/agent.yml +++ b/packages/aws/data_stream/s3access/fields/agent.yml @@ -32,6 +32,7 @@ example: "stretch" description: > OS codename, if any. + - name: log.offset type: long description: Log offset diff --git a/packages/aws/data_stream/sns/agent/stream/stream.yml.hbs b/packages/aws/data_stream/sns/agent/stream/stream.yml.hbs index 82a4266c70f..5dac007eaab 100644 --- a/packages/aws/data_stream/sns/agent/stream/stream.yml.hbs +++ b/packages/aws/data_stream/sns/agent/stream/stream.yml.hbs @@ -24,6 +24,9 @@ shared_credential_file: {{shared_credential_file}} {{#if role_arn}} role_arn: {{role_arn}} {{/if}} +{{#if endpoint}} +endpoint: {{endpoint}} +{{/if}} {{#if default_region}} default_region: {{default_region}} {{/if}} diff --git a/packages/aws/data_stream/sqs/agent/stream/stream.yml.hbs b/packages/aws/data_stream/sqs/agent/stream/stream.yml.hbs index 062e6f846aa..6caeea86f8d 100644 --- a/packages/aws/data_stream/sqs/agent/stream/stream.yml.hbs +++ b/packages/aws/data_stream/sqs/agent/stream/stream.yml.hbs @@ -24,6 +24,9 @@ shared_credential_file: {{shared_credential_file}} {{#if role_arn}} role_arn: {{role_arn}} {{/if}} +{{#if endpoint}} +endpoint: {{endpoint}} +{{/if}} {{#if default_region}} default_region: {{default_region}} {{/if}} diff --git a/packages/aws/data_stream/transitgateway/agent/stream/stream.yml.hbs b/packages/aws/data_stream/transitgateway/agent/stream/stream.yml.hbs index 92be4c7bdf8..bfa2ac9496a 100644 --- a/packages/aws/data_stream/transitgateway/agent/stream/stream.yml.hbs +++ b/packages/aws/data_stream/transitgateway/agent/stream/stream.yml.hbs @@ -24,6 +24,9 @@ shared_credential_file: {{shared_credential_file}} {{#if role_arn}} role_arn: {{role_arn}} {{/if}} +{{#if endpoint}} +endpoint: {{endpoint}} +{{/if}} {{#if default_region}} default_region: {{default_region}} {{/if}} diff --git a/packages/aws/data_stream/usage/agent/stream/stream.yml.hbs b/packages/aws/data_stream/usage/agent/stream/stream.yml.hbs index 9602b37d578..303a58177ab 100644 --- a/packages/aws/data_stream/usage/agent/stream/stream.yml.hbs +++ b/packages/aws/data_stream/usage/agent/stream/stream.yml.hbs @@ -24,6 +24,9 @@ shared_credential_file: {{shared_credential_file}} {{#if role_arn}} role_arn: {{role_arn}} {{/if}} +{{#if endpoint}} +endpoint: {{endpoint}} +{{/if}} {{#if default_region}} default_region: {{default_region}} {{/if}} @@ -44,4 +47,4 @@ proxy_url: {{proxy_url}} {{/if}} metrics: - namespace: AWS/Usage - statistic: ["Sum"] \ No newline at end of file + statistic: ["Sum"] diff --git a/packages/aws/data_stream/vpcflow/agent/stream/aws-cloudwatch.yml.hbs b/packages/aws/data_stream/vpcflow/agent/stream/aws-cloudwatch.yml.hbs index 934f97a44c3..8cc06c88c22 100644 --- a/packages/aws/data_stream/vpcflow/agent/stream/aws-cloudwatch.yml.hbs +++ b/packages/aws/data_stream/vpcflow/agent/stream/aws-cloudwatch.yml.hbs @@ -66,6 +66,9 @@ shared_credential_file: {{shared_credential_file}} {{#if api_timeout}} api_timeout: {{api_timeout}} {{/if}} +{{#if endpoint}} +endpoint: {{endpoint}} +{{/if}} {{#if default_region}} default_region: {{default_region}} {{/if}} diff --git a/packages/aws/data_stream/vpcflow/agent/stream/aws-s3.yml.hbs b/packages/aws/data_stream/vpcflow/agent/stream/aws-s3.yml.hbs index 64796e6215f..76c57cd135d 100644 --- a/packages/aws/data_stream/vpcflow/agent/stream/aws-s3.yml.hbs +++ b/packages/aws/data_stream/vpcflow/agent/stream/aws-s3.yml.hbs @@ -36,6 +36,9 @@ credential_profile_name: {{credential_profile_name}} {{#if shared_credential_file}} shared_credential_file: {{shared_credential_file}} {{/if}} +{{#if endpoint}} +endpoint: {{endpoint}} +{{/if}} {{#if default_region}} default_region: {{default_region}} {{/if}} @@ -70,4 +73,4 @@ publisher_pipeline.disable_host: true {{#if processors}} processors: {{processors}} -{{/if}} \ No newline at end of file +{{/if}} diff --git a/packages/aws/data_stream/vpcflow/fields/agent.yml b/packages/aws/data_stream/vpcflow/fields/agent.yml index 196042e493f..b688ceac1b4 100644 --- a/packages/aws/data_stream/vpcflow/fields/agent.yml +++ b/packages/aws/data_stream/vpcflow/fields/agent.yml @@ -32,6 +32,7 @@ example: "stretch" description: > OS codename, if any. + - name: log.offset type: long description: Log offset diff --git a/packages/aws/data_stream/vpn/agent/stream/stream.yml.hbs b/packages/aws/data_stream/vpn/agent/stream/stream.yml.hbs index 017bbefd6b8..a41724ba223 100644 --- a/packages/aws/data_stream/vpn/agent/stream/stream.yml.hbs +++ b/packages/aws/data_stream/vpn/agent/stream/stream.yml.hbs @@ -24,6 +24,9 @@ shared_credential_file: {{shared_credential_file}} {{#if role_arn}} role_arn: {{role_arn}} {{/if}} +{{#if endpoint}} +endpoint: {{endpoint}} +{{/if}} {{#if default_region}} default_region: {{default_region}} {{/if}} diff --git a/packages/aws/data_stream/waf/agent/stream/aws-cloudwatch.yml.hbs b/packages/aws/data_stream/waf/agent/stream/aws-cloudwatch.yml.hbs index 934f97a44c3..8cc06c88c22 100644 --- a/packages/aws/data_stream/waf/agent/stream/aws-cloudwatch.yml.hbs +++ b/packages/aws/data_stream/waf/agent/stream/aws-cloudwatch.yml.hbs @@ -66,6 +66,9 @@ shared_credential_file: {{shared_credential_file}} {{#if api_timeout}} api_timeout: {{api_timeout}} {{/if}} +{{#if endpoint}} +endpoint: {{endpoint}} +{{/if}} {{#if default_region}} default_region: {{default_region}} {{/if}} diff --git a/packages/aws/data_stream/waf/agent/stream/aws-s3.yml.hbs b/packages/aws/data_stream/waf/agent/stream/aws-s3.yml.hbs index c38defcb33c..d0917006fb6 100644 --- a/packages/aws/data_stream/waf/agent/stream/aws-s3.yml.hbs +++ b/packages/aws/data_stream/waf/agent/stream/aws-s3.yml.hbs @@ -15,6 +15,9 @@ api_timeout: {{api_timeout}} {{#if max_number_of_messages}} max_number_of_messages: {{max_number_of_messages}} {{/if}} +{{#if endpoint}} +endpoint: {{endpoint}} +{{/if}} {{#if default_region}} default_region: {{default_region}} {{/if}} @@ -49,4 +52,4 @@ publisher_pipeline.disable_host: true {{#if processors}} processors: {{processors}} -{{/if}} \ No newline at end of file +{{/if}} diff --git a/packages/aws/data_stream/waf/fields/agent.yml b/packages/aws/data_stream/waf/fields/agent.yml index 1e09834aad6..2ddef01a27b 100644 --- a/packages/aws/data_stream/waf/fields/agent.yml +++ b/packages/aws/data_stream/waf/fields/agent.yml @@ -32,6 +32,7 @@ example: "stretch" description: > OS codename, if any. + - name: log.offset type: long description: Log offset diff --git a/packages/aws/manifest.yml b/packages/aws/manifest.yml index 9f962e3de16..ca693e875c8 100644 --- a/packages/aws/manifest.yml +++ b/packages/aws/manifest.yml @@ -1,7 +1,7 @@ format_version: 3.0.0 name: aws title: AWS -version: 2.15.1 +version: 2.15.2-preview01 description: Collect logs and metrics from Amazon Web Services (AWS) with Elastic Agent. type: integration categories: @@ -70,6 +70,14 @@ vars: show_user: false default: "" description: Default region to use prior to connecting to region specific services/endpoints if no AWS region is set from environment variable, credentials or instance profile. If none of the above are set and no default region is set as well, `us-east-1` is used. A region, either from environment variable, credentials or instance profile or from this default region setting, needs to be set when using regions in non-regular AWS environments such as AWS China or US Government Isolated. + - name: endpoint + type: text + title: Endpoint + multi: false + required: false + show_user: false + default: "" + description: URL of the entry point for an AWS web service - name: proxy_url type: text title: Proxy URL From 9a975420d5bb3bf00f856ea958939fb0e49d146c Mon Sep 17 00:00:00 2001 From: Kyle Pollich Date: Tue, 14 May 2024 15:36:45 -0400 Subject: [PATCH 02/14] Update PR in changelog --- packages/aws/changelog.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/aws/changelog.yml b/packages/aws/changelog.yml index 84d910067e2..b8465316fe0 100644 --- a/packages/aws/changelog.yml +++ b/packages/aws/changelog.yml @@ -3,7 +3,7 @@ changes: - description: Port endpoint package-level variable from AWS Custom Logs integration to AWS integration type: enhancement - link: https://github.com/elastic/integrations/pull/9705 + link: https://github.com/elastic/integrations/pull/9865 - version: "2.15.1" changes: - description: Don't index empty AWS Security Hub responses. From a881ffafe7fff638aedfc6bf99b433f892a45b65 Mon Sep 17 00:00:00 2001 From: Kyle Pollich Date: Thu, 23 May 2024 11:07:03 -0400 Subject: [PATCH 03/14] Add endpoint + region var to all SQS based policy templates --- packages/aws/changelog.yml | 2 +- .../agent/stream/aws-s3.yml.hbs | 3 +++ .../data_stream/apigateway_logs/manifest.yml | 16 ++++++++++++++ .../agent/stream/aws-s3.yml.hbs | 3 +++ .../data_stream/cloudfront_logs/manifest.yml | 16 ++++++++++++++ .../cloudtrail/agent/stream/aws-s3.yml.hbs | 4 +++- .../aws/data_stream/cloudtrail/manifest.yml | 22 ++++++++++++++++--- .../ec2_logs/agent/stream/aws-s3.yml.hbs | 3 +++ .../aws/data_stream/ec2_logs/manifest.yml | 16 ++++++++++++++ .../elb_logs/agent/stream/aws-s3.yml.hbs | 4 ++-- .../aws/data_stream/elb_logs/manifest.yml | 16 ++++++++++++++ .../emr_logs/agent/stream/aws-s3.yml.hbs | 3 +++ .../aws/data_stream/emr_logs/manifest.yml | 16 ++++++++++++++ .../firewall_logs/agent/stream/aws-s3.yml.hbs | 3 +++ .../data_stream/firewall_logs/manifest.yml | 16 ++++++++++++++ .../guardduty/agent/stream/aws-s3.yml.hbs | 8 ++++++- .../aws/data_stream/guardduty/manifest.yml | 16 ++++++++++++++ .../agent/stream/aws-s3.yml.hbs | 3 +++ .../route53_resolver_logs/manifest.yml | 16 ++++++++++++++ .../s3access/agent/stream/aws-s3.yml.hbs | 3 +++ .../aws/data_stream/s3access/manifest.yml | 16 ++++++++++++++ .../vpcflow/agent/stream/aws-s3.yml.hbs | 3 +++ packages/aws/data_stream/vpcflow/manifest.yml | 16 ++++++++++++++ .../waf/agent/stream/aws-s3.yml.hbs | 3 +++ packages/aws/data_stream/waf/manifest.yml | 16 ++++++++++++++ packages/aws/manifest.yml | 8 ------- 26 files changed, 235 insertions(+), 16 deletions(-) diff --git a/packages/aws/changelog.yml b/packages/aws/changelog.yml index 6a6f9f18a0d..9e41f3dcc09 100644 --- a/packages/aws/changelog.yml +++ b/packages/aws/changelog.yml @@ -1,7 +1,7 @@ # newer versions go on top - version: "2.15.3" changes: - - description: Port endpoint package-level variable from AWS Custom Logs integration to AWS integration + - description: Add endpoint + region variables to all SQS based AWS integrations. type: enhancement link: https://github.com/elastic/integrations/pull/9865 - version: "2.15.2" diff --git a/packages/aws/data_stream/apigateway_logs/agent/stream/aws-s3.yml.hbs b/packages/aws/data_stream/apigateway_logs/agent/stream/aws-s3.yml.hbs index 7c66e7a31ee..0d19877d02d 100644 --- a/packages/aws/data_stream/apigateway_logs/agent/stream/aws-s3.yml.hbs +++ b/packages/aws/data_stream/apigateway_logs/agent/stream/aws-s3.yml.hbs @@ -38,6 +38,9 @@ shared_credential_file: {{shared_credential_file}} {{#if endpoint}} endpoint: {{endpoint}} {{/if}} +{{#if region}} +region: {{region}} +{{/if}} {{#if default_region}} default_region: {{default_region}} {{/if}} diff --git a/packages/aws/data_stream/apigateway_logs/manifest.yml b/packages/aws/data_stream/apigateway_logs/manifest.yml index 9a46702de0d..964761ce462 100644 --- a/packages/aws/data_stream/apigateway_logs/manifest.yml +++ b/packages/aws/data_stream/apigateway_logs/manifest.yml @@ -29,6 +29,22 @@ streams: required: false show_user: true description: Mandatory if the "Collect logs via S3 Bucket" switch is off. URL of the AWS SQS queue that messages will be received from. + - name: endpoint + type: text + title: Endpoint + multi: false + required: false + show_user: false + default: "" + description: URL of the entry point for an AWS web service + - name: region + type: text + title: Region + multi: false + required: false + show_user: false + default: "" + description: The name of the AWS region of the endpoint - name: bucket_list_prefix type: text title: "[S3] Bucket Prefix" diff --git a/packages/aws/data_stream/cloudfront_logs/agent/stream/aws-s3.yml.hbs b/packages/aws/data_stream/cloudfront_logs/agent/stream/aws-s3.yml.hbs index 8b130a4c568..4d4388d441b 100644 --- a/packages/aws/data_stream/cloudfront_logs/agent/stream/aws-s3.yml.hbs +++ b/packages/aws/data_stream/cloudfront_logs/agent/stream/aws-s3.yml.hbs @@ -17,6 +17,9 @@ max_number_of_messages: {{max_number_of_messages}} {{#if endpoint}} endpoint: {{endpoint}} {{/if}} +{{#if region}} +region: {{region}} +{{/if}} {{#if default_region}} default_region: {{default_region}} {{/if}} diff --git a/packages/aws/data_stream/cloudfront_logs/manifest.yml b/packages/aws/data_stream/cloudfront_logs/manifest.yml index 674490658eb..349d530f418 100644 --- a/packages/aws/data_stream/cloudfront_logs/manifest.yml +++ b/packages/aws/data_stream/cloudfront_logs/manifest.yml @@ -27,6 +27,22 @@ streams: required: true show_user: true description: URL of the AWS SQS queue that messages will be received from. + - name: endpoint + type: text + title: Endpoint + multi: false + required: false + show_user: false + default: "" + description: URL of the entry point for an AWS web service + - name: region + type: text + title: Region + multi: false + required: false + show_user: false + default: "" + description: The name of the AWS region of the endpoint - name: fips_enabled type: bool title: Enable S3 FIPS diff --git a/packages/aws/data_stream/cloudtrail/agent/stream/aws-s3.yml.hbs b/packages/aws/data_stream/cloudtrail/agent/stream/aws-s3.yml.hbs index 038d3b5f5e8..6d36e8cee47 100644 --- a/packages/aws/data_stream/cloudtrail/agent/stream/aws-s3.yml.hbs +++ b/packages/aws/data_stream/cloudtrail/agent/stream/aws-s3.yml.hbs @@ -49,10 +49,12 @@ credential_profile_name: {{credential_profile_name}} {{#if shared_credential_file}} shared_credential_file: {{shared_credential_file}} {{/if}} - {{#if endpoint}} endpoint: {{endpoint}} {{/if}} +{{#if region}} +region: {{region}} +{{/if}} {{#if default_region}} default_region: {{default_region}} {{/if}} diff --git a/packages/aws/data_stream/cloudtrail/manifest.yml b/packages/aws/data_stream/cloudtrail/manifest.yml index ca548f11d96..21790c2fded 100644 --- a/packages/aws/data_stream/cloudtrail/manifest.yml +++ b/packages/aws/data_stream/cloudtrail/manifest.yml @@ -28,6 +28,22 @@ streams: required: false show_user: true description: Mandatory if the "Collect logs via S3 Bucket" switch is off. URL of the AWS SQS queue that messages will be received from. + - name: endpoint + type: text + title: Endpoint + multi: false + required: false + show_user: false + default: "" + description: URL of the entry point for an AWS web service + - name: region + type: text + title: Region + multi: false + required: false + show_user: false + default: "" + description: The name of the AWS region of the endpoint - name: bucket_list_prefix type: text title: "[S3] Bucket Prefix" @@ -101,7 +117,7 @@ streams: - name: cloudtrail_regex type: text title: CloudTrail Logs regex - default: '/CloudTrail/' + default: "/CloudTrail/" required: false show_user: false description: | @@ -110,7 +126,7 @@ streams: - name: cloudtrail_digest_regex type: text title: CloudTrail Digest Logs regex - default: '/CloudTrail-Digest/' + default: "/CloudTrail-Digest/" required: false show_user: false description: | @@ -119,7 +135,7 @@ streams: - name: cloudtrail_insight_regex type: text title: CloudTrail Insight Logs regex - default: '/CloudTrail-Insight/' + default: "/CloudTrail-Insight/" required: false show_user: false description: | diff --git a/packages/aws/data_stream/ec2_logs/agent/stream/aws-s3.yml.hbs b/packages/aws/data_stream/ec2_logs/agent/stream/aws-s3.yml.hbs index dfdeeb91a28..152e77a7665 100644 --- a/packages/aws/data_stream/ec2_logs/agent/stream/aws-s3.yml.hbs +++ b/packages/aws/data_stream/ec2_logs/agent/stream/aws-s3.yml.hbs @@ -17,6 +17,9 @@ max_number_of_messages: {{max_number_of_messages}} {{#if endpoint}} endpoint: {{endpoint}} {{/if}} +{{#if region}} +region: {{region}} +{{/if}} {{#if default_region}} default_region: {{default_region}} {{/if}} diff --git a/packages/aws/data_stream/ec2_logs/manifest.yml b/packages/aws/data_stream/ec2_logs/manifest.yml index df69ef4bcc3..a0c2580a5e1 100644 --- a/packages/aws/data_stream/ec2_logs/manifest.yml +++ b/packages/aws/data_stream/ec2_logs/manifest.yml @@ -28,6 +28,22 @@ streams: required: true show_user: true description: URL of the AWS SQS queue that messages will be received from. + - name: endpoint + type: text + title: Endpoint + multi: false + required: false + show_user: false + default: "" + description: URL of the entry point for an AWS web service + - name: region + type: text + title: Region + multi: false + required: false + show_user: false + default: "" + description: The name of the AWS region of the endpoint - name: fips_enabled type: bool title: Enable S3 FIPS diff --git a/packages/aws/data_stream/elb_logs/agent/stream/aws-s3.yml.hbs b/packages/aws/data_stream/elb_logs/agent/stream/aws-s3.yml.hbs index d4fdba80f0e..152e77a7665 100644 --- a/packages/aws/data_stream/elb_logs/agent/stream/aws-s3.yml.hbs +++ b/packages/aws/data_stream/elb_logs/agent/stream/aws-s3.yml.hbs @@ -17,8 +17,8 @@ max_number_of_messages: {{max_number_of_messages}} {{#if endpoint}} endpoint: {{endpoint}} {{/if}} -{{#if endpoint}} -endpoint: {{endpoint}} +{{#if region}} +region: {{region}} {{/if}} {{#if default_region}} default_region: {{default_region}} diff --git a/packages/aws/data_stream/elb_logs/manifest.yml b/packages/aws/data_stream/elb_logs/manifest.yml index 17dbabac05e..cc2842b3829 100644 --- a/packages/aws/data_stream/elb_logs/manifest.yml +++ b/packages/aws/data_stream/elb_logs/manifest.yml @@ -27,6 +27,22 @@ streams: required: true show_user: true description: URL of the AWS SQS queue that messages will be received from. + - name: endpoint + type: text + title: Endpoint + multi: false + required: false + show_user: false + default: "" + description: URL of the entry point for an AWS web service + - name: region + type: text + title: Region + multi: false + required: false + show_user: false + default: "" + description: The name of the AWS region of the endpoint - name: fips_enabled type: bool title: Enable S3 FIPS diff --git a/packages/aws/data_stream/emr_logs/agent/stream/aws-s3.yml.hbs b/packages/aws/data_stream/emr_logs/agent/stream/aws-s3.yml.hbs index 9e59987afdb..2f110dad920 100644 --- a/packages/aws/data_stream/emr_logs/agent/stream/aws-s3.yml.hbs +++ b/packages/aws/data_stream/emr_logs/agent/stream/aws-s3.yml.hbs @@ -37,6 +37,9 @@ file_selectors: {{#if endpoint}} endpoint: {{endpoint}} {{/if}} +{{#if region}} +region: {{region}} +{{/if}} {{#if default_region}} default_region: {{default_region}} {{/if}} diff --git a/packages/aws/data_stream/emr_logs/manifest.yml b/packages/aws/data_stream/emr_logs/manifest.yml index 27e72b79859..684e7f4b227 100644 --- a/packages/aws/data_stream/emr_logs/manifest.yml +++ b/packages/aws/data_stream/emr_logs/manifest.yml @@ -30,6 +30,22 @@ streams: show_user: true default: false description: Mandatory if the "Collect logs via S3 Bucket" switch is off. URL of the AWS SQS queue that messages will be received from. + - name: endpoint + type: text + title: Endpoint + multi: false + required: false + show_user: false + default: "" + description: URL of the entry point for an AWS web service + - name: region + type: text + title: Region + multi: false + required: false + show_user: false + default: "" + description: The name of the AWS region of the end point - name: bucket_list_prefix type: text title: "[S3] Bucket Prefix" diff --git a/packages/aws/data_stream/firewall_logs/agent/stream/aws-s3.yml.hbs b/packages/aws/data_stream/firewall_logs/agent/stream/aws-s3.yml.hbs index 196dda8a63e..b3c81c9f38d 100644 --- a/packages/aws/data_stream/firewall_logs/agent/stream/aws-s3.yml.hbs +++ b/packages/aws/data_stream/firewall_logs/agent/stream/aws-s3.yml.hbs @@ -17,6 +17,9 @@ max_number_of_messages: {{max_number_of_messages}} {{#if endpoint}} endpoint: {{endpoint}} {{/if}} +{{#if region}} +region: {{region}} +{{/if}} {{#if default_region}} default_region: {{default_region}} {{/if}} diff --git a/packages/aws/data_stream/firewall_logs/manifest.yml b/packages/aws/data_stream/firewall_logs/manifest.yml index 8c1334a94b4..c7a274a5068 100644 --- a/packages/aws/data_stream/firewall_logs/manifest.yml +++ b/packages/aws/data_stream/firewall_logs/manifest.yml @@ -27,6 +27,22 @@ streams: required: true show_user: true description: URL of the AWS SQS queue that messages will be received from. + - name: endpoint + type: text + title: Endpoint + multi: false + required: false + show_user: false + default: "" + description: URL of the entry point for an AWS web service + - name: region + type: text + title: Region + multi: false + required: false + show_user: false + default: "" + description: The name of the AWS region of the end point - name: fips_enabled type: bool title: Enable S3 FIPS diff --git a/packages/aws/data_stream/guardduty/agent/stream/aws-s3.yml.hbs b/packages/aws/data_stream/guardduty/agent/stream/aws-s3.yml.hbs index 9d48e69a97c..6c621c8aac5 100644 --- a/packages/aws/data_stream/guardduty/agent/stream/aws-s3.yml.hbs +++ b/packages/aws/data_stream/guardduty/agent/stream/aws-s3.yml.hbs @@ -34,6 +34,12 @@ file_selectors: {{/if}} +{{#if endpoint}} +endpoint: {{endpoint}} +{{/if}} +{{#if region}} +region: {{region}} +{{/if}} {{#if access_key_id}} access_key_id: {{access_key_id}} {{/if}} @@ -79,4 +85,4 @@ publisher_pipeline.disable_host: true {{#if processors}} processors: {{processors}} -{{/if}} \ No newline at end of file +{{/if}} diff --git a/packages/aws/data_stream/guardduty/manifest.yml b/packages/aws/data_stream/guardduty/manifest.yml index 08af59ad4c4..b8e1485f9cb 100644 --- a/packages/aws/data_stream/guardduty/manifest.yml +++ b/packages/aws/data_stream/guardduty/manifest.yml @@ -197,6 +197,22 @@ streams: required: false show_user: false description: If the SQS queue will have events that correspond to files that this integration shouldn’t process, file_selectors can be used to limit the files that are downloaded. This is a list of selectors which are made up of regex and expand_event_list_from_field options. The regex should match the S3 object key in the SQS message, and the optional expand_event_list_from_field is the same as the global setting. If file_selectors is given, then any global expand_event_list_from_field value is ignored in favor of the ones specified in the file_selectors. Regexes use [RE2 syntax](https://pkg.go.dev/regexp/syntax). Files that don’t match one of the regexes will not be processed. + - name: endpoint + type: text + title: Endpoint + multi: false + required: false + show_user: false + default: "" + description: URL of the entry point for an AWS web service + - name: region + type: text + title: Region + multi: false + required: false + show_user: false + default: "" + description: The name of the AWS region of the end point - name: fips_enabled type: bool title: Enable S3 FIPS diff --git a/packages/aws/data_stream/route53_resolver_logs/agent/stream/aws-s3.yml.hbs b/packages/aws/data_stream/route53_resolver_logs/agent/stream/aws-s3.yml.hbs index 196dda8a63e..b3c81c9f38d 100644 --- a/packages/aws/data_stream/route53_resolver_logs/agent/stream/aws-s3.yml.hbs +++ b/packages/aws/data_stream/route53_resolver_logs/agent/stream/aws-s3.yml.hbs @@ -17,6 +17,9 @@ max_number_of_messages: {{max_number_of_messages}} {{#if endpoint}} endpoint: {{endpoint}} {{/if}} +{{#if region}} +region: {{region}} +{{/if}} {{#if default_region}} default_region: {{default_region}} {{/if}} diff --git a/packages/aws/data_stream/route53_resolver_logs/manifest.yml b/packages/aws/data_stream/route53_resolver_logs/manifest.yml index 294a6b204fd..140936cf050 100644 --- a/packages/aws/data_stream/route53_resolver_logs/manifest.yml +++ b/packages/aws/data_stream/route53_resolver_logs/manifest.yml @@ -143,6 +143,22 @@ streams: required: true show_user: true description: URL of the AWS SQS queue that messages will be received from. + - name: endpoint + type: text + title: Endpoint + multi: false + required: false + show_user: false + default: "" + description: URL of the entry point for an AWS web service + - name: region + type: text + title: Region + multi: false + required: false + show_user: false + default: "" + description: The name of the AWS region of the end point - name: fips_enabled type: bool title: Enable S3 FIPS diff --git a/packages/aws/data_stream/s3access/agent/stream/aws-s3.yml.hbs b/packages/aws/data_stream/s3access/agent/stream/aws-s3.yml.hbs index dfdeeb91a28..152e77a7665 100644 --- a/packages/aws/data_stream/s3access/agent/stream/aws-s3.yml.hbs +++ b/packages/aws/data_stream/s3access/agent/stream/aws-s3.yml.hbs @@ -17,6 +17,9 @@ max_number_of_messages: {{max_number_of_messages}} {{#if endpoint}} endpoint: {{endpoint}} {{/if}} +{{#if region}} +region: {{region}} +{{/if}} {{#if default_region}} default_region: {{default_region}} {{/if}} diff --git a/packages/aws/data_stream/s3access/manifest.yml b/packages/aws/data_stream/s3access/manifest.yml index 8240665215a..08853d6f0fb 100644 --- a/packages/aws/data_stream/s3access/manifest.yml +++ b/packages/aws/data_stream/s3access/manifest.yml @@ -27,6 +27,22 @@ streams: required: true show_user: true description: URL of the AWS SQS queue that messages will be received from. + - name: endpoint + type: text + title: Endpoint + multi: false + required: false + show_user: false + default: "" + description: URL of the entry point for an AWS web service + - name: region + type: text + title: Region + multi: false + required: false + show_user: false + default: "" + description: The name of the AWS region of the end point - name: fips_enabled type: bool title: Enable S3 FIPS diff --git a/packages/aws/data_stream/vpcflow/agent/stream/aws-s3.yml.hbs b/packages/aws/data_stream/vpcflow/agent/stream/aws-s3.yml.hbs index 76c57cd135d..272d9011769 100644 --- a/packages/aws/data_stream/vpcflow/agent/stream/aws-s3.yml.hbs +++ b/packages/aws/data_stream/vpcflow/agent/stream/aws-s3.yml.hbs @@ -39,6 +39,9 @@ shared_credential_file: {{shared_credential_file}} {{#if endpoint}} endpoint: {{endpoint}} {{/if}} +{{#if region}} +region: {{region}} +{{/if}} {{#if default_region}} default_region: {{default_region}} {{/if}} diff --git a/packages/aws/data_stream/vpcflow/manifest.yml b/packages/aws/data_stream/vpcflow/manifest.yml index a8117ff5a00..0f900819dd1 100644 --- a/packages/aws/data_stream/vpcflow/manifest.yml +++ b/packages/aws/data_stream/vpcflow/manifest.yml @@ -73,6 +73,22 @@ streams: default: 5 required: false show_user: false + - name: endpoint + type: text + title: Endpoint + multi: false + required: false + show_user: false + default: "" + description: URL of the entry point for an AWS web service + - name: region + type: text + title: Region + multi: false + required: false + show_user: false + default: "" + description: The name of the AWS region of the end point - name: fips_enabled type: bool title: Enable S3 FIPS diff --git a/packages/aws/data_stream/waf/agent/stream/aws-s3.yml.hbs b/packages/aws/data_stream/waf/agent/stream/aws-s3.yml.hbs index d0917006fb6..54b0042b770 100644 --- a/packages/aws/data_stream/waf/agent/stream/aws-s3.yml.hbs +++ b/packages/aws/data_stream/waf/agent/stream/aws-s3.yml.hbs @@ -18,6 +18,9 @@ max_number_of_messages: {{max_number_of_messages}} {{#if endpoint}} endpoint: {{endpoint}} {{/if}} +{{#if region}} +region: {{region}} +{{/if}} {{#if default_region}} default_region: {{default_region}} {{/if}} diff --git a/packages/aws/data_stream/waf/manifest.yml b/packages/aws/data_stream/waf/manifest.yml index 892bfb078a8..eff771d14c9 100644 --- a/packages/aws/data_stream/waf/manifest.yml +++ b/packages/aws/data_stream/waf/manifest.yml @@ -27,6 +27,22 @@ streams: required: true show_user: true description: URL of the AWS SQS queue that messages will be received from. + - name: endpoint + type: text + title: Endpoint + multi: false + required: false + show_user: false + default: "" + description: URL of the entry point for an AWS web service + - name: region + type: text + title: Region + multi: false + required: false + show_user: false + default: "" + description: The name of the AWS region of the end point - name: fips_enabled type: bool title: Enable S3 FIPS diff --git a/packages/aws/manifest.yml b/packages/aws/manifest.yml index b1bd95f8896..50d2e1c0e50 100644 --- a/packages/aws/manifest.yml +++ b/packages/aws/manifest.yml @@ -70,14 +70,6 @@ vars: show_user: false default: "" description: Default region to use prior to connecting to region specific services/endpoints if no AWS region is set from environment variable, credentials or instance profile. If none of the above are set and no default region is set as well, `us-east-1` is used. A region, either from environment variable, credentials or instance profile or from this default region setting, needs to be set when using regions in non-regular AWS environments such as AWS China or US Government Isolated. - - name: endpoint - type: text - title: Endpoint - multi: false - required: false - show_user: false - default: "" - description: URL of the entry point for an AWS web service - name: proxy_url type: text title: Proxy URL From 8d7bb242fc722cb20a23fe3e6ed743f7bcbdea7b Mon Sep 17 00:00:00 2001 From: Kyle Pollich Date: Thu, 23 May 2024 11:30:57 -0400 Subject: [PATCH 04/14] Remove endpoint from non-SQS inputs --- .../apigateway_metrics/agent/stream/stream.yml.hbs | 3 --- .../aws/data_stream/billing/agent/stream/stream.yml.hbs | 3 --- .../cloudwatch_logs/agent/stream/aws-cloudwatch.yml.hbs | 3 --- .../cloudwatch_metrics/agent/stream/stream.yml.hbs | 3 --- .../aws/data_stream/dynamodb/agent/stream/stream.yml.hbs | 3 --- packages/aws/data_stream/ebs/agent/stream/stream.yml.hbs | 3 --- .../aws/data_stream/ec2_metrics/agent/stream/stream.yml.hbs | 3 --- .../aws/data_stream/ecs_metrics/agent/stream/stream.yml.hbs | 3 --- .../aws/data_stream/elb_metrics/agent/stream/stream.yml.hbs | 3 --- .../aws/data_stream/emr_metrics/agent/stream/stream.yml.hbs | 3 --- .../firewall_metrics/agent/stream/stream.yml.hbs | 3 --- .../data_stream/kafka_metrics/agent/stream/stream.yml.hbs | 3 --- .../aws/data_stream/kinesis/agent/stream/stream.yml.hbs | 3 --- packages/aws/data_stream/lambda/agent/stream/stream.yml.hbs | 3 --- .../aws/data_stream/natgateway/agent/stream/stream.yml.hbs | 3 --- packages/aws/data_stream/rds/agent/stream/stream.yml.hbs | 3 --- .../aws/data_stream/redshift/agent/stream/stream.yml.hbs | 6 ------ .../route53_public_logs/agent/stream/aws-cloudwatch.yml.hbs | 3 --- .../agent/stream/aws-cloudwatch.yml.hbs | 3 --- .../s3_daily_storage/agent/stream/stream.yml.hbs | 3 --- .../aws/data_stream/s3_request/agent/stream/stream.yml.hbs | 3 --- .../data_stream/s3_storage_lens/agent/stream/stream.yml.hbs | 3 --- packages/aws/data_stream/sns/agent/stream/stream.yml.hbs | 3 --- packages/aws/data_stream/sqs/agent/stream/stream.yml.hbs | 3 --- .../data_stream/transitgateway/agent/stream/stream.yml.hbs | 3 --- packages/aws/data_stream/usage/agent/stream/stream.yml.hbs | 3 --- 26 files changed, 81 deletions(-) diff --git a/packages/aws/data_stream/apigateway_metrics/agent/stream/stream.yml.hbs b/packages/aws/data_stream/apigateway_metrics/agent/stream/stream.yml.hbs index 321ba873a5a..0b1a916ac72 100644 --- a/packages/aws/data_stream/apigateway_metrics/agent/stream/stream.yml.hbs +++ b/packages/aws/data_stream/apigateway_metrics/agent/stream/stream.yml.hbs @@ -24,9 +24,6 @@ shared_credential_file: {{shared_credential_file}} {{#if role_arn}} role_arn: {{role_arn}} {{/if}} -{{#if endpoint}} -endpoint: {{endpoint}} -{{/if}} {{#if default_region}} default_region: {{default_region}} {{/if}} diff --git a/packages/aws/data_stream/billing/agent/stream/stream.yml.hbs b/packages/aws/data_stream/billing/agent/stream/stream.yml.hbs index 8e0cce0ca62..fbbfd6733a7 100644 --- a/packages/aws/data_stream/billing/agent/stream/stream.yml.hbs +++ b/packages/aws/data_stream/billing/agent/stream/stream.yml.hbs @@ -24,9 +24,6 @@ shared_credential_file: {{shared_credential_file}} {{#if role_arn}} role_arn: {{role_arn}} {{/if}} -{{#if endpoint}} -endpoint: {{endpoint}} -{{/if}} {{#if default_region}} default_region: {{default_region}} {{/if}} diff --git a/packages/aws/data_stream/cloudwatch_logs/agent/stream/aws-cloudwatch.yml.hbs b/packages/aws/data_stream/cloudwatch_logs/agent/stream/aws-cloudwatch.yml.hbs index 0f789a733b2..9588557d158 100644 --- a/packages/aws/data_stream/cloudwatch_logs/agent/stream/aws-cloudwatch.yml.hbs +++ b/packages/aws/data_stream/cloudwatch_logs/agent/stream/aws-cloudwatch.yml.hbs @@ -69,9 +69,6 @@ shared_credential_file: {{shared_credential_file}} {{#if api_timeout}} api_timeout: {{api_timeout}} {{/if}} -{{#if endpoint}} -endpoint: {{endpoint}} -{{/if}} {{#if default_region}} default_region: {{default_region}} {{/if}} diff --git a/packages/aws/data_stream/cloudwatch_metrics/agent/stream/stream.yml.hbs b/packages/aws/data_stream/cloudwatch_metrics/agent/stream/stream.yml.hbs index 9bccc83fb15..fbf98ab3a0a 100644 --- a/packages/aws/data_stream/cloudwatch_metrics/agent/stream/stream.yml.hbs +++ b/packages/aws/data_stream/cloudwatch_metrics/agent/stream/stream.yml.hbs @@ -24,9 +24,6 @@ shared_credential_file: {{shared_credential_file}} {{#if role_arn}} role_arn: {{role_arn}} {{/if}} -{{#if endpoint}} -endpoint: {{endpoint}} -{{/if}} {{#if default_region}} default_region: {{default_region}} {{/if}} diff --git a/packages/aws/data_stream/dynamodb/agent/stream/stream.yml.hbs b/packages/aws/data_stream/dynamodb/agent/stream/stream.yml.hbs index 8121c7b3555..0f40351ea3b 100644 --- a/packages/aws/data_stream/dynamodb/agent/stream/stream.yml.hbs +++ b/packages/aws/data_stream/dynamodb/agent/stream/stream.yml.hbs @@ -24,9 +24,6 @@ shared_credential_file: {{shared_credential_file}} {{#if role_arn}} role_arn: {{role_arn}} {{/if}} -{{#if endpoint}} -endpoint: {{endpoint}} -{{/if}} {{#if default_region}} default_region: {{default_region}} {{/if}} diff --git a/packages/aws/data_stream/ebs/agent/stream/stream.yml.hbs b/packages/aws/data_stream/ebs/agent/stream/stream.yml.hbs index 6c8340e738a..89615c3737a 100644 --- a/packages/aws/data_stream/ebs/agent/stream/stream.yml.hbs +++ b/packages/aws/data_stream/ebs/agent/stream/stream.yml.hbs @@ -24,9 +24,6 @@ shared_credential_file: {{shared_credential_file}} {{#if role_arn}} role_arn: {{role_arn}} {{/if}} -{{#if endpoint}} -endpoint: {{endpoint}} -{{/if}} {{#if default_region}} default_region: {{default_region}} {{/if}} diff --git a/packages/aws/data_stream/ec2_metrics/agent/stream/stream.yml.hbs b/packages/aws/data_stream/ec2_metrics/agent/stream/stream.yml.hbs index 02dbe07a92a..12d3f6848f3 100644 --- a/packages/aws/data_stream/ec2_metrics/agent/stream/stream.yml.hbs +++ b/packages/aws/data_stream/ec2_metrics/agent/stream/stream.yml.hbs @@ -24,9 +24,6 @@ shared_credential_file: {{shared_credential_file}} {{#if role_arn}} role_arn: {{role_arn}} {{/if}} -{{#if endpoint}} -endpoint: {{endpoint}} -{{/if}} {{#if default_region}} default_region: {{default_region}} {{/if}} diff --git a/packages/aws/data_stream/ecs_metrics/agent/stream/stream.yml.hbs b/packages/aws/data_stream/ecs_metrics/agent/stream/stream.yml.hbs index 67788f32081..16607804cec 100644 --- a/packages/aws/data_stream/ecs_metrics/agent/stream/stream.yml.hbs +++ b/packages/aws/data_stream/ecs_metrics/agent/stream/stream.yml.hbs @@ -27,9 +27,6 @@ role_arn: {{role_arn}} {{#if endpoint}} endpoint: {{endpoint}} {{/if}} -{{#if endpoint}} -endpoint: {{endpoint}} -{{/if}} {{#if default_region}} default_region: {{default_region}} {{/if}} diff --git a/packages/aws/data_stream/elb_metrics/agent/stream/stream.yml.hbs b/packages/aws/data_stream/elb_metrics/agent/stream/stream.yml.hbs index cf3d2ff7917..154f98e296f 100644 --- a/packages/aws/data_stream/elb_metrics/agent/stream/stream.yml.hbs +++ b/packages/aws/data_stream/elb_metrics/agent/stream/stream.yml.hbs @@ -24,9 +24,6 @@ shared_credential_file: {{shared_credential_file}} {{#if role_arn}} role_arn: {{role_arn}} {{/if}} -{{#if endpoint}} -endpoint: {{endpoint}} -{{/if}} {{#if default_region}} default_region: {{default_region}} {{/if}} diff --git a/packages/aws/data_stream/emr_metrics/agent/stream/stream.yml.hbs b/packages/aws/data_stream/emr_metrics/agent/stream/stream.yml.hbs index 67fd238e8c9..00fcc848016 100644 --- a/packages/aws/data_stream/emr_metrics/agent/stream/stream.yml.hbs +++ b/packages/aws/data_stream/emr_metrics/agent/stream/stream.yml.hbs @@ -24,9 +24,6 @@ shared_credential_file: {{shared_credential_file}} {{#if role_arn}} role_arn: {{role_arn}} {{/if}} -{{#if endpoint}} -endpoint: {{endpoint}} -{{/if}} {{#if default_region}} default_region: {{default_region}} {{/if}} diff --git a/packages/aws/data_stream/firewall_metrics/agent/stream/stream.yml.hbs b/packages/aws/data_stream/firewall_metrics/agent/stream/stream.yml.hbs index 9bccc83fb15..fbf98ab3a0a 100644 --- a/packages/aws/data_stream/firewall_metrics/agent/stream/stream.yml.hbs +++ b/packages/aws/data_stream/firewall_metrics/agent/stream/stream.yml.hbs @@ -24,9 +24,6 @@ shared_credential_file: {{shared_credential_file}} {{#if role_arn}} role_arn: {{role_arn}} {{/if}} -{{#if endpoint}} -endpoint: {{endpoint}} -{{/if}} {{#if default_region}} default_region: {{default_region}} {{/if}} diff --git a/packages/aws/data_stream/kafka_metrics/agent/stream/stream.yml.hbs b/packages/aws/data_stream/kafka_metrics/agent/stream/stream.yml.hbs index b3194ecf505..395ce09518d 100644 --- a/packages/aws/data_stream/kafka_metrics/agent/stream/stream.yml.hbs +++ b/packages/aws/data_stream/kafka_metrics/agent/stream/stream.yml.hbs @@ -24,9 +24,6 @@ shared_credential_file: {{shared_credential_file}} {{#if role_arn}} role_arn: {{role_arn}} {{/if}} -{{#if endpoint}} -endpoint: {{endpoint}} -{{/if}} {{#if default_region}} default_region: {{default_region}} {{/if}} diff --git a/packages/aws/data_stream/kinesis/agent/stream/stream.yml.hbs b/packages/aws/data_stream/kinesis/agent/stream/stream.yml.hbs index 0d75c9dcc5d..e0ec3daefc3 100644 --- a/packages/aws/data_stream/kinesis/agent/stream/stream.yml.hbs +++ b/packages/aws/data_stream/kinesis/agent/stream/stream.yml.hbs @@ -24,9 +24,6 @@ shared_credential_file: {{shared_credential_file}} {{#if role_arn}} role_arn: {{role_arn}} {{/if}} -{{#if endpoint}} -endpoint: {{endpoint}} -{{/if}} {{#if default_region}} default_region: {{default_region}} {{/if}} diff --git a/packages/aws/data_stream/lambda/agent/stream/stream.yml.hbs b/packages/aws/data_stream/lambda/agent/stream/stream.yml.hbs index 7a43c14e13f..633fc3f698f 100644 --- a/packages/aws/data_stream/lambda/agent/stream/stream.yml.hbs +++ b/packages/aws/data_stream/lambda/agent/stream/stream.yml.hbs @@ -24,9 +24,6 @@ shared_credential_file: {{shared_credential_file}} {{#if role_arn}} role_arn: {{role_arn}} {{/if}} -{{#if endpoint}} -endpoint: {{endpoint}} -{{/if}} {{#if default_region}} default_region: {{default_region}} {{/if}} diff --git a/packages/aws/data_stream/natgateway/agent/stream/stream.yml.hbs b/packages/aws/data_stream/natgateway/agent/stream/stream.yml.hbs index 268edb1bdf0..c260b3d0aa7 100644 --- a/packages/aws/data_stream/natgateway/agent/stream/stream.yml.hbs +++ b/packages/aws/data_stream/natgateway/agent/stream/stream.yml.hbs @@ -24,9 +24,6 @@ shared_credential_file: {{shared_credential_file}} {{#if role_arn}} role_arn: {{role_arn}} {{/if}} -{{#if endpoint}} -endpoint: {{endpoint}} -{{/if}} {{#if default_region}} default_region: {{default_region}} {{/if}} diff --git a/packages/aws/data_stream/rds/agent/stream/stream.yml.hbs b/packages/aws/data_stream/rds/agent/stream/stream.yml.hbs index fc4c0739092..a19b2b409d6 100644 --- a/packages/aws/data_stream/rds/agent/stream/stream.yml.hbs +++ b/packages/aws/data_stream/rds/agent/stream/stream.yml.hbs @@ -24,9 +24,6 @@ shared_credential_file: {{shared_credential_file}} {{#if role_arn}} role_arn: {{role_arn}} {{/if}} -{{#if endpoint}} -endpoint: {{endpoint}} -{{/if}} {{#if default_region}} default_region: {{default_region}} {{/if}} diff --git a/packages/aws/data_stream/redshift/agent/stream/stream.yml.hbs b/packages/aws/data_stream/redshift/agent/stream/stream.yml.hbs index 5e430cedcfa..d3d0eff8669 100644 --- a/packages/aws/data_stream/redshift/agent/stream/stream.yml.hbs +++ b/packages/aws/data_stream/redshift/agent/stream/stream.yml.hbs @@ -24,12 +24,6 @@ shared_credential_file: {{shared_credential_file}} {{#if role_arn}} role_arn: {{role_arn}} {{/if}} -{{#if endpoint}} -endpoint: {{endpoint}} -{{/if}} -{{#if endpoint}} -endpoint: {{endpoint}} -{{/if}} {{#if default_region}} default_region: {{default_region}} {{/if}} diff --git a/packages/aws/data_stream/route53_public_logs/agent/stream/aws-cloudwatch.yml.hbs b/packages/aws/data_stream/route53_public_logs/agent/stream/aws-cloudwatch.yml.hbs index 8cc06c88c22..934f97a44c3 100644 --- a/packages/aws/data_stream/route53_public_logs/agent/stream/aws-cloudwatch.yml.hbs +++ b/packages/aws/data_stream/route53_public_logs/agent/stream/aws-cloudwatch.yml.hbs @@ -66,9 +66,6 @@ shared_credential_file: {{shared_credential_file}} {{#if api_timeout}} api_timeout: {{api_timeout}} {{/if}} -{{#if endpoint}} -endpoint: {{endpoint}} -{{/if}} {{#if default_region}} default_region: {{default_region}} {{/if}} diff --git a/packages/aws/data_stream/route53_resolver_logs/agent/stream/aws-cloudwatch.yml.hbs b/packages/aws/data_stream/route53_resolver_logs/agent/stream/aws-cloudwatch.yml.hbs index 8cc06c88c22..934f97a44c3 100644 --- a/packages/aws/data_stream/route53_resolver_logs/agent/stream/aws-cloudwatch.yml.hbs +++ b/packages/aws/data_stream/route53_resolver_logs/agent/stream/aws-cloudwatch.yml.hbs @@ -66,9 +66,6 @@ shared_credential_file: {{shared_credential_file}} {{#if api_timeout}} api_timeout: {{api_timeout}} {{/if}} -{{#if endpoint}} -endpoint: {{endpoint}} -{{/if}} {{#if default_region}} default_region: {{default_region}} {{/if}} diff --git a/packages/aws/data_stream/s3_daily_storage/agent/stream/stream.yml.hbs b/packages/aws/data_stream/s3_daily_storage/agent/stream/stream.yml.hbs index 531cb9d47e2..b072f249870 100644 --- a/packages/aws/data_stream/s3_daily_storage/agent/stream/stream.yml.hbs +++ b/packages/aws/data_stream/s3_daily_storage/agent/stream/stream.yml.hbs @@ -24,9 +24,6 @@ shared_credential_file: {{shared_credential_file}} {{#if role_arn}} role_arn: {{role_arn}} {{/if}} -{{#if endpoint}} -endpoint: {{endpoint}} -{{/if}} {{#if default_region}} default_region: {{default_region}} {{/if}} diff --git a/packages/aws/data_stream/s3_request/agent/stream/stream.yml.hbs b/packages/aws/data_stream/s3_request/agent/stream/stream.yml.hbs index 9e83fe9ea92..3cf5faaac4d 100644 --- a/packages/aws/data_stream/s3_request/agent/stream/stream.yml.hbs +++ b/packages/aws/data_stream/s3_request/agent/stream/stream.yml.hbs @@ -24,9 +24,6 @@ shared_credential_file: {{shared_credential_file}} {{#if role_arn}} role_arn: {{role_arn}} {{/if}} -{{#if endpoint}} -endpoint: {{endpoint}} -{{/if}} {{#if default_region}} default_region: {{default_region}} {{/if}} diff --git a/packages/aws/data_stream/s3_storage_lens/agent/stream/stream.yml.hbs b/packages/aws/data_stream/s3_storage_lens/agent/stream/stream.yml.hbs index ca6eb412848..d547cceb8c2 100644 --- a/packages/aws/data_stream/s3_storage_lens/agent/stream/stream.yml.hbs +++ b/packages/aws/data_stream/s3_storage_lens/agent/stream/stream.yml.hbs @@ -24,9 +24,6 @@ shared_credential_file: {{shared_credential_file}} {{#if role_arn}} role_arn: {{role_arn}} {{/if}} -{{#if endpoint}} -endpoint: {{endpoint}} -{{/if}} {{#if default_region}} default_region: {{default_region}} {{/if}} diff --git a/packages/aws/data_stream/sns/agent/stream/stream.yml.hbs b/packages/aws/data_stream/sns/agent/stream/stream.yml.hbs index 5dac007eaab..82a4266c70f 100644 --- a/packages/aws/data_stream/sns/agent/stream/stream.yml.hbs +++ b/packages/aws/data_stream/sns/agent/stream/stream.yml.hbs @@ -24,9 +24,6 @@ shared_credential_file: {{shared_credential_file}} {{#if role_arn}} role_arn: {{role_arn}} {{/if}} -{{#if endpoint}} -endpoint: {{endpoint}} -{{/if}} {{#if default_region}} default_region: {{default_region}} {{/if}} diff --git a/packages/aws/data_stream/sqs/agent/stream/stream.yml.hbs b/packages/aws/data_stream/sqs/agent/stream/stream.yml.hbs index 6caeea86f8d..062e6f846aa 100644 --- a/packages/aws/data_stream/sqs/agent/stream/stream.yml.hbs +++ b/packages/aws/data_stream/sqs/agent/stream/stream.yml.hbs @@ -24,9 +24,6 @@ shared_credential_file: {{shared_credential_file}} {{#if role_arn}} role_arn: {{role_arn}} {{/if}} -{{#if endpoint}} -endpoint: {{endpoint}} -{{/if}} {{#if default_region}} default_region: {{default_region}} {{/if}} diff --git a/packages/aws/data_stream/transitgateway/agent/stream/stream.yml.hbs b/packages/aws/data_stream/transitgateway/agent/stream/stream.yml.hbs index bfa2ac9496a..92be4c7bdf8 100644 --- a/packages/aws/data_stream/transitgateway/agent/stream/stream.yml.hbs +++ b/packages/aws/data_stream/transitgateway/agent/stream/stream.yml.hbs @@ -24,9 +24,6 @@ shared_credential_file: {{shared_credential_file}} {{#if role_arn}} role_arn: {{role_arn}} {{/if}} -{{#if endpoint}} -endpoint: {{endpoint}} -{{/if}} {{#if default_region}} default_region: {{default_region}} {{/if}} diff --git a/packages/aws/data_stream/usage/agent/stream/stream.yml.hbs b/packages/aws/data_stream/usage/agent/stream/stream.yml.hbs index 303a58177ab..d20ef53b9c0 100644 --- a/packages/aws/data_stream/usage/agent/stream/stream.yml.hbs +++ b/packages/aws/data_stream/usage/agent/stream/stream.yml.hbs @@ -24,9 +24,6 @@ shared_credential_file: {{shared_credential_file}} {{#if role_arn}} role_arn: {{role_arn}} {{/if}} -{{#if endpoint}} -endpoint: {{endpoint}} -{{/if}} {{#if default_region}} default_region: {{default_region}} {{/if}} From d9ef5eb73092d2a76d615a91641467c71422c50b Mon Sep 17 00:00:00 2001 From: Kyle Pollich Date: Thu, 23 May 2024 11:36:23 -0400 Subject: [PATCH 05/14] fixup! Remove endpoint from non-SQS inputs --- .../aws/data_stream/waf/agent/stream/aws-cloudwatch.yml.hbs | 3 --- 1 file changed, 3 deletions(-) diff --git a/packages/aws/data_stream/waf/agent/stream/aws-cloudwatch.yml.hbs b/packages/aws/data_stream/waf/agent/stream/aws-cloudwatch.yml.hbs index 8cc06c88c22..934f97a44c3 100644 --- a/packages/aws/data_stream/waf/agent/stream/aws-cloudwatch.yml.hbs +++ b/packages/aws/data_stream/waf/agent/stream/aws-cloudwatch.yml.hbs @@ -66,9 +66,6 @@ shared_credential_file: {{shared_credential_file}} {{#if api_timeout}} api_timeout: {{api_timeout}} {{/if}} -{{#if endpoint}} -endpoint: {{endpoint}} -{{/if}} {{#if default_region}} default_region: {{default_region}} {{/if}} From 52e0d49a9eacaf81b70e06d57c1329cdce0ec128 Mon Sep 17 00:00:00 2001 From: Kyle Pollich Date: Thu, 23 May 2024 11:38:50 -0400 Subject: [PATCH 06/14] fixup! fixup! Remove endpoint from non-SQS inputs --- .../apigateway_logs/agent/stream/aws-cloudwatch.yml.hbs | 3 --- .../cloudtrail/agent/stream/aws-cloudwatch.yml.hbs | 3 --- .../ec2_logs/agent/stream/aws-cloudwatch.yml.hbs | 3 --- .../elb_logs/agent/stream/aws-cloudwatch.yml.hbs | 6 ------ .../emr_logs/agent/stream/aws-cloudwatch.yml.hbs | 3 --- .../firewall_logs/agent/stream/aws-cloudwatch.yml.hbs | 3 --- .../data_stream/vpcflow/agent/stream/aws-cloudwatch.yml.hbs | 3 --- 7 files changed, 24 deletions(-) diff --git a/packages/aws/data_stream/apigateway_logs/agent/stream/aws-cloudwatch.yml.hbs b/packages/aws/data_stream/apigateway_logs/agent/stream/aws-cloudwatch.yml.hbs index 8cc06c88c22..934f97a44c3 100644 --- a/packages/aws/data_stream/apigateway_logs/agent/stream/aws-cloudwatch.yml.hbs +++ b/packages/aws/data_stream/apigateway_logs/agent/stream/aws-cloudwatch.yml.hbs @@ -66,9 +66,6 @@ shared_credential_file: {{shared_credential_file}} {{#if api_timeout}} api_timeout: {{api_timeout}} {{/if}} -{{#if endpoint}} -endpoint: {{endpoint}} -{{/if}} {{#if default_region}} default_region: {{default_region}} {{/if}} diff --git a/packages/aws/data_stream/cloudtrail/agent/stream/aws-cloudwatch.yml.hbs b/packages/aws/data_stream/cloudtrail/agent/stream/aws-cloudwatch.yml.hbs index 8cc06c88c22..934f97a44c3 100644 --- a/packages/aws/data_stream/cloudtrail/agent/stream/aws-cloudwatch.yml.hbs +++ b/packages/aws/data_stream/cloudtrail/agent/stream/aws-cloudwatch.yml.hbs @@ -66,9 +66,6 @@ shared_credential_file: {{shared_credential_file}} {{#if api_timeout}} api_timeout: {{api_timeout}} {{/if}} -{{#if endpoint}} -endpoint: {{endpoint}} -{{/if}} {{#if default_region}} default_region: {{default_region}} {{/if}} diff --git a/packages/aws/data_stream/ec2_logs/agent/stream/aws-cloudwatch.yml.hbs b/packages/aws/data_stream/ec2_logs/agent/stream/aws-cloudwatch.yml.hbs index 8cc06c88c22..934f97a44c3 100644 --- a/packages/aws/data_stream/ec2_logs/agent/stream/aws-cloudwatch.yml.hbs +++ b/packages/aws/data_stream/ec2_logs/agent/stream/aws-cloudwatch.yml.hbs @@ -66,9 +66,6 @@ shared_credential_file: {{shared_credential_file}} {{#if api_timeout}} api_timeout: {{api_timeout}} {{/if}} -{{#if endpoint}} -endpoint: {{endpoint}} -{{/if}} {{#if default_region}} default_region: {{default_region}} {{/if}} diff --git a/packages/aws/data_stream/elb_logs/agent/stream/aws-cloudwatch.yml.hbs b/packages/aws/data_stream/elb_logs/agent/stream/aws-cloudwatch.yml.hbs index 59974851e5f..934f97a44c3 100644 --- a/packages/aws/data_stream/elb_logs/agent/stream/aws-cloudwatch.yml.hbs +++ b/packages/aws/data_stream/elb_logs/agent/stream/aws-cloudwatch.yml.hbs @@ -66,12 +66,6 @@ shared_credential_file: {{shared_credential_file}} {{#if api_timeout}} api_timeout: {{api_timeout}} {{/if}} -{{#if endpoint}} -endpoint: {{endpoint}} -{{/if}} -{{#if endpoint}} -endpoint: {{endpoint}} -{{/if}} {{#if default_region}} default_region: {{default_region}} {{/if}} diff --git a/packages/aws/data_stream/emr_logs/agent/stream/aws-cloudwatch.yml.hbs b/packages/aws/data_stream/emr_logs/agent/stream/aws-cloudwatch.yml.hbs index 8cc06c88c22..934f97a44c3 100644 --- a/packages/aws/data_stream/emr_logs/agent/stream/aws-cloudwatch.yml.hbs +++ b/packages/aws/data_stream/emr_logs/agent/stream/aws-cloudwatch.yml.hbs @@ -66,9 +66,6 @@ shared_credential_file: {{shared_credential_file}} {{#if api_timeout}} api_timeout: {{api_timeout}} {{/if}} -{{#if endpoint}} -endpoint: {{endpoint}} -{{/if}} {{#if default_region}} default_region: {{default_region}} {{/if}} diff --git a/packages/aws/data_stream/firewall_logs/agent/stream/aws-cloudwatch.yml.hbs b/packages/aws/data_stream/firewall_logs/agent/stream/aws-cloudwatch.yml.hbs index 8cc06c88c22..934f97a44c3 100644 --- a/packages/aws/data_stream/firewall_logs/agent/stream/aws-cloudwatch.yml.hbs +++ b/packages/aws/data_stream/firewall_logs/agent/stream/aws-cloudwatch.yml.hbs @@ -66,9 +66,6 @@ shared_credential_file: {{shared_credential_file}} {{#if api_timeout}} api_timeout: {{api_timeout}} {{/if}} -{{#if endpoint}} -endpoint: {{endpoint}} -{{/if}} {{#if default_region}} default_region: {{default_region}} {{/if}} diff --git a/packages/aws/data_stream/vpcflow/agent/stream/aws-cloudwatch.yml.hbs b/packages/aws/data_stream/vpcflow/agent/stream/aws-cloudwatch.yml.hbs index 8cc06c88c22..934f97a44c3 100644 --- a/packages/aws/data_stream/vpcflow/agent/stream/aws-cloudwatch.yml.hbs +++ b/packages/aws/data_stream/vpcflow/agent/stream/aws-cloudwatch.yml.hbs @@ -66,9 +66,6 @@ shared_credential_file: {{shared_credential_file}} {{#if api_timeout}} api_timeout: {{api_timeout}} {{/if}} -{{#if endpoint}} -endpoint: {{endpoint}} -{{/if}} {{#if default_region}} default_region: {{default_region}} {{/if}} From 572297bc657968283f0569e86a5ca761800b4488 Mon Sep 17 00:00:00 2001 From: Kyle Pollich Date: Thu, 23 May 2024 15:40:34 -0400 Subject: [PATCH 07/14] (wip) Add custom_configurations variable --- packages/aws/data_stream/apigateway_logs/manifest.yml | 7 +++++++ packages/aws/data_stream/cloudfront_logs/manifest.yml | 7 +++++++ packages/aws/data_stream/cloudtrail/manifest.yml | 7 +++++++ .../aws/data_stream/ec2_logs/agent/stream/aws-s3.yml.hbs | 3 +++ packages/aws/data_stream/ec2_logs/manifest.yml | 7 +++++++ packages/aws/data_stream/elb_logs/manifest.yml | 7 +++++++ packages/aws/manifest.yml | 7 +++++++ 7 files changed, 45 insertions(+) diff --git a/packages/aws/data_stream/apigateway_logs/manifest.yml b/packages/aws/data_stream/apigateway_logs/manifest.yml index 964761ce462..2ace9ff8a92 100644 --- a/packages/aws/data_stream/apigateway_logs/manifest.yml +++ b/packages/aws/data_stream/apigateway_logs/manifest.yml @@ -123,6 +123,13 @@ streams: type: bool multi: false default: false + - name: custom_configurations + type: yaml + title: Custom configurations + multi: false + required: false + show_user: false + description: Custom settings for the underlying inputs - input: aws-cloudwatch template_path: aws-cloudwatch.yml.hbs title: AWS API Gateway Logs via CloudWatch diff --git a/packages/aws/data_stream/cloudfront_logs/manifest.yml b/packages/aws/data_stream/cloudfront_logs/manifest.yml index 349d530f418..4c697ae4b0f 100644 --- a/packages/aws/data_stream/cloudfront_logs/manifest.yml +++ b/packages/aws/data_stream/cloudfront_logs/manifest.yml @@ -89,3 +89,10 @@ streams: default: 5 required: false show_user: false + - name: custom_configurations + type: yaml + title: Custom configurations + multi: false + required: false + show_user: false + description: Custom settings for the underlying inputs diff --git a/packages/aws/data_stream/cloudtrail/manifest.yml b/packages/aws/data_stream/cloudtrail/manifest.yml index 21790c2fded..690bec36ca1 100644 --- a/packages/aws/data_stream/cloudtrail/manifest.yml +++ b/packages/aws/data_stream/cloudtrail/manifest.yml @@ -148,6 +148,13 @@ streams: default: 5 required: false show_user: false + - name: custom_configurations + type: yaml + title: Custom configurations + multi: false + required: false + show_user: false + description: Custom settings for the underlying inputs - input: httpjson title: AWS CloudTrail Logs via Splunk Enterprise REST API description: Collect AWS CloudTrail logs via Splunk Enterprise REST API diff --git a/packages/aws/data_stream/ec2_logs/agent/stream/aws-s3.yml.hbs b/packages/aws/data_stream/ec2_logs/agent/stream/aws-s3.yml.hbs index 152e77a7665..81a1ccf7e71 100644 --- a/packages/aws/data_stream/ec2_logs/agent/stream/aws-s3.yml.hbs +++ b/packages/aws/data_stream/ec2_logs/agent/stream/aws-s3.yml.hbs @@ -55,3 +55,6 @@ publisher_pipeline.disable_host: true processors: {{processors}} {{/if}} +{{#if custom_configurations}} +{{custom_configurations}} +{{/if}} diff --git a/packages/aws/data_stream/ec2_logs/manifest.yml b/packages/aws/data_stream/ec2_logs/manifest.yml index a0c2580a5e1..fae95a32852 100644 --- a/packages/aws/data_stream/ec2_logs/manifest.yml +++ b/packages/aws/data_stream/ec2_logs/manifest.yml @@ -85,6 +85,13 @@ streams: default: 5 required: false show_user: false + - name: custom_configurations + type: yaml + title: Custom configurations + multi: false + required: false + show_user: false + description: Custom settings for the underlying inputs - input: aws-cloudwatch template_path: aws-cloudwatch.yml.hbs title: AWS EC2 Logs via CloudWatch diff --git a/packages/aws/data_stream/elb_logs/manifest.yml b/packages/aws/data_stream/elb_logs/manifest.yml index cc2842b3829..3e5759265af 100644 --- a/packages/aws/data_stream/elb_logs/manifest.yml +++ b/packages/aws/data_stream/elb_logs/manifest.yml @@ -84,6 +84,13 @@ streams: default: 5 required: false show_user: false + - name: custom_configurations + type: yaml + title: Custom configurations + multi: false + required: false + show_user: false + description: Custom settings for the underlying inputs - input: aws-cloudwatch template_path: aws-cloudwatch.yml.hbs title: AWS ELB Logs via CloudWatch diff --git a/packages/aws/manifest.yml b/packages/aws/manifest.yml index 50d2e1c0e50..0b7977cd915 100644 --- a/packages/aws/manifest.yml +++ b/packages/aws/manifest.yml @@ -77,6 +77,13 @@ vars: required: false show_user: false description: URL to proxy connections in the form of http\[s\]://:@: + - name: custom_configurations + type: yaml + title: Custom Configurations + multi: false + required: false + show_user: false + description: Custom settings for the underlying inputs policy_templates: - name: billing title: AWS Billing From 8e7687e5c6ec01083289d6496e2008ba664042f5 Mon Sep 17 00:00:00 2001 From: Kyle Pollich Date: Tue, 28 May 2024 13:25:49 -0400 Subject: [PATCH 08/14] Add custom configurations to S3 inputs --- .../apigateway_logs/agent/stream/aws-s3.yml.hbs | 3 +++ packages/aws/data_stream/apigateway_logs/manifest.yml | 5 +++-- .../cloudfront_logs/agent/stream/aws-s3.yml.hbs | 3 +++ packages/aws/data_stream/cloudfront_logs/manifest.yml | 5 +++-- .../data_stream/cloudtrail/agent/stream/aws-s3.yml.hbs | 3 +++ packages/aws/data_stream/cloudtrail/manifest.yml | 8 ++++++++ .../ec2_logs/agent/stream/aws-cloudwatch.yml.hbs | 3 +++ packages/aws/data_stream/ec2_logs/manifest.yml | 8 ++++++++ .../aws/data_stream/elb_logs/agent/stream/aws-s3.yml.hbs | 3 +++ packages/aws/data_stream/elb_logs/manifest.yml | 5 +++-- .../aws/data_stream/emr_logs/agent/stream/aws-s3.yml.hbs | 3 +++ packages/aws/data_stream/emr_logs/manifest.yml | 8 ++++++++ .../data_stream/firewall_logs/agent/stream/aws-s3.yml.hbs | 3 +++ packages/aws/data_stream/firewall_logs/manifest.yml | 8 ++++++++ .../aws/data_stream/guardduty/agent/stream/aws-s3.yml.hbs | 3 +++ packages/aws/data_stream/guardduty/manifest.yml | 8 ++++++++ .../route53_resolver_logs/agent/stream/aws-s3.yml.hbs | 3 +++ .../aws/data_stream/route53_resolver_logs/manifest.yml | 8 ++++++++ .../aws/data_stream/s3access/agent/stream/aws-s3.yml.hbs | 3 +++ packages/aws/data_stream/s3access/manifest.yml | 8 ++++++++ .../aws/data_stream/vpcflow/agent/stream/aws-s3.yml.hbs | 3 +++ packages/aws/data_stream/vpcflow/manifest.yml | 8 ++++++++ packages/aws/data_stream/vpn/agent/stream/stream.yml.hbs | 3 --- packages/aws/data_stream/waf/agent/stream/aws-s3.yml.hbs | 3 +++ packages/aws/data_stream/waf/manifest.yml | 8 ++++++++ 25 files changed, 117 insertions(+), 9 deletions(-) diff --git a/packages/aws/data_stream/apigateway_logs/agent/stream/aws-s3.yml.hbs b/packages/aws/data_stream/apigateway_logs/agent/stream/aws-s3.yml.hbs index 0d19877d02d..e556e48e608 100644 --- a/packages/aws/data_stream/apigateway_logs/agent/stream/aws-s3.yml.hbs +++ b/packages/aws/data_stream/apigateway_logs/agent/stream/aws-s3.yml.hbs @@ -76,3 +76,6 @@ publisher_pipeline.disable_host: true processors: {{processors}} {{/if}} +{{#if custom}} +{{custom}} +{{/if}} diff --git a/packages/aws/data_stream/apigateway_logs/manifest.yml b/packages/aws/data_stream/apigateway_logs/manifest.yml index 2ace9ff8a92..16b69e383db 100644 --- a/packages/aws/data_stream/apigateway_logs/manifest.yml +++ b/packages/aws/data_stream/apigateway_logs/manifest.yml @@ -123,13 +123,14 @@ streams: type: bool multi: false default: false - - name: custom_configurations + - name: custom type: yaml title: Custom configurations multi: false required: false show_user: false - description: Custom settings for the underlying inputs + description: > + Additional settings to be added to the configuration. Be careful using this as it might break the input as those settings are not validated and can override the settings specified above. See [`aws-s3` input settings docs](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-aws-s3.html) for details. - input: aws-cloudwatch template_path: aws-cloudwatch.yml.hbs title: AWS API Gateway Logs via CloudWatch diff --git a/packages/aws/data_stream/cloudfront_logs/agent/stream/aws-s3.yml.hbs b/packages/aws/data_stream/cloudfront_logs/agent/stream/aws-s3.yml.hbs index 4d4388d441b..acb64f7b40a 100644 --- a/packages/aws/data_stream/cloudfront_logs/agent/stream/aws-s3.yml.hbs +++ b/packages/aws/data_stream/cloudfront_logs/agent/stream/aws-s3.yml.hbs @@ -59,3 +59,6 @@ processors: {{#if processors}} {{processors}} {{/if}} +{{#if custom}} +{{custom}} +{{/if}} diff --git a/packages/aws/data_stream/cloudfront_logs/manifest.yml b/packages/aws/data_stream/cloudfront_logs/manifest.yml index 4c697ae4b0f..6fdc5435879 100644 --- a/packages/aws/data_stream/cloudfront_logs/manifest.yml +++ b/packages/aws/data_stream/cloudfront_logs/manifest.yml @@ -89,10 +89,11 @@ streams: default: 5 required: false show_user: false - - name: custom_configurations + - name: custom type: yaml title: Custom configurations multi: false required: false show_user: false - description: Custom settings for the underlying inputs + description: > + Additional settings to be added to the configuration. Be careful using this as it might break the input as those settings are not validated and can override the settings specified above. See [`aws-s3` input settings docs](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-aws-s3.html) for details. diff --git a/packages/aws/data_stream/cloudtrail/agent/stream/aws-s3.yml.hbs b/packages/aws/data_stream/cloudtrail/agent/stream/aws-s3.yml.hbs index 6d36e8cee47..c45ab6f37e9 100644 --- a/packages/aws/data_stream/cloudtrail/agent/stream/aws-s3.yml.hbs +++ b/packages/aws/data_stream/cloudtrail/agent/stream/aws-s3.yml.hbs @@ -91,3 +91,6 @@ publisher_pipeline.disable_host: true processors: {{processors}} {{/if}} +{{#if custom}} +{{custom}} +{{/if}} diff --git a/packages/aws/data_stream/cloudtrail/manifest.yml b/packages/aws/data_stream/cloudtrail/manifest.yml index 690bec36ca1..d400eab5b68 100644 --- a/packages/aws/data_stream/cloudtrail/manifest.yml +++ b/packages/aws/data_stream/cloudtrail/manifest.yml @@ -382,3 +382,11 @@ streams: type: bool multi: false default: false + - name: custom + type: yaml + title: Custom configurations + multi: false + required: false + show_user: false + description: > + Additional settings to be added to the configuration. Be careful using this as it might break the input as those settings are not validated and can override the settings specified above. See [`aws-s3` input settings docs](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-aws-s3.html) for details. diff --git a/packages/aws/data_stream/ec2_logs/agent/stream/aws-cloudwatch.yml.hbs b/packages/aws/data_stream/ec2_logs/agent/stream/aws-cloudwatch.yml.hbs index 934f97a44c3..209b9de6801 100644 --- a/packages/aws/data_stream/ec2_logs/agent/stream/aws-cloudwatch.yml.hbs +++ b/packages/aws/data_stream/ec2_logs/agent/stream/aws-cloudwatch.yml.hbs @@ -98,3 +98,6 @@ publisher_pipeline.disable_host: true processors: {{processors}} {{/if}} +{{#if custom}} +{{custom}} +{{/if}} diff --git a/packages/aws/data_stream/ec2_logs/manifest.yml b/packages/aws/data_stream/ec2_logs/manifest.yml index fae95a32852..15da94a3f96 100644 --- a/packages/aws/data_stream/ec2_logs/manifest.yml +++ b/packages/aws/data_stream/ec2_logs/manifest.yml @@ -210,6 +210,14 @@ streams: type: bool multi: false default: false + - name: custom + type: yaml + title: Custom configurations + multi: false + required: false + show_user: false + description: > + Additional settings to be added to the configuration. Be careful using this as it might break the input as those settings are not validated and can override the settings specified above. See [`aws-s3` input settings docs](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-aws-s3.html) for details. # Ensures agents have permissions to write data to `logs-*-*` elasticsearch: dynamic_dataset: true diff --git a/packages/aws/data_stream/elb_logs/agent/stream/aws-s3.yml.hbs b/packages/aws/data_stream/elb_logs/agent/stream/aws-s3.yml.hbs index 152e77a7665..153def8018a 100644 --- a/packages/aws/data_stream/elb_logs/agent/stream/aws-s3.yml.hbs +++ b/packages/aws/data_stream/elb_logs/agent/stream/aws-s3.yml.hbs @@ -55,3 +55,6 @@ publisher_pipeline.disable_host: true processors: {{processors}} {{/if}} +{{#if custom}} +{{custom}} +{{/if}} diff --git a/packages/aws/data_stream/elb_logs/manifest.yml b/packages/aws/data_stream/elb_logs/manifest.yml index 3e5759265af..00b9a300e58 100644 --- a/packages/aws/data_stream/elb_logs/manifest.yml +++ b/packages/aws/data_stream/elb_logs/manifest.yml @@ -84,13 +84,14 @@ streams: default: 5 required: false show_user: false - - name: custom_configurations + - name: custom type: yaml title: Custom configurations multi: false required: false show_user: false - description: Custom settings for the underlying inputs + description: > + Additional settings to be added to the configuration. Be careful using this as it might break the input as those settings are not validated and can override the settings specified above. See [`aws-s3` input settings docs](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-aws-s3.html) for details. - input: aws-cloudwatch template_path: aws-cloudwatch.yml.hbs title: AWS ELB Logs via CloudWatch diff --git a/packages/aws/data_stream/emr_logs/agent/stream/aws-s3.yml.hbs b/packages/aws/data_stream/emr_logs/agent/stream/aws-s3.yml.hbs index 2f110dad920..ead858d32e0 100644 --- a/packages/aws/data_stream/emr_logs/agent/stream/aws-s3.yml.hbs +++ b/packages/aws/data_stream/emr_logs/agent/stream/aws-s3.yml.hbs @@ -81,3 +81,6 @@ publisher_pipeline.disable_host: true processors: {{processors}} {{/if}} +{{#if custom}} +{{custom}} +{{/if}} diff --git a/packages/aws/data_stream/emr_logs/manifest.yml b/packages/aws/data_stream/emr_logs/manifest.yml index 684e7f4b227..0dcb8dd5032 100644 --- a/packages/aws/data_stream/emr_logs/manifest.yml +++ b/packages/aws/data_stream/emr_logs/manifest.yml @@ -124,6 +124,14 @@ streams: type: bool multi: false default: false + - name: custom + type: yaml + title: Custom configurations + multi: false + required: false + show_user: false + description: > + Additional settings to be added to the configuration. Be careful using this as it might break the input as those settings are not validated and can override the settings specified above. See [`aws-s3` input settings docs](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-aws-s3.html) for details. - input: aws-cloudwatch template_path: aws-cloudwatch.yml.hbs title: AWS EMR Logs via CloudWatch diff --git a/packages/aws/data_stream/firewall_logs/agent/stream/aws-s3.yml.hbs b/packages/aws/data_stream/firewall_logs/agent/stream/aws-s3.yml.hbs index b3c81c9f38d..077994e4d79 100644 --- a/packages/aws/data_stream/firewall_logs/agent/stream/aws-s3.yml.hbs +++ b/packages/aws/data_stream/firewall_logs/agent/stream/aws-s3.yml.hbs @@ -56,3 +56,6 @@ publisher_pipeline.disable_host: true processors: {{processors}} {{/if}} +{{#if custom}} +{{custom}} +{{/if}} diff --git a/packages/aws/data_stream/firewall_logs/manifest.yml b/packages/aws/data_stream/firewall_logs/manifest.yml index c7a274a5068..2ec4bf7b422 100644 --- a/packages/aws/data_stream/firewall_logs/manifest.yml +++ b/packages/aws/data_stream/firewall_logs/manifest.yml @@ -84,6 +84,14 @@ streams: default: 5 required: false show_user: false + - name: custom + type: yaml + title: Custom configurations + multi: false + required: false + show_user: false + description: > + Additional settings to be added to the configuration. Be careful using this as it might break the input as those settings are not validated and can override the settings specified above. See [`aws-s3` input settings docs](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-aws-s3.html) for details. - input: aws-cloudwatch template_path: aws-cloudwatch.yml.hbs title: AWS ELB Logs via CloudWatch diff --git a/packages/aws/data_stream/guardduty/agent/stream/aws-s3.yml.hbs b/packages/aws/data_stream/guardduty/agent/stream/aws-s3.yml.hbs index 6c621c8aac5..9a8da946871 100644 --- a/packages/aws/data_stream/guardduty/agent/stream/aws-s3.yml.hbs +++ b/packages/aws/data_stream/guardduty/agent/stream/aws-s3.yml.hbs @@ -86,3 +86,6 @@ publisher_pipeline.disable_host: true processors: {{processors}} {{/if}} +{{#if custom}} +{{custom}} +{{/if}} diff --git a/packages/aws/data_stream/guardduty/manifest.yml b/packages/aws/data_stream/guardduty/manifest.yml index b8e1485f9cb..a67fca063ae 100644 --- a/packages/aws/data_stream/guardduty/manifest.yml +++ b/packages/aws/data_stream/guardduty/manifest.yml @@ -254,3 +254,11 @@ streams: show_user: false description: >- Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details. + - name: custom + type: yaml + title: Custom configurations + multi: false + required: false + show_user: false + description: > + Additional settings to be added to the configuration. Be careful using this as it might break the input as those settings are not validated and can override the settings specified above. See [`aws-s3` input settings docs](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-aws-s3.html) for details. diff --git a/packages/aws/data_stream/route53_resolver_logs/agent/stream/aws-s3.yml.hbs b/packages/aws/data_stream/route53_resolver_logs/agent/stream/aws-s3.yml.hbs index b3c81c9f38d..077994e4d79 100644 --- a/packages/aws/data_stream/route53_resolver_logs/agent/stream/aws-s3.yml.hbs +++ b/packages/aws/data_stream/route53_resolver_logs/agent/stream/aws-s3.yml.hbs @@ -56,3 +56,6 @@ publisher_pipeline.disable_host: true processors: {{processors}} {{/if}} +{{#if custom}} +{{custom}} +{{/if}} diff --git a/packages/aws/data_stream/route53_resolver_logs/manifest.yml b/packages/aws/data_stream/route53_resolver_logs/manifest.yml index 140936cf050..58887a5d8ac 100644 --- a/packages/aws/data_stream/route53_resolver_logs/manifest.yml +++ b/packages/aws/data_stream/route53_resolver_logs/manifest.yml @@ -200,3 +200,11 @@ streams: default: 5 required: false show_user: false + - name: custom + type: yaml + title: Custom configurations + multi: false + required: false + show_user: false + description: > + Additional settings to be added to the configuration. Be careful using this as it might break the input as those settings are not validated and can override the settings specified above. See [`aws-s3` input settings docs](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-aws-s3.html) for details. diff --git a/packages/aws/data_stream/s3access/agent/stream/aws-s3.yml.hbs b/packages/aws/data_stream/s3access/agent/stream/aws-s3.yml.hbs index 152e77a7665..153def8018a 100644 --- a/packages/aws/data_stream/s3access/agent/stream/aws-s3.yml.hbs +++ b/packages/aws/data_stream/s3access/agent/stream/aws-s3.yml.hbs @@ -55,3 +55,6 @@ publisher_pipeline.disable_host: true processors: {{processors}} {{/if}} +{{#if custom}} +{{custom}} +{{/if}} diff --git a/packages/aws/data_stream/s3access/manifest.yml b/packages/aws/data_stream/s3access/manifest.yml index 08853d6f0fb..efa16e39116 100644 --- a/packages/aws/data_stream/s3access/manifest.yml +++ b/packages/aws/data_stream/s3access/manifest.yml @@ -84,3 +84,11 @@ streams: default: 5 required: false show_user: false + - name: custom + type: yaml + title: Custom configurations + multi: false + required: false + show_user: false + description: > + Additional settings to be added to the configuration. Be careful using this as it might break the input as those settings are not validated and can override the settings specified above. See [`aws-s3` input settings docs](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-aws-s3.html) for details. diff --git a/packages/aws/data_stream/vpcflow/agent/stream/aws-s3.yml.hbs b/packages/aws/data_stream/vpcflow/agent/stream/aws-s3.yml.hbs index 272d9011769..b5909cdcc51 100644 --- a/packages/aws/data_stream/vpcflow/agent/stream/aws-s3.yml.hbs +++ b/packages/aws/data_stream/vpcflow/agent/stream/aws-s3.yml.hbs @@ -77,3 +77,6 @@ publisher_pipeline.disable_host: true processors: {{processors}} {{/if}} +{{#if custom}} +{{custom}} +{{/if}} diff --git a/packages/aws/data_stream/vpcflow/manifest.yml b/packages/aws/data_stream/vpcflow/manifest.yml index 0f900819dd1..9338f76f316 100644 --- a/packages/aws/data_stream/vpcflow/manifest.yml +++ b/packages/aws/data_stream/vpcflow/manifest.yml @@ -123,6 +123,14 @@ streams: type: bool multi: false default: false + - name: custom + type: yaml + title: Custom configurations + multi: false + required: false + show_user: false + description: > + Additional settings to be added to the configuration. Be careful using this as it might break the input as those settings are not validated and can override the settings specified above. See [`aws-s3` input settings docs](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-aws-s3.html) for details. - input: aws-cloudwatch template_path: aws-cloudwatch.yml.hbs title: AWS VPC Flow Logs via CloudWatch diff --git a/packages/aws/data_stream/vpn/agent/stream/stream.yml.hbs b/packages/aws/data_stream/vpn/agent/stream/stream.yml.hbs index a41724ba223..017bbefd6b8 100644 --- a/packages/aws/data_stream/vpn/agent/stream/stream.yml.hbs +++ b/packages/aws/data_stream/vpn/agent/stream/stream.yml.hbs @@ -24,9 +24,6 @@ shared_credential_file: {{shared_credential_file}} {{#if role_arn}} role_arn: {{role_arn}} {{/if}} -{{#if endpoint}} -endpoint: {{endpoint}} -{{/if}} {{#if default_region}} default_region: {{default_region}} {{/if}} diff --git a/packages/aws/data_stream/waf/agent/stream/aws-s3.yml.hbs b/packages/aws/data_stream/waf/agent/stream/aws-s3.yml.hbs index 54b0042b770..fd68da1ff68 100644 --- a/packages/aws/data_stream/waf/agent/stream/aws-s3.yml.hbs +++ b/packages/aws/data_stream/waf/agent/stream/aws-s3.yml.hbs @@ -56,3 +56,6 @@ publisher_pipeline.disable_host: true processors: {{processors}} {{/if}} +{{#if custom}} +{{custom}} +{{/if}} diff --git a/packages/aws/data_stream/waf/manifest.yml b/packages/aws/data_stream/waf/manifest.yml index eff771d14c9..1071d6408fc 100644 --- a/packages/aws/data_stream/waf/manifest.yml +++ b/packages/aws/data_stream/waf/manifest.yml @@ -84,6 +84,14 @@ streams: default: 5 required: false show_user: false + - name: custom + type: yaml + title: Custom configurations + multi: false + required: false + show_user: false + description: > + Additional settings to be added to the configuration. Be careful using this as it might break the input as those settings are not validated and can override the settings specified above. See [`aws-s3` input settings docs](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-aws-s3.html) for details. - input: aws-cloudwatch template_path: aws-cloudwatch.yml.hbs title: AWS WAF logs via CloudWatch From 31f1d68f8cc7a9dfa5301c19075767a94fbbba3e Mon Sep 17 00:00:00 2001 From: Kyle Pollich Date: Tue, 28 May 2024 13:27:03 -0400 Subject: [PATCH 09/14] Remove endpoint from ecs_metrics --- .../aws/data_stream/ecs_metrics/agent/stream/stream.yml.hbs | 3 --- 1 file changed, 3 deletions(-) diff --git a/packages/aws/data_stream/ecs_metrics/agent/stream/stream.yml.hbs b/packages/aws/data_stream/ecs_metrics/agent/stream/stream.yml.hbs index 16607804cec..d8b2cf6475f 100644 --- a/packages/aws/data_stream/ecs_metrics/agent/stream/stream.yml.hbs +++ b/packages/aws/data_stream/ecs_metrics/agent/stream/stream.yml.hbs @@ -24,9 +24,6 @@ shared_credential_file: {{shared_credential_file}} {{#if role_arn}} role_arn: {{role_arn}} {{/if}} -{{#if endpoint}} -endpoint: {{endpoint}} -{{/if}} {{#if default_region}} default_region: {{default_region}} {{/if}} From fbd458b83bb54ef30092a76fea8499b925c44f25 Mon Sep 17 00:00:00 2001 From: Kyle Pollich Date: Tue, 28 May 2024 13:30:23 -0400 Subject: [PATCH 10/14] Remove top-level integration custom settings for now --- packages/aws/manifest.yml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/packages/aws/manifest.yml b/packages/aws/manifest.yml index 0b7977cd915..50d2e1c0e50 100644 --- a/packages/aws/manifest.yml +++ b/packages/aws/manifest.yml @@ -77,13 +77,6 @@ vars: required: false show_user: false description: URL to proxy connections in the form of http\[s\]://:@: - - name: custom_configurations - type: yaml - title: Custom Configurations - multi: false - required: false - show_user: false - description: Custom settings for the underlying inputs policy_templates: - name: billing title: AWS Billing From b30738d80f1463212cc57aef1c5e04a2859469aa Mon Sep 17 00:00:00 2001 From: Kyle Pollich Date: Tue, 28 May 2024 13:55:03 -0400 Subject: [PATCH 11/14] Fix ec2 logs --- .../ec2_logs/agent/stream/aws-cloudwatch.yml.hbs | 3 --- .../ec2_logs/agent/stream/aws-s3.yml.hbs | 4 ++-- packages/aws/data_stream/ec2_logs/manifest.yml | 13 +++---------- 3 files changed, 5 insertions(+), 15 deletions(-) diff --git a/packages/aws/data_stream/ec2_logs/agent/stream/aws-cloudwatch.yml.hbs b/packages/aws/data_stream/ec2_logs/agent/stream/aws-cloudwatch.yml.hbs index 209b9de6801..934f97a44c3 100644 --- a/packages/aws/data_stream/ec2_logs/agent/stream/aws-cloudwatch.yml.hbs +++ b/packages/aws/data_stream/ec2_logs/agent/stream/aws-cloudwatch.yml.hbs @@ -98,6 +98,3 @@ publisher_pipeline.disable_host: true processors: {{processors}} {{/if}} -{{#if custom}} -{{custom}} -{{/if}} diff --git a/packages/aws/data_stream/ec2_logs/agent/stream/aws-s3.yml.hbs b/packages/aws/data_stream/ec2_logs/agent/stream/aws-s3.yml.hbs index 81a1ccf7e71..153def8018a 100644 --- a/packages/aws/data_stream/ec2_logs/agent/stream/aws-s3.yml.hbs +++ b/packages/aws/data_stream/ec2_logs/agent/stream/aws-s3.yml.hbs @@ -55,6 +55,6 @@ publisher_pipeline.disable_host: true processors: {{processors}} {{/if}} -{{#if custom_configurations}} -{{custom_configurations}} +{{#if custom}} +{{custom}} {{/if}} diff --git a/packages/aws/data_stream/ec2_logs/manifest.yml b/packages/aws/data_stream/ec2_logs/manifest.yml index 15da94a3f96..2b5f8f061cf 100644 --- a/packages/aws/data_stream/ec2_logs/manifest.yml +++ b/packages/aws/data_stream/ec2_logs/manifest.yml @@ -85,13 +85,14 @@ streams: default: 5 required: false show_user: false - - name: custom_configurations + - name: custom type: yaml title: Custom configurations multi: false required: false show_user: false - description: Custom settings for the underlying inputs + description: > + Additional settings to be added to the configuration. Be careful using this as it might break the input as those settings are not validated and can override the settings specified above. See [`aws-s3` input settings docs](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-aws-s3.html) for details. - input: aws-cloudwatch template_path: aws-cloudwatch.yml.hbs title: AWS EC2 Logs via CloudWatch @@ -210,14 +211,6 @@ streams: type: bool multi: false default: false - - name: custom - type: yaml - title: Custom configurations - multi: false - required: false - show_user: false - description: > - Additional settings to be added to the configuration. Be careful using this as it might break the input as those settings are not validated and can override the settings specified above. See [`aws-s3` input settings docs](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-aws-s3.html) for details. # Ensures agents have permissions to write data to `logs-*-*` elasticsearch: dynamic_dataset: true From 9b4314a7cd3b253c4380eb98f9e96ea74569d26d Mon Sep 17 00:00:00 2001 From: Kyle Pollich Date: Tue, 28 May 2024 13:55:44 -0400 Subject: [PATCH 12/14] Run elastic-package check --- packages/aws/data_stream/apigateway_logs/manifest.yml | 1 + packages/aws/data_stream/cloudfront_logs/manifest.yml | 1 + packages/aws/data_stream/cloudtrail/manifest.yml | 1 + packages/aws/data_stream/ec2_logs/manifest.yml | 1 + packages/aws/data_stream/elb_logs/manifest.yml | 1 + packages/aws/data_stream/emr_logs/manifest.yml | 1 + packages/aws/data_stream/firewall_logs/manifest.yml | 1 + packages/aws/data_stream/guardduty/manifest.yml | 1 + packages/aws/data_stream/route53_resolver_logs/manifest.yml | 1 + packages/aws/data_stream/s3access/manifest.yml | 1 + packages/aws/data_stream/vpcflow/manifest.yml | 1 + packages/aws/data_stream/waf/manifest.yml | 1 + 12 files changed, 12 insertions(+) diff --git a/packages/aws/data_stream/apigateway_logs/manifest.yml b/packages/aws/data_stream/apigateway_logs/manifest.yml index 16b69e383db..33d87aaefeb 100644 --- a/packages/aws/data_stream/apigateway_logs/manifest.yml +++ b/packages/aws/data_stream/apigateway_logs/manifest.yml @@ -131,6 +131,7 @@ streams: show_user: false description: > Additional settings to be added to the configuration. Be careful using this as it might break the input as those settings are not validated and can override the settings specified above. See [`aws-s3` input settings docs](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-aws-s3.html) for details. + - input: aws-cloudwatch template_path: aws-cloudwatch.yml.hbs title: AWS API Gateway Logs via CloudWatch diff --git a/packages/aws/data_stream/cloudfront_logs/manifest.yml b/packages/aws/data_stream/cloudfront_logs/manifest.yml index 6fdc5435879..00148471b8d 100644 --- a/packages/aws/data_stream/cloudfront_logs/manifest.yml +++ b/packages/aws/data_stream/cloudfront_logs/manifest.yml @@ -97,3 +97,4 @@ streams: show_user: false description: > Additional settings to be added to the configuration. Be careful using this as it might break the input as those settings are not validated and can override the settings specified above. See [`aws-s3` input settings docs](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-aws-s3.html) for details. + diff --git a/packages/aws/data_stream/cloudtrail/manifest.yml b/packages/aws/data_stream/cloudtrail/manifest.yml index d400eab5b68..bb72e908328 100644 --- a/packages/aws/data_stream/cloudtrail/manifest.yml +++ b/packages/aws/data_stream/cloudtrail/manifest.yml @@ -390,3 +390,4 @@ streams: show_user: false description: > Additional settings to be added to the configuration. Be careful using this as it might break the input as those settings are not validated and can override the settings specified above. See [`aws-s3` input settings docs](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-aws-s3.html) for details. + diff --git a/packages/aws/data_stream/ec2_logs/manifest.yml b/packages/aws/data_stream/ec2_logs/manifest.yml index 2b5f8f061cf..8e52a9d9a64 100644 --- a/packages/aws/data_stream/ec2_logs/manifest.yml +++ b/packages/aws/data_stream/ec2_logs/manifest.yml @@ -93,6 +93,7 @@ streams: show_user: false description: > Additional settings to be added to the configuration. Be careful using this as it might break the input as those settings are not validated and can override the settings specified above. See [`aws-s3` input settings docs](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-aws-s3.html) for details. + - input: aws-cloudwatch template_path: aws-cloudwatch.yml.hbs title: AWS EC2 Logs via CloudWatch diff --git a/packages/aws/data_stream/elb_logs/manifest.yml b/packages/aws/data_stream/elb_logs/manifest.yml index 00b9a300e58..72c842f3092 100644 --- a/packages/aws/data_stream/elb_logs/manifest.yml +++ b/packages/aws/data_stream/elb_logs/manifest.yml @@ -92,6 +92,7 @@ streams: show_user: false description: > Additional settings to be added to the configuration. Be careful using this as it might break the input as those settings are not validated and can override the settings specified above. See [`aws-s3` input settings docs](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-aws-s3.html) for details. + - input: aws-cloudwatch template_path: aws-cloudwatch.yml.hbs title: AWS ELB Logs via CloudWatch diff --git a/packages/aws/data_stream/emr_logs/manifest.yml b/packages/aws/data_stream/emr_logs/manifest.yml index 0dcb8dd5032..ef2b598e807 100644 --- a/packages/aws/data_stream/emr_logs/manifest.yml +++ b/packages/aws/data_stream/emr_logs/manifest.yml @@ -132,6 +132,7 @@ streams: show_user: false description: > Additional settings to be added to the configuration. Be careful using this as it might break the input as those settings are not validated and can override the settings specified above. See [`aws-s3` input settings docs](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-aws-s3.html) for details. + - input: aws-cloudwatch template_path: aws-cloudwatch.yml.hbs title: AWS EMR Logs via CloudWatch diff --git a/packages/aws/data_stream/firewall_logs/manifest.yml b/packages/aws/data_stream/firewall_logs/manifest.yml index 2ec4bf7b422..2825aa12d00 100644 --- a/packages/aws/data_stream/firewall_logs/manifest.yml +++ b/packages/aws/data_stream/firewall_logs/manifest.yml @@ -92,6 +92,7 @@ streams: show_user: false description: > Additional settings to be added to the configuration. Be careful using this as it might break the input as those settings are not validated and can override the settings specified above. See [`aws-s3` input settings docs](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-aws-s3.html) for details. + - input: aws-cloudwatch template_path: aws-cloudwatch.yml.hbs title: AWS ELB Logs via CloudWatch diff --git a/packages/aws/data_stream/guardduty/manifest.yml b/packages/aws/data_stream/guardduty/manifest.yml index a67fca063ae..ad84f1a251d 100644 --- a/packages/aws/data_stream/guardduty/manifest.yml +++ b/packages/aws/data_stream/guardduty/manifest.yml @@ -262,3 +262,4 @@ streams: show_user: false description: > Additional settings to be added to the configuration. Be careful using this as it might break the input as those settings are not validated and can override the settings specified above. See [`aws-s3` input settings docs](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-aws-s3.html) for details. + diff --git a/packages/aws/data_stream/route53_resolver_logs/manifest.yml b/packages/aws/data_stream/route53_resolver_logs/manifest.yml index 58887a5d8ac..480ecbca438 100644 --- a/packages/aws/data_stream/route53_resolver_logs/manifest.yml +++ b/packages/aws/data_stream/route53_resolver_logs/manifest.yml @@ -208,3 +208,4 @@ streams: show_user: false description: > Additional settings to be added to the configuration. Be careful using this as it might break the input as those settings are not validated and can override the settings specified above. See [`aws-s3` input settings docs](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-aws-s3.html) for details. + diff --git a/packages/aws/data_stream/s3access/manifest.yml b/packages/aws/data_stream/s3access/manifest.yml index efa16e39116..b85d580b090 100644 --- a/packages/aws/data_stream/s3access/manifest.yml +++ b/packages/aws/data_stream/s3access/manifest.yml @@ -92,3 +92,4 @@ streams: show_user: false description: > Additional settings to be added to the configuration. Be careful using this as it might break the input as those settings are not validated and can override the settings specified above. See [`aws-s3` input settings docs](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-aws-s3.html) for details. + diff --git a/packages/aws/data_stream/vpcflow/manifest.yml b/packages/aws/data_stream/vpcflow/manifest.yml index 9338f76f316..873a0855c30 100644 --- a/packages/aws/data_stream/vpcflow/manifest.yml +++ b/packages/aws/data_stream/vpcflow/manifest.yml @@ -131,6 +131,7 @@ streams: show_user: false description: > Additional settings to be added to the configuration. Be careful using this as it might break the input as those settings are not validated and can override the settings specified above. See [`aws-s3` input settings docs](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-aws-s3.html) for details. + - input: aws-cloudwatch template_path: aws-cloudwatch.yml.hbs title: AWS VPC Flow Logs via CloudWatch diff --git a/packages/aws/data_stream/waf/manifest.yml b/packages/aws/data_stream/waf/manifest.yml index 1071d6408fc..4eaec3ed939 100644 --- a/packages/aws/data_stream/waf/manifest.yml +++ b/packages/aws/data_stream/waf/manifest.yml @@ -92,6 +92,7 @@ streams: show_user: false description: > Additional settings to be added to the configuration. Be careful using this as it might break the input as those settings are not validated and can override the settings specified above. See [`aws-s3` input settings docs](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-aws-s3.html) for details. + - input: aws-cloudwatch template_path: aws-cloudwatch.yml.hbs title: AWS WAF logs via CloudWatch From 52b0ff7db6c4585e225f80f1f94fcdfb51e65282 Mon Sep 17 00:00:00 2001 From: Kyle Pollich Date: Tue, 28 May 2024 13:57:35 -0400 Subject: [PATCH 13/14] Fix cloudtrail --- packages/aws/data_stream/cloudtrail/manifest.yml | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/packages/aws/data_stream/cloudtrail/manifest.yml b/packages/aws/data_stream/cloudtrail/manifest.yml index bb72e908328..044f3fc836c 100644 --- a/packages/aws/data_stream/cloudtrail/manifest.yml +++ b/packages/aws/data_stream/cloudtrail/manifest.yml @@ -147,14 +147,14 @@ streams: description: The maximum number of SQS messages that can be inflight at any time. default: 5 required: false - show_user: false - - name: custom_configurations + - name: custom type: yaml title: Custom configurations multi: false required: false show_user: false - description: Custom settings for the underlying inputs + description: > + Additional settings to be added to the configuration. Be careful using this as it might break the input as those settings are not validated and can override the settings specified above. See [`aws-s3` input settings docs](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-aws-s3.html) for details. - input: httpjson title: AWS CloudTrail Logs via Splunk Enterprise REST API description: Collect AWS CloudTrail logs via Splunk Enterprise REST API @@ -382,12 +382,4 @@ streams: type: bool multi: false default: false - - name: custom - type: yaml - title: Custom configurations - multi: false - required: false - show_user: false - description: > - Additional settings to be added to the configuration. Be careful using this as it might break the input as those settings are not validated and can override the settings specified above. See [`aws-s3` input settings docs](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-aws-s3.html) for details. From c5f7f533c883c48b8adffb1eaacc3fe5c35ca0b7 Mon Sep 17 00:00:00 2001 From: Kyle Pollich Date: Thu, 30 May 2024 07:57:41 -0400 Subject: [PATCH 14/14] Remove blank line from cloudtrail manifest --- packages/aws/data_stream/cloudtrail/manifest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/aws/data_stream/cloudtrail/manifest.yml b/packages/aws/data_stream/cloudtrail/manifest.yml index 044f3fc836c..e36aa8a155b 100644 --- a/packages/aws/data_stream/cloudtrail/manifest.yml +++ b/packages/aws/data_stream/cloudtrail/manifest.yml @@ -155,6 +155,7 @@ streams: show_user: false description: > Additional settings to be added to the configuration. Be careful using this as it might break the input as those settings are not validated and can override the settings specified above. See [`aws-s3` input settings docs](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-aws-s3.html) for details. + - input: httpjson title: AWS CloudTrail Logs via Splunk Enterprise REST API description: Collect AWS CloudTrail logs via Splunk Enterprise REST API @@ -382,4 +383,3 @@ streams: type: bool multi: false default: false -