diff --git a/packages/aws/_dev/benchmark/rally/billing-benchmark/config.yml b/packages/aws/_dev/benchmark/rally/billing-benchmark/config.yml index d8c08c13dc3..04482d3bfd9 100644 --- a/packages/aws/_dev/benchmark/rally/billing-benchmark/config.yml +++ b/packages/aws/_dev/benchmark/rally/billing-benchmark/config.yml @@ -49,6 +49,6 @@ fields: - name: aws.billing.group_definition.type value: "DIMENSION" - name: aws.billing.group_by.INSTANCE_TYPE - enum: ["NoInstanceType", "a1.large", "c5.2xlarge", "c5.xlarge", "c6i.2xlarge", "db.r6g.2xlarge", "db.t2.micro", "dc2.large", "m5.large", "t1.micro", "t2.medium", "t2.micro", "t2.small", "t2.xlarge", "t3.2xlarge", "t3.medium", "t3.xlarge","t3.xlarge"] + enum: ["NoInstanceType", "a1.large", "c5.2xlarge", "c5.xlarge", "c6i.2xlarge", "db.r6g.2xlarge", "db.t2.micro", "dc2.large", "m5.large", "t1.micro", "t2.medium", "t2.micro", "t2.small", "t2.xlarge", "t3.2xlarge", "t3.medium", "t3.xlarge", "t3.xlarge"] - name: aws.billing.group_by.SERVICE - enum: ["Amazon Simple Storage Service", "Amazon Elastic Compute Cloud - Compute", "EC2 - Other", "Amazon Kinesis", "Amazon Relational Database Service", "Amazon Elastic Load Balancing", "AmazonCloudWatch", "AWS CloudTrail", "AWS Config", "AWS Key Management Service", "AWS Lambda", "AWS Secrets Manager", "AWS Service Catalog", "Amazon API Gateway", "Amazon DynamoDB", "Amazon EC2 Container Registry (ECR)", "Amazon Elastic Container Service for Kubernetes", "Amazon Kinesis Firehose", "Amazon Redshift", "Amazon Simple Notification Service", "Amazon Simple Queue Service", "Amazon Virtual Private Cloud"] \ No newline at end of file + enum: ["Amazon Simple Storage Service", "Amazon Elastic Compute Cloud - Compute", "EC2 - Other", "Amazon Kinesis", "Amazon Relational Database Service", "Amazon Elastic Load Balancing", "AmazonCloudWatch", "AWS CloudTrail", "AWS Config", "AWS Key Management Service", "AWS Lambda", "AWS Secrets Manager", "AWS Service Catalog", "Amazon API Gateway", "Amazon DynamoDB", "Amazon EC2 Container Registry (ECR)", "Amazon Elastic Container Service for Kubernetes", "Amazon Kinesis Firehose", "Amazon Redshift", "Amazon Simple Notification Service", "Amazon Simple Queue Service", "Amazon Virtual Private Cloud"] diff --git a/packages/aws/_dev/benchmark/rally/billing-benchmark/fields.yml b/packages/aws/_dev/benchmark/rally/billing-benchmark/fields.yml index d82177e1a45..c784220ad5a 100644 --- a/packages/aws/_dev/benchmark/rally/billing-benchmark/fields.yml +++ b/packages/aws/_dev/benchmark/rally/billing-benchmark/fields.yml @@ -48,4 +48,4 @@ - name: aws.billing.group_by.INSTANCE_TYPE type: keyword - name: aws.billing.group_by.SERVICE - type: keyword \ No newline at end of file + type: keyword diff --git a/packages/aws/_dev/benchmark/rally/ec2_metrics-benchmark.yml b/packages/aws/_dev/benchmark/rally/ec2_metrics-benchmark.yml index cfe99dfae65..ea0faa02699 100644 --- a/packages/aws/_dev/benchmark/rally/ec2_metrics-benchmark.yml +++ b/packages/aws/_dev/benchmark/rally/ec2_metrics-benchmark.yml @@ -11,4 +11,4 @@ corpora: config: path: ./ec2metrics-benchmark/config.yml fields: - path: ./ec2metrics-benchmark/fields.yml \ No newline at end of file + path: ./ec2metrics-benchmark/fields.yml diff --git a/packages/aws/_dev/benchmark/rally/sqs-benchmark.yml b/packages/aws/_dev/benchmark/rally/sqs-benchmark.yml index bbfd29b6022..d663df3be37 100644 --- a/packages/aws/_dev/benchmark/rally/sqs-benchmark.yml +++ b/packages/aws/_dev/benchmark/rally/sqs-benchmark.yml @@ -11,4 +11,4 @@ corpora: config: path: ./sqs-benchmark/config.yml fields: - path: ./sqs-benchmark/fields.yml \ No newline at end of file + path: ./sqs-benchmark/fields.yml diff --git a/packages/aws/changelog.yml b/packages/aws/changelog.yml index 1a3d2296c62..9e41f3dcc09 100644 --- a/packages/aws/changelog.yml +++ b/packages/aws/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "2.15.3" + changes: + - description: Add endpoint + region variables to all SQS based AWS integrations. + type: enhancement + link: https://github.com/elastic/integrations/pull/9865 - version: "2.15.2" changes: - description: Fix AWS Cloudtrail resources field processing. diff --git a/packages/aws/data_stream/apigateway_logs/agent/stream/aws-s3.yml.hbs b/packages/aws/data_stream/apigateway_logs/agent/stream/aws-s3.yml.hbs index dfdf91c9e58..e556e48e608 100644 --- a/packages/aws/data_stream/apigateway_logs/agent/stream/aws-s3.yml.hbs +++ b/packages/aws/data_stream/apigateway_logs/agent/stream/aws-s3.yml.hbs @@ -35,6 +35,12 @@ credential_profile_name: {{credential_profile_name}} {{#if shared_credential_file}} shared_credential_file: {{shared_credential_file}} {{/if}} +{{#if endpoint}} +endpoint: {{endpoint}} +{{/if}} +{{#if region}} +region: {{region}} +{{/if}} {{#if default_region}} default_region: {{default_region}} {{/if}} @@ -69,4 +75,7 @@ publisher_pipeline.disable_host: true {{#if processors}} processors: {{processors}} -{{/if}} \ No newline at end of file +{{/if}} +{{#if custom}} +{{custom}} +{{/if}} diff --git a/packages/aws/data_stream/apigateway_logs/fields/agent.yml b/packages/aws/data_stream/apigateway_logs/fields/agent.yml index 196042e493f..b688ceac1b4 100644 --- a/packages/aws/data_stream/apigateway_logs/fields/agent.yml +++ b/packages/aws/data_stream/apigateway_logs/fields/agent.yml @@ -32,6 +32,7 @@ example: "stretch" description: > OS codename, if any. + - name: log.offset type: long description: Log offset diff --git a/packages/aws/data_stream/apigateway_logs/manifest.yml b/packages/aws/data_stream/apigateway_logs/manifest.yml index 9a46702de0d..33d87aaefeb 100644 --- a/packages/aws/data_stream/apigateway_logs/manifest.yml +++ b/packages/aws/data_stream/apigateway_logs/manifest.yml @@ -29,6 +29,22 @@ streams: required: false show_user: true description: Mandatory if the "Collect logs via S3 Bucket" switch is off. URL of the AWS SQS queue that messages will be received from. + - name: endpoint + type: text + title: Endpoint + multi: false + required: false + show_user: false + default: "" + description: URL of the entry point for an AWS web service + - name: region + type: text + title: Region + multi: false + required: false + show_user: false + default: "" + description: The name of the AWS region of the endpoint - name: bucket_list_prefix type: text title: "[S3] Bucket Prefix" @@ -107,6 +123,15 @@ streams: type: bool multi: false default: false + - name: custom + type: yaml + title: Custom configurations + multi: false + required: false + show_user: false + description: > + Additional settings to be added to the configuration. Be careful using this as it might break the input as those settings are not validated and can override the settings specified above. See [`aws-s3` input settings docs](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-aws-s3.html) for details. + - input: aws-cloudwatch template_path: aws-cloudwatch.yml.hbs title: AWS API Gateway Logs via CloudWatch diff --git a/packages/aws/data_stream/apigateway_metrics/agent/stream/stream.yml.hbs b/packages/aws/data_stream/apigateway_metrics/agent/stream/stream.yml.hbs index a16d15846a2..0b1a916ac72 100644 --- a/packages/aws/data_stream/apigateway_metrics/agent/stream/stream.yml.hbs +++ b/packages/aws/data_stream/apigateway_metrics/agent/stream/stream.yml.hbs @@ -63,4 +63,4 @@ metrics: name: - Latency - IntegrationLatency - - DataProcessed \ No newline at end of file + - DataProcessed diff --git a/packages/aws/data_stream/billing/manifest.yml b/packages/aws/data_stream/billing/manifest.yml index 3fda2894036..97d2c4a444a 100644 --- a/packages/aws/data_stream/billing/manifest.yml +++ b/packages/aws/data_stream/billing/manifest.yml @@ -58,6 +58,7 @@ streams: default: false description: > Enable leaderelection between a set of Elastic Agents running on Kubernetes. + - name: processors type: yaml title: Processors @@ -66,5 +67,6 @@ streams: show_user: false description: > Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the events are shipped. See [Processors](https://www.elastic.co/guide/en/fleet/current/elastic-agent-processor-configuration.html) for details. + title: AWS Billing Metrics description: Collect billing metrics from Amazon Web Services with Elastic Agent. diff --git a/packages/aws/data_stream/cloudfront_logs/agent/stream/aws-s3.yml.hbs b/packages/aws/data_stream/cloudfront_logs/agent/stream/aws-s3.yml.hbs index 1d0bbc280fc..acb64f7b40a 100644 --- a/packages/aws/data_stream/cloudfront_logs/agent/stream/aws-s3.yml.hbs +++ b/packages/aws/data_stream/cloudfront_logs/agent/stream/aws-s3.yml.hbs @@ -14,6 +14,12 @@ api_timeout: {{api_timeout}} {{#if max_number_of_messages}} max_number_of_messages: {{max_number_of_messages}} {{/if}} +{{#if endpoint}} +endpoint: {{endpoint}} +{{/if}} +{{#if region}} +region: {{region}} +{{/if}} {{#if default_region}} default_region: {{default_region}} {{/if}} @@ -52,4 +58,7 @@ processors: message: "^#.*" {{#if processors}} {{processors}} -{{/if}} \ No newline at end of file +{{/if}} +{{#if custom}} +{{custom}} +{{/if}} diff --git a/packages/aws/data_stream/cloudfront_logs/fields/agent.yml b/packages/aws/data_stream/cloudfront_logs/fields/agent.yml index 1e09834aad6..2ddef01a27b 100644 --- a/packages/aws/data_stream/cloudfront_logs/fields/agent.yml +++ b/packages/aws/data_stream/cloudfront_logs/fields/agent.yml @@ -32,6 +32,7 @@ example: "stretch" description: > OS codename, if any. + - name: log.offset type: long description: Log offset diff --git a/packages/aws/data_stream/cloudfront_logs/manifest.yml b/packages/aws/data_stream/cloudfront_logs/manifest.yml index 674490658eb..00148471b8d 100644 --- a/packages/aws/data_stream/cloudfront_logs/manifest.yml +++ b/packages/aws/data_stream/cloudfront_logs/manifest.yml @@ -27,6 +27,22 @@ streams: required: true show_user: true description: URL of the AWS SQS queue that messages will be received from. + - name: endpoint + type: text + title: Endpoint + multi: false + required: false + show_user: false + default: "" + description: URL of the entry point for an AWS web service + - name: region + type: text + title: Region + multi: false + required: false + show_user: false + default: "" + description: The name of the AWS region of the endpoint - name: fips_enabled type: bool title: Enable S3 FIPS @@ -73,3 +89,12 @@ streams: default: 5 required: false show_user: false + - name: custom + type: yaml + title: Custom configurations + multi: false + required: false + show_user: false + description: > + Additional settings to be added to the configuration. Be careful using this as it might break the input as those settings are not validated and can override the settings specified above. See [`aws-s3` input settings docs](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-aws-s3.html) for details. + diff --git a/packages/aws/data_stream/cloudtrail/agent/stream/aws-s3.yml.hbs b/packages/aws/data_stream/cloudtrail/agent/stream/aws-s3.yml.hbs index b54c36bb530..c45ab6f37e9 100644 --- a/packages/aws/data_stream/cloudtrail/agent/stream/aws-s3.yml.hbs +++ b/packages/aws/data_stream/cloudtrail/agent/stream/aws-s3.yml.hbs @@ -49,8 +49,12 @@ credential_profile_name: {{credential_profile_name}} {{#if shared_credential_file}} shared_credential_file: {{shared_credential_file}} {{/if}} - - +{{#if endpoint}} +endpoint: {{endpoint}} +{{/if}} +{{#if region}} +region: {{region}} +{{/if}} {{#if default_region}} default_region: {{default_region}} {{/if}} @@ -86,4 +90,7 @@ publisher_pipeline.disable_host: true {{#if processors}} processors: {{processors}} -{{/if}} \ No newline at end of file +{{/if}} +{{#if custom}} +{{custom}} +{{/if}} diff --git a/packages/aws/data_stream/cloudtrail/fields/agent.yml b/packages/aws/data_stream/cloudtrail/fields/agent.yml index 196042e493f..b688ceac1b4 100644 --- a/packages/aws/data_stream/cloudtrail/fields/agent.yml +++ b/packages/aws/data_stream/cloudtrail/fields/agent.yml @@ -32,6 +32,7 @@ example: "stretch" description: > OS codename, if any. + - name: log.offset type: long description: Log offset diff --git a/packages/aws/data_stream/cloudtrail/manifest.yml b/packages/aws/data_stream/cloudtrail/manifest.yml index ca548f11d96..e36aa8a155b 100644 --- a/packages/aws/data_stream/cloudtrail/manifest.yml +++ b/packages/aws/data_stream/cloudtrail/manifest.yml @@ -28,6 +28,22 @@ streams: required: false show_user: true description: Mandatory if the "Collect logs via S3 Bucket" switch is off. URL of the AWS SQS queue that messages will be received from. + - name: endpoint + type: text + title: Endpoint + multi: false + required: false + show_user: false + default: "" + description: URL of the entry point for an AWS web service + - name: region + type: text + title: Region + multi: false + required: false + show_user: false + default: "" + description: The name of the AWS region of the endpoint - name: bucket_list_prefix type: text title: "[S3] Bucket Prefix" @@ -101,7 +117,7 @@ streams: - name: cloudtrail_regex type: text title: CloudTrail Logs regex - default: '/CloudTrail/' + default: "/CloudTrail/" required: false show_user: false description: | @@ -110,7 +126,7 @@ streams: - name: cloudtrail_digest_regex type: text title: CloudTrail Digest Logs regex - default: '/CloudTrail-Digest/' + default: "/CloudTrail-Digest/" required: false show_user: false description: | @@ -119,7 +135,7 @@ streams: - name: cloudtrail_insight_regex type: text title: CloudTrail Insight Logs regex - default: '/CloudTrail-Insight/' + default: "/CloudTrail-Insight/" required: false show_user: false description: | @@ -131,7 +147,15 @@ streams: description: The maximum number of SQS messages that can be inflight at any time. default: 5 required: false + - name: custom + type: yaml + title: Custom configurations + multi: false + required: false show_user: false + description: > + Additional settings to be added to the configuration. Be careful using this as it might break the input as those settings are not validated and can override the settings specified above. See [`aws-s3` input settings docs](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-aws-s3.html) for details. + - input: httpjson title: AWS CloudTrail Logs via Splunk Enterprise REST API description: Collect AWS CloudTrail logs via Splunk Enterprise REST API diff --git a/packages/aws/data_stream/cloudwatch_metrics/fields/base-fields.yml b/packages/aws/data_stream/cloudwatch_metrics/fields/base-fields.yml index e13227ae1d1..2b34ac77b17 100644 --- a/packages/aws/data_stream/cloudwatch_metrics/fields/base-fields.yml +++ b/packages/aws/data_stream/cloudwatch_metrics/fields/base-fields.yml @@ -13,4 +13,4 @@ - name: event.module type: constant_keyword description: Event module - value: aws \ No newline at end of file + value: aws diff --git a/packages/aws/data_stream/dynamodb/agent/stream/stream.yml.hbs b/packages/aws/data_stream/dynamodb/agent/stream/stream.yml.hbs index 74b1ae36bfc..0f40351ea3b 100644 --- a/packages/aws/data_stream/dynamodb/agent/stream/stream.yml.hbs +++ b/packages/aws/data_stream/dynamodb/agent/stream/stream.yml.hbs @@ -81,4 +81,4 @@ metrics: - AccountMaxTableLevelWrites - AccountMaxWrites - MaxProvisionedTableReadCapacityUtilization - - MaxProvisionedTableWriteCapacityUtilization \ No newline at end of file + - MaxProvisionedTableWriteCapacityUtilization diff --git a/packages/aws/data_stream/ebs/agent/stream/stream.yml.hbs b/packages/aws/data_stream/ebs/agent/stream/stream.yml.hbs index ab300cc6746..89615c3737a 100644 --- a/packages/aws/data_stream/ebs/agent/stream/stream.yml.hbs +++ b/packages/aws/data_stream/ebs/agent/stream/stream.yml.hbs @@ -50,4 +50,4 @@ metrics: - namespace: AWS/EBS statistic: ["Sum"] name: ["VolumeTotalReadTime", "VolumeTotalWriteTime", "VolumeIdleTime"] - resource_type: ec2 \ No newline at end of file + resource_type: ec2 diff --git a/packages/aws/data_stream/ec2_logs/agent/stream/aws-s3.yml.hbs b/packages/aws/data_stream/ec2_logs/agent/stream/aws-s3.yml.hbs index 77fae5d0dc8..153def8018a 100644 --- a/packages/aws/data_stream/ec2_logs/agent/stream/aws-s3.yml.hbs +++ b/packages/aws/data_stream/ec2_logs/agent/stream/aws-s3.yml.hbs @@ -14,6 +14,12 @@ api_timeout: {{api_timeout}} {{#if max_number_of_messages}} max_number_of_messages: {{max_number_of_messages}} {{/if}} +{{#if endpoint}} +endpoint: {{endpoint}} +{{/if}} +{{#if region}} +region: {{region}} +{{/if}} {{#if default_region}} default_region: {{default_region}} {{/if}} @@ -48,4 +54,7 @@ publisher_pipeline.disable_host: true {{#if processors}} processors: {{processors}} -{{/if}} \ No newline at end of file +{{/if}} +{{#if custom}} +{{custom}} +{{/if}} diff --git a/packages/aws/data_stream/ec2_logs/fields/agent.yml b/packages/aws/data_stream/ec2_logs/fields/agent.yml index 1e09834aad6..2ddef01a27b 100644 --- a/packages/aws/data_stream/ec2_logs/fields/agent.yml +++ b/packages/aws/data_stream/ec2_logs/fields/agent.yml @@ -32,6 +32,7 @@ example: "stretch" description: > OS codename, if any. + - name: log.offset type: long description: Log offset diff --git a/packages/aws/data_stream/ec2_logs/manifest.yml b/packages/aws/data_stream/ec2_logs/manifest.yml index df69ef4bcc3..8e52a9d9a64 100644 --- a/packages/aws/data_stream/ec2_logs/manifest.yml +++ b/packages/aws/data_stream/ec2_logs/manifest.yml @@ -28,6 +28,22 @@ streams: required: true show_user: true description: URL of the AWS SQS queue that messages will be received from. + - name: endpoint + type: text + title: Endpoint + multi: false + required: false + show_user: false + default: "" + description: URL of the entry point for an AWS web service + - name: region + type: text + title: Region + multi: false + required: false + show_user: false + default: "" + description: The name of the AWS region of the endpoint - name: fips_enabled type: bool title: Enable S3 FIPS @@ -69,6 +85,15 @@ streams: default: 5 required: false show_user: false + - name: custom + type: yaml + title: Custom configurations + multi: false + required: false + show_user: false + description: > + Additional settings to be added to the configuration. Be careful using this as it might break the input as those settings are not validated and can override the settings specified above. See [`aws-s3` input settings docs](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-aws-s3.html) for details. + - input: aws-cloudwatch template_path: aws-cloudwatch.yml.hbs title: AWS EC2 Logs via CloudWatch diff --git a/packages/aws/data_stream/elb_logs/agent/stream/aws-s3.yml.hbs b/packages/aws/data_stream/elb_logs/agent/stream/aws-s3.yml.hbs index 77fae5d0dc8..153def8018a 100644 --- a/packages/aws/data_stream/elb_logs/agent/stream/aws-s3.yml.hbs +++ b/packages/aws/data_stream/elb_logs/agent/stream/aws-s3.yml.hbs @@ -14,6 +14,12 @@ api_timeout: {{api_timeout}} {{#if max_number_of_messages}} max_number_of_messages: {{max_number_of_messages}} {{/if}} +{{#if endpoint}} +endpoint: {{endpoint}} +{{/if}} +{{#if region}} +region: {{region}} +{{/if}} {{#if default_region}} default_region: {{default_region}} {{/if}} @@ -48,4 +54,7 @@ publisher_pipeline.disable_host: true {{#if processors}} processors: {{processors}} -{{/if}} \ No newline at end of file +{{/if}} +{{#if custom}} +{{custom}} +{{/if}} diff --git a/packages/aws/data_stream/elb_logs/fields/agent.yml b/packages/aws/data_stream/elb_logs/fields/agent.yml index 1e09834aad6..2ddef01a27b 100644 --- a/packages/aws/data_stream/elb_logs/fields/agent.yml +++ b/packages/aws/data_stream/elb_logs/fields/agent.yml @@ -32,6 +32,7 @@ example: "stretch" description: > OS codename, if any. + - name: log.offset type: long description: Log offset diff --git a/packages/aws/data_stream/elb_logs/manifest.yml b/packages/aws/data_stream/elb_logs/manifest.yml index 17dbabac05e..72c842f3092 100644 --- a/packages/aws/data_stream/elb_logs/manifest.yml +++ b/packages/aws/data_stream/elb_logs/manifest.yml @@ -27,6 +27,22 @@ streams: required: true show_user: true description: URL of the AWS SQS queue that messages will be received from. + - name: endpoint + type: text + title: Endpoint + multi: false + required: false + show_user: false + default: "" + description: URL of the entry point for an AWS web service + - name: region + type: text + title: Region + multi: false + required: false + show_user: false + default: "" + description: The name of the AWS region of the endpoint - name: fips_enabled type: bool title: Enable S3 FIPS @@ -68,6 +84,15 @@ streams: default: 5 required: false show_user: false + - name: custom + type: yaml + title: Custom configurations + multi: false + required: false + show_user: false + description: > + Additional settings to be added to the configuration. Be careful using this as it might break the input as those settings are not validated and can override the settings specified above. See [`aws-s3` input settings docs](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-aws-s3.html) for details. + - input: aws-cloudwatch template_path: aws-cloudwatch.yml.hbs title: AWS ELB Logs via CloudWatch diff --git a/packages/aws/data_stream/elb_metrics/agent/stream/stream.yml.hbs b/packages/aws/data_stream/elb_metrics/agent/stream/stream.yml.hbs index 52f762fcc89..154f98e296f 100644 --- a/packages/aws/data_stream/elb_metrics/agent/stream/stream.yml.hbs +++ b/packages/aws/data_stream/elb_metrics/agent/stream/stream.yml.hbs @@ -140,4 +140,4 @@ metrics: statistic: ["Maximum"] name: - HealthyHostCount - - UnHealthyHostCount \ No newline at end of file + - UnHealthyHostCount diff --git a/packages/aws/data_stream/emr_logs/agent/stream/aws-s3.yml.hbs b/packages/aws/data_stream/emr_logs/agent/stream/aws-s3.yml.hbs index c479d4b6dd0..ead858d32e0 100644 --- a/packages/aws/data_stream/emr_logs/agent/stream/aws-s3.yml.hbs +++ b/packages/aws/data_stream/emr_logs/agent/stream/aws-s3.yml.hbs @@ -34,7 +34,12 @@ file_selectors: {{/if}} - +{{#if endpoint}} +endpoint: {{endpoint}} +{{/if}} +{{#if region}} +region: {{region}} +{{/if}} {{#if default_region}} default_region: {{default_region}} {{/if}} @@ -75,4 +80,7 @@ publisher_pipeline.disable_host: true {{#if processors}} processors: {{processors}} -{{/if}} \ No newline at end of file +{{/if}} +{{#if custom}} +{{custom}} +{{/if}} diff --git a/packages/aws/data_stream/emr_logs/fields/agent.yml b/packages/aws/data_stream/emr_logs/fields/agent.yml index 196042e493f..b688ceac1b4 100644 --- a/packages/aws/data_stream/emr_logs/fields/agent.yml +++ b/packages/aws/data_stream/emr_logs/fields/agent.yml @@ -32,6 +32,7 @@ example: "stretch" description: > OS codename, if any. + - name: log.offset type: long description: Log offset diff --git a/packages/aws/data_stream/emr_logs/manifest.yml b/packages/aws/data_stream/emr_logs/manifest.yml index 27e72b79859..ef2b598e807 100644 --- a/packages/aws/data_stream/emr_logs/manifest.yml +++ b/packages/aws/data_stream/emr_logs/manifest.yml @@ -30,6 +30,22 @@ streams: show_user: true default: false description: Mandatory if the "Collect logs via S3 Bucket" switch is off. URL of the AWS SQS queue that messages will be received from. + - name: endpoint + type: text + title: Endpoint + multi: false + required: false + show_user: false + default: "" + description: URL of the entry point for an AWS web service + - name: region + type: text + title: Region + multi: false + required: false + show_user: false + default: "" + description: The name of the AWS region of the end point - name: bucket_list_prefix type: text title: "[S3] Bucket Prefix" @@ -108,6 +124,15 @@ streams: type: bool multi: false default: false + - name: custom + type: yaml + title: Custom configurations + multi: false + required: false + show_user: false + description: > + Additional settings to be added to the configuration. Be careful using this as it might break the input as those settings are not validated and can override the settings specified above. See [`aws-s3` input settings docs](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-aws-s3.html) for details. + - input: aws-cloudwatch template_path: aws-cloudwatch.yml.hbs title: AWS EMR Logs via CloudWatch diff --git a/packages/aws/data_stream/firewall_logs/agent/stream/aws-s3.yml.hbs b/packages/aws/data_stream/firewall_logs/agent/stream/aws-s3.yml.hbs index 6edb2d8079b..077994e4d79 100644 --- a/packages/aws/data_stream/firewall_logs/agent/stream/aws-s3.yml.hbs +++ b/packages/aws/data_stream/firewall_logs/agent/stream/aws-s3.yml.hbs @@ -14,6 +14,12 @@ api_timeout: {{api_timeout}} {{#if max_number_of_messages}} max_number_of_messages: {{max_number_of_messages}} {{/if}} +{{#if endpoint}} +endpoint: {{endpoint}} +{{/if}} +{{#if region}} +region: {{region}} +{{/if}} {{#if default_region}} default_region: {{default_region}} {{/if}} @@ -49,4 +55,7 @@ publisher_pipeline.disable_host: true {{#if processors}} processors: {{processors}} -{{/if}} \ No newline at end of file +{{/if}} +{{#if custom}} +{{custom}} +{{/if}} diff --git a/packages/aws/data_stream/firewall_logs/fields/agent.yml b/packages/aws/data_stream/firewall_logs/fields/agent.yml index 1e09834aad6..2ddef01a27b 100644 --- a/packages/aws/data_stream/firewall_logs/fields/agent.yml +++ b/packages/aws/data_stream/firewall_logs/fields/agent.yml @@ -32,6 +32,7 @@ example: "stretch" description: > OS codename, if any. + - name: log.offset type: long description: Log offset diff --git a/packages/aws/data_stream/firewall_logs/manifest.yml b/packages/aws/data_stream/firewall_logs/manifest.yml index 8c1334a94b4..2825aa12d00 100644 --- a/packages/aws/data_stream/firewall_logs/manifest.yml +++ b/packages/aws/data_stream/firewall_logs/manifest.yml @@ -27,6 +27,22 @@ streams: required: true show_user: true description: URL of the AWS SQS queue that messages will be received from. + - name: endpoint + type: text + title: Endpoint + multi: false + required: false + show_user: false + default: "" + description: URL of the entry point for an AWS web service + - name: region + type: text + title: Region + multi: false + required: false + show_user: false + default: "" + description: The name of the AWS region of the end point - name: fips_enabled type: bool title: Enable S3 FIPS @@ -68,6 +84,15 @@ streams: default: 5 required: false show_user: false + - name: custom + type: yaml + title: Custom configurations + multi: false + required: false + show_user: false + description: > + Additional settings to be added to the configuration. Be careful using this as it might break the input as those settings are not validated and can override the settings specified above. See [`aws-s3` input settings docs](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-aws-s3.html) for details. + - input: aws-cloudwatch template_path: aws-cloudwatch.yml.hbs title: AWS ELB Logs via CloudWatch diff --git a/packages/aws/data_stream/guardduty/agent/stream/aws-s3.yml.hbs b/packages/aws/data_stream/guardduty/agent/stream/aws-s3.yml.hbs index 9d48e69a97c..9a8da946871 100644 --- a/packages/aws/data_stream/guardduty/agent/stream/aws-s3.yml.hbs +++ b/packages/aws/data_stream/guardduty/agent/stream/aws-s3.yml.hbs @@ -34,6 +34,12 @@ file_selectors: {{/if}} +{{#if endpoint}} +endpoint: {{endpoint}} +{{/if}} +{{#if region}} +region: {{region}} +{{/if}} {{#if access_key_id}} access_key_id: {{access_key_id}} {{/if}} @@ -79,4 +85,7 @@ publisher_pipeline.disable_host: true {{#if processors}} processors: {{processors}} -{{/if}} \ No newline at end of file +{{/if}} +{{#if custom}} +{{custom}} +{{/if}} diff --git a/packages/aws/data_stream/guardduty/manifest.yml b/packages/aws/data_stream/guardduty/manifest.yml index 08af59ad4c4..ad84f1a251d 100644 --- a/packages/aws/data_stream/guardduty/manifest.yml +++ b/packages/aws/data_stream/guardduty/manifest.yml @@ -197,6 +197,22 @@ streams: required: false show_user: false description: If the SQS queue will have events that correspond to files that this integration shouldn’t process, file_selectors can be used to limit the files that are downloaded. This is a list of selectors which are made up of regex and expand_event_list_from_field options. The regex should match the S3 object key in the SQS message, and the optional expand_event_list_from_field is the same as the global setting. If file_selectors is given, then any global expand_event_list_from_field value is ignored in favor of the ones specified in the file_selectors. Regexes use [RE2 syntax](https://pkg.go.dev/regexp/syntax). Files that don’t match one of the regexes will not be processed. + - name: endpoint + type: text + title: Endpoint + multi: false + required: false + show_user: false + default: "" + description: URL of the entry point for an AWS web service + - name: region + type: text + title: Region + multi: false + required: false + show_user: false + default: "" + description: The name of the AWS region of the end point - name: fips_enabled type: bool title: Enable S3 FIPS @@ -238,3 +254,12 @@ streams: show_user: false description: >- Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details. + - name: custom + type: yaml + title: Custom configurations + multi: false + required: false + show_user: false + description: > + Additional settings to be added to the configuration. Be careful using this as it might break the input as those settings are not validated and can override the settings specified above. See [`aws-s3` input settings docs](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-aws-s3.html) for details. + diff --git a/packages/aws/data_stream/rds/agent/stream/stream.yml.hbs b/packages/aws/data_stream/rds/agent/stream/stream.yml.hbs index 4c5b7abca05..a19b2b409d6 100644 --- a/packages/aws/data_stream/rds/agent/stream/stream.yml.hbs +++ b/packages/aws/data_stream/rds/agent/stream/stream.yml.hbs @@ -45,4 +45,4 @@ proxy_url: {{proxy_url}} metrics: - namespace: AWS/RDS resource_type: rds - statistic: ["Average"] \ No newline at end of file + statistic: ["Average"] diff --git a/packages/aws/data_stream/route53_resolver_logs/agent/stream/aws-s3.yml.hbs b/packages/aws/data_stream/route53_resolver_logs/agent/stream/aws-s3.yml.hbs index 6edb2d8079b..077994e4d79 100644 --- a/packages/aws/data_stream/route53_resolver_logs/agent/stream/aws-s3.yml.hbs +++ b/packages/aws/data_stream/route53_resolver_logs/agent/stream/aws-s3.yml.hbs @@ -14,6 +14,12 @@ api_timeout: {{api_timeout}} {{#if max_number_of_messages}} max_number_of_messages: {{max_number_of_messages}} {{/if}} +{{#if endpoint}} +endpoint: {{endpoint}} +{{/if}} +{{#if region}} +region: {{region}} +{{/if}} {{#if default_region}} default_region: {{default_region}} {{/if}} @@ -49,4 +55,7 @@ publisher_pipeline.disable_host: true {{#if processors}} processors: {{processors}} -{{/if}} \ No newline at end of file +{{/if}} +{{#if custom}} +{{custom}} +{{/if}} diff --git a/packages/aws/data_stream/route53_resolver_logs/fields/agent.yml b/packages/aws/data_stream/route53_resolver_logs/fields/agent.yml index 1e09834aad6..2ddef01a27b 100644 --- a/packages/aws/data_stream/route53_resolver_logs/fields/agent.yml +++ b/packages/aws/data_stream/route53_resolver_logs/fields/agent.yml @@ -32,6 +32,7 @@ example: "stretch" description: > OS codename, if any. + - name: log.offset type: long description: Log offset diff --git a/packages/aws/data_stream/route53_resolver_logs/manifest.yml b/packages/aws/data_stream/route53_resolver_logs/manifest.yml index 294a6b204fd..480ecbca438 100644 --- a/packages/aws/data_stream/route53_resolver_logs/manifest.yml +++ b/packages/aws/data_stream/route53_resolver_logs/manifest.yml @@ -143,6 +143,22 @@ streams: required: true show_user: true description: URL of the AWS SQS queue that messages will be received from. + - name: endpoint + type: text + title: Endpoint + multi: false + required: false + show_user: false + default: "" + description: URL of the entry point for an AWS web service + - name: region + type: text + title: Region + multi: false + required: false + show_user: false + default: "" + description: The name of the AWS region of the end point - name: fips_enabled type: bool title: Enable S3 FIPS @@ -184,3 +200,12 @@ streams: default: 5 required: false show_user: false + - name: custom + type: yaml + title: Custom configurations + multi: false + required: false + show_user: false + description: > + Additional settings to be added to the configuration. Be careful using this as it might break the input as those settings are not validated and can override the settings specified above. See [`aws-s3` input settings docs](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-aws-s3.html) for details. + diff --git a/packages/aws/data_stream/s3_storage_lens/agent/stream/stream.yml.hbs b/packages/aws/data_stream/s3_storage_lens/agent/stream/stream.yml.hbs index 08fabbf30b3..d547cceb8c2 100644 --- a/packages/aws/data_stream/s3_storage_lens/agent/stream/stream.yml.hbs +++ b/packages/aws/data_stream/s3_storage_lens/agent/stream/stream.yml.hbs @@ -107,4 +107,4 @@ processors: - drop_fields: ignore_missing: true fields: - - "aws.storage-lens" \ No newline at end of file + - "aws.storage-lens" diff --git a/packages/aws/data_stream/s3access/agent/stream/aws-s3.yml.hbs b/packages/aws/data_stream/s3access/agent/stream/aws-s3.yml.hbs index 77fae5d0dc8..153def8018a 100644 --- a/packages/aws/data_stream/s3access/agent/stream/aws-s3.yml.hbs +++ b/packages/aws/data_stream/s3access/agent/stream/aws-s3.yml.hbs @@ -14,6 +14,12 @@ api_timeout: {{api_timeout}} {{#if max_number_of_messages}} max_number_of_messages: {{max_number_of_messages}} {{/if}} +{{#if endpoint}} +endpoint: {{endpoint}} +{{/if}} +{{#if region}} +region: {{region}} +{{/if}} {{#if default_region}} default_region: {{default_region}} {{/if}} @@ -48,4 +54,7 @@ publisher_pipeline.disable_host: true {{#if processors}} processors: {{processors}} -{{/if}} \ No newline at end of file +{{/if}} +{{#if custom}} +{{custom}} +{{/if}} diff --git a/packages/aws/data_stream/s3access/fields/agent.yml b/packages/aws/data_stream/s3access/fields/agent.yml index 1e09834aad6..2ddef01a27b 100644 --- a/packages/aws/data_stream/s3access/fields/agent.yml +++ b/packages/aws/data_stream/s3access/fields/agent.yml @@ -32,6 +32,7 @@ example: "stretch" description: > OS codename, if any. + - name: log.offset type: long description: Log offset diff --git a/packages/aws/data_stream/s3access/manifest.yml b/packages/aws/data_stream/s3access/manifest.yml index 8240665215a..b85d580b090 100644 --- a/packages/aws/data_stream/s3access/manifest.yml +++ b/packages/aws/data_stream/s3access/manifest.yml @@ -27,6 +27,22 @@ streams: required: true show_user: true description: URL of the AWS SQS queue that messages will be received from. + - name: endpoint + type: text + title: Endpoint + multi: false + required: false + show_user: false + default: "" + description: URL of the entry point for an AWS web service + - name: region + type: text + title: Region + multi: false + required: false + show_user: false + default: "" + description: The name of the AWS region of the end point - name: fips_enabled type: bool title: Enable S3 FIPS @@ -68,3 +84,12 @@ streams: default: 5 required: false show_user: false + - name: custom + type: yaml + title: Custom configurations + multi: false + required: false + show_user: false + description: > + Additional settings to be added to the configuration. Be careful using this as it might break the input as those settings are not validated and can override the settings specified above. See [`aws-s3` input settings docs](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-aws-s3.html) for details. + diff --git a/packages/aws/data_stream/usage/agent/stream/stream.yml.hbs b/packages/aws/data_stream/usage/agent/stream/stream.yml.hbs index 9602b37d578..d20ef53b9c0 100644 --- a/packages/aws/data_stream/usage/agent/stream/stream.yml.hbs +++ b/packages/aws/data_stream/usage/agent/stream/stream.yml.hbs @@ -44,4 +44,4 @@ proxy_url: {{proxy_url}} {{/if}} metrics: - namespace: AWS/Usage - statistic: ["Sum"] \ No newline at end of file + statistic: ["Sum"] diff --git a/packages/aws/data_stream/vpcflow/agent/stream/aws-s3.yml.hbs b/packages/aws/data_stream/vpcflow/agent/stream/aws-s3.yml.hbs index 64796e6215f..b5909cdcc51 100644 --- a/packages/aws/data_stream/vpcflow/agent/stream/aws-s3.yml.hbs +++ b/packages/aws/data_stream/vpcflow/agent/stream/aws-s3.yml.hbs @@ -36,6 +36,12 @@ credential_profile_name: {{credential_profile_name}} {{#if shared_credential_file}} shared_credential_file: {{shared_credential_file}} {{/if}} +{{#if endpoint}} +endpoint: {{endpoint}} +{{/if}} +{{#if region}} +region: {{region}} +{{/if}} {{#if default_region}} default_region: {{default_region}} {{/if}} @@ -70,4 +76,7 @@ publisher_pipeline.disable_host: true {{#if processors}} processors: {{processors}} -{{/if}} \ No newline at end of file +{{/if}} +{{#if custom}} +{{custom}} +{{/if}} diff --git a/packages/aws/data_stream/vpcflow/fields/agent.yml b/packages/aws/data_stream/vpcflow/fields/agent.yml index 196042e493f..b688ceac1b4 100644 --- a/packages/aws/data_stream/vpcflow/fields/agent.yml +++ b/packages/aws/data_stream/vpcflow/fields/agent.yml @@ -32,6 +32,7 @@ example: "stretch" description: > OS codename, if any. + - name: log.offset type: long description: Log offset diff --git a/packages/aws/data_stream/vpcflow/manifest.yml b/packages/aws/data_stream/vpcflow/manifest.yml index a8117ff5a00..873a0855c30 100644 --- a/packages/aws/data_stream/vpcflow/manifest.yml +++ b/packages/aws/data_stream/vpcflow/manifest.yml @@ -73,6 +73,22 @@ streams: default: 5 required: false show_user: false + - name: endpoint + type: text + title: Endpoint + multi: false + required: false + show_user: false + default: "" + description: URL of the entry point for an AWS web service + - name: region + type: text + title: Region + multi: false + required: false + show_user: false + default: "" + description: The name of the AWS region of the end point - name: fips_enabled type: bool title: Enable S3 FIPS @@ -107,6 +123,15 @@ streams: type: bool multi: false default: false + - name: custom + type: yaml + title: Custom configurations + multi: false + required: false + show_user: false + description: > + Additional settings to be added to the configuration. Be careful using this as it might break the input as those settings are not validated and can override the settings specified above. See [`aws-s3` input settings docs](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-aws-s3.html) for details. + - input: aws-cloudwatch template_path: aws-cloudwatch.yml.hbs title: AWS VPC Flow Logs via CloudWatch diff --git a/packages/aws/data_stream/waf/agent/stream/aws-s3.yml.hbs b/packages/aws/data_stream/waf/agent/stream/aws-s3.yml.hbs index c38defcb33c..fd68da1ff68 100644 --- a/packages/aws/data_stream/waf/agent/stream/aws-s3.yml.hbs +++ b/packages/aws/data_stream/waf/agent/stream/aws-s3.yml.hbs @@ -15,6 +15,12 @@ api_timeout: {{api_timeout}} {{#if max_number_of_messages}} max_number_of_messages: {{max_number_of_messages}} {{/if}} +{{#if endpoint}} +endpoint: {{endpoint}} +{{/if}} +{{#if region}} +region: {{region}} +{{/if}} {{#if default_region}} default_region: {{default_region}} {{/if}} @@ -49,4 +55,7 @@ publisher_pipeline.disable_host: true {{#if processors}} processors: {{processors}} -{{/if}} \ No newline at end of file +{{/if}} +{{#if custom}} +{{custom}} +{{/if}} diff --git a/packages/aws/data_stream/waf/fields/agent.yml b/packages/aws/data_stream/waf/fields/agent.yml index 1e09834aad6..2ddef01a27b 100644 --- a/packages/aws/data_stream/waf/fields/agent.yml +++ b/packages/aws/data_stream/waf/fields/agent.yml @@ -32,6 +32,7 @@ example: "stretch" description: > OS codename, if any. + - name: log.offset type: long description: Log offset diff --git a/packages/aws/data_stream/waf/manifest.yml b/packages/aws/data_stream/waf/manifest.yml index 892bfb078a8..4eaec3ed939 100644 --- a/packages/aws/data_stream/waf/manifest.yml +++ b/packages/aws/data_stream/waf/manifest.yml @@ -27,6 +27,22 @@ streams: required: true show_user: true description: URL of the AWS SQS queue that messages will be received from. + - name: endpoint + type: text + title: Endpoint + multi: false + required: false + show_user: false + default: "" + description: URL of the entry point for an AWS web service + - name: region + type: text + title: Region + multi: false + required: false + show_user: false + default: "" + description: The name of the AWS region of the end point - name: fips_enabled type: bool title: Enable S3 FIPS @@ -68,6 +84,15 @@ streams: default: 5 required: false show_user: false + - name: custom + type: yaml + title: Custom configurations + multi: false + required: false + show_user: false + description: > + Additional settings to be added to the configuration. Be careful using this as it might break the input as those settings are not validated and can override the settings specified above. See [`aws-s3` input settings docs](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-aws-s3.html) for details. + - input: aws-cloudwatch template_path: aws-cloudwatch.yml.hbs title: AWS WAF logs via CloudWatch diff --git a/packages/aws/manifest.yml b/packages/aws/manifest.yml index e727aafc2af..50d2e1c0e50 100644 --- a/packages/aws/manifest.yml +++ b/packages/aws/manifest.yml @@ -1,7 +1,7 @@ format_version: 3.0.0 name: aws title: AWS -version: 2.15.2 +version: 2.15.3 description: Collect logs and metrics from Amazon Web Services (AWS) with Elastic Agent. type: integration categories: