Add S3 input to retrieve logs from AWS S3 buckets

CHANGELOG.next.asciidoc

@@ -250,6 +250,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Parse more fields from Elasticsearch slowlogs. {pull}11939[11939]
 - Update module pipelines to enrich events with autonomous system fields. {pull}13036[13036]
 - Add module for ingesting IBM MQ logs. {pull}8782[8782]
+- Add S3 input to retrieve logs from AWS S3 buckets. {pull}12640[12640] {issue}12582[12582]
 
 *Heartbeat*
 

filebeat/docs/fields.asciidoc

@@ -14,6 +14,7 @@ grouped in the following categories:
 
 * <<exported-fields-apache>>
 * <<exported-fields-auditd>>
+* <<exported-fields-aws>>
 * <<exported-fields-beat-common>>
 * <<exported-fields-cisco>>
 * <<exported-fields-cloud>>
@@ -986,6 +987,40 @@ alias to: destination.address
 
 --
 
+[[exported-fields-aws]]
+== aws fields
+
+AWS fields from s3 input.
+
+
+
+[float]
+=== s3
+
+S3 related information.
+
+
+
+*`s3.bucket_name`*::
++
+--
+Name of the S3 bucket that this log retrieved from.
+
+
+type: keyword
+
+--
+
+*`s3.object_key`*::
++
+--
+Name of the S3 object that this log retrieved from.
+
+
+type: keyword
+
+--
+
 [[exported-fields-beat-common]]
 == Beat fields
 

filebeat/docs/filebeat-options.asciidoc

@@ -50,6 +50,7 @@ You can configure {beatname_uc} to use the following inputs:
 * <<{beatname_lc}-input-docker>>
 * <<{beatname_lc}-input-tcp>>
 * <<{beatname_lc}-input-syslog>>
+* <<{beatname_lc}-input-s3>>
 * <<{beatname_lc}-input-netflow>>
 * <<{beatname_lc}-input-google-pubsub>>
 
@@ -70,6 +71,8 @@ include::inputs/input-tcp.asciidoc[]
 
 include::inputs/input-syslog.asciidoc[]
 
+include::../../x-pack/filebeat/docs/inputs/input-aws-s3.asciidoc[]
+
 include::../../x-pack/filebeat/docs/inputs/input-netflow.asciidoc[]
 
 include::../../x-pack/filebeat/docs/inputs/input-google-pubsub.asciidoc[]

metricbeat/docs/modules/aws.asciidoc

@@ -88,7 +88,7 @@ for a list of AWS services that publish metrics to CloudWatch.
 dimensions to Amazon CloudWatch every minute.
 
 [id="aws-credentials-config"]
-include::{libbeat-xpack-dir}/docs/aws-credentials-config.asciidoc[]
+include::../../../../../metricbeat/docs/aws-credentials-examples.asciidoc
 
 
 [float]

vendor/vendor.json

@@ -252,6 +252,7 @@
 			"path": "github.com/aws/aws-sdk-go-v2/aws/awserr",
 			"revision": "098e15df3044cf1b04a222c1c33c3e6135ac89f3",
 			"revisionTime": "2019-05-28T21:51:27Z",
+			"tree": true,
 			"version": "v0.9.0",
 			"versionExact": "v0.9.0"
 		},
@@ -512,7 +513,15 @@
 			"versionExact": "v0.9.0"
 		},
 		{
-			"checksumSHA1": "VOdJIdzSIvHiu23lwsxFOTeBrWk=",
+			"checksumSHA1": "e0qV+W56A3/QgA1xDkaAwTIffAw=",
+			"path": "github.com/aws/aws-sdk-go-v2/service/s3/s3iface",
+			"revision": "098e15df3044cf1b04a222c1c33c3e6135ac89f3",
+			"revisionTime": "2019-05-28T21:51:27Z",
+			"version": "v0.9.0",
+			"versionExact": "v0.9.0"
+		},
+		{
+			"checksumSHA1": "Y+mpKnu57hYbC4z3nrSFR9BkAec=",
 			"path": "github.com/aws/aws-sdk-go-v2/service/sqs",
 			"revision": "098e15df3044cf1b04a222c1c33c3e6135ac89f3",
 			"revisionTime": "2019-05-28T21:51:27Z",

x-pack/filebeat/_meta/common.reference.inputs.yml

@@ -53,3 +53,23 @@
 
   # Path to a JSON file containing the credentials and key used to subscribe.
   credentials_file: ${path.config}/my-pubsub-subscriber-credentials.json
+
+#------------------------------ S3 input --------------------------------
+# Beta: Config options for AWS S3 input
+#- type: s3
+  #enabled: false
+
+  # AWS Credentials
+  # If access_key_id and secret_access_key are configured, then use them to make api calls.
+  # If not, s3 input will load default AWS config or load with given profile name.
+  #access_key_id: '${AWS_ACCESS_KEY_ID:""}'
+  #secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}'
+  #session_token: '${AWS_SESSION_TOKEN:"”}'
+  #credential_profile_name: test-s3-input
+
+  # Queue urls (required) to receive queue messages from
+  #queue_urls: ["https://sqs.us-east-1.amazonaws.com/1234/test-s3-logs-queue"]
+
+  # The duration (in seconds) that the received messages are hidden from subsequent
+  # retrieve requests after being retrieved by a ReceiveMessage request.
+  #visibility_timeout: 300