diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index fb2d643ed606..53fce687243a 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -237,6 +237,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Fix date parsing in GSuite/login fileset. {issue}24694[24694] - Improve Cisco ASA/FTD parsing of messages - better support for identity FW messages. Change network.bytes, source.bytes, and destination.bytes to long from integer since value can exceed integer capacity. Add descriptions for various processors for easier pipeline editing in Kibana UI. {pull}23766[23766] - Improve PanOS parsing and ingest pipeline. {issue}22413[22413] {issue}22748[22748] {pull}24799[24799] +- Fix S3 input validation for non amazonaws.com domains. {issue}24420[24420] {pull}24861[24861] *Heartbeat* diff --git a/x-pack/filebeat/input/awss3/collector.go b/x-pack/filebeat/input/awss3/collector.go index ceec461797fe..1ace01df8823 100644 --- a/x-pack/filebeat/input/awss3/collector.go +++ b/x-pack/filebeat/input/awss3/collector.go @@ -226,14 +226,20 @@ func (c *s3Collector) changeVisibilityTimeout(queueURL string, visibilityTimeout return err } -func getRegionFromQueueURL(queueURL string) (string, error) { +func getRegionFromQueueURL(queueURL string, endpoint string) (string, error) { // get region from queueURL // Example: https://sqs.us-east-1.amazonaws.com/627959692251/test-s3-logs - queueURLSplit := strings.Split(queueURL, ".") - if queueURLSplit[0] == "https://sqs" && queueURLSplit[2] == "amazonaws" { - return queueURLSplit[1], nil + url, err := url.Parse(queueURL) + if err != nil { + return "", fmt.Errorf(queueURL + " is not a valid URL") + } + if url.Scheme == "https" && url.Host != "" { + queueHostSplit := strings.Split(url.Host, ".") + if len(queueHostSplit) > 2 && (strings.Join(queueHostSplit[2:], ".") == endpoint || (endpoint == "" && queueHostSplit[2] == "amazonaws")) { + return queueHostSplit[1], nil + } } - return "", fmt.Errorf("queueURL is not in format: https://sqs.{REGION_ENDPOINT}.amazonaws.com/{ACCOUNT_NUMBER}/{QUEUE_NAME}") + return "", fmt.Errorf("QueueURL is not in format: https://sqs.{REGION_ENDPOINT}.{ENDPOINT}/{ACCOUNT_NUMBER}/{QUEUE_NAME}") } // handle message diff --git a/x-pack/filebeat/input/awss3/collector_test.go b/x-pack/filebeat/input/awss3/collector_test.go index 912b4cf076ae..47313d7c8776 100644 --- a/x-pack/filebeat/input/awss3/collector_test.go +++ b/x-pack/filebeat/input/awss3/collector_test.go @@ -58,10 +58,68 @@ func (m *MockS3Client) GetObjectRequest(input *s3.GetObjectInput) s3.GetObjectRe } func TestGetRegionFromQueueURL(t *testing.T) { - queueURL := "https://sqs.us-east-1.amazonaws.com/627959692251/test-s3-logs" - regionName, err := getRegionFromQueueURL(queueURL) - assert.NoError(t, err) - assert.Equal(t, "us-east-1", regionName) + casesPositive := []struct { + title string + queueURL string + endpoint string + expectedRegion string + }{ + { + "QueueURL using amazonaws.com domain with blank Endpoint", + "https://sqs.us-east-1.amazonaws.com/627959692251/test-s3-logs", + "", + "us-east-1", + }, + { + "QueueURL using abc.xyz and domain with matching Endpoint", + "https://sqs.us-east-1.abc.xyz/627959692251/test-s3-logs", + "abc.xyz", + "us-east-1", + }, + } + + for _, c := range casesPositive { + t.Run(c.title, func(t *testing.T) { + regionName, err := getRegionFromQueueURL(c.queueURL, c.endpoint) + assert.NoError(t, err) + assert.Equal(t, c.expectedRegion, regionName) + }) + } + + casesNegative := []struct { + title string + queueURL string + endpoint string + expectedRegion string + }{ + { + "QueueURL using abc.xyz and domain with blank Endpoint", + "https://sqs.us-east-1.abc.xyz/627959692251/test-s3-logs", + "", + "", + }, + { + "QueueURL using abc.xyz and domain with different Endpoint", + "https://sqs.us-east-1.abc.xyz/627959692251/test-s3-logs", + "googlecloud.com", + "", + }, + { + "QueueURL is an invalid URL", + ":foo", + "", + "", + }, + } + + for _, c := range casesNegative { + t.Run(c.title, func(t *testing.T) { + regionName, err := getRegionFromQueueURL(c.queueURL, c.endpoint) + assert.Error(t, err) + assert.Empty(t, regionName) + }) + } + } func TestHandleMessage(t *testing.T) { diff --git a/x-pack/filebeat/input/awss3/input.go b/x-pack/filebeat/input/awss3/input.go index ccbe105974d8..bdb5976bf2da 100644 --- a/x-pack/filebeat/input/awss3/input.go +++ b/x-pack/filebeat/input/awss3/input.go @@ -87,7 +87,7 @@ func (in *s3Input) createCollector(ctx v2.Context, pipeline beat.Pipeline) (*s3C return nil, err } - regionName, err := getRegionFromQueueURL(in.config.QueueURL) + regionName, err := getRegionFromQueueURL(in.config.QueueURL, in.config.AwsConfig.Endpoint) if err != nil { err := fmt.Errorf("getRegionFromQueueURL failed: %w", err) log.Error(err)