From 86c946b7c4fa84eaa7ccd9e926a31811c7b4481c Mon Sep 17 00:00:00 2001 From: Milen Pankov Date: Thu, 12 Dec 2024 09:56:30 +0200 Subject: [PATCH 1/3] lambda-promtail: remove s3 bucket ownership verification --- tools/lambda-promtail/lambda-promtail/s3.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tools/lambda-promtail/lambda-promtail/s3.go b/tools/lambda-promtail/lambda-promtail/s3.go index a6255811378dd..14949ebc1b3eb 100644 --- a/tools/lambda-promtail/lambda-promtail/s3.go +++ b/tools/lambda-promtail/lambda-promtail/s3.go @@ -252,7 +252,6 @@ func getLabels(record events.S3EventRecord) (map[string]string, error) { labels["key"] = record.S3.Object.Key labels["bucket"] = record.S3.Bucket.Name - labels["bucket_owner"] = record.S3.Bucket.OwnerIdentity.PrincipalID labels["bucket_region"] = record.AWSRegion for key, p := range parsers { if p.filenameRegex.MatchString(labels["key"]) { @@ -292,10 +291,9 @@ func processS3Event(ctx context.Context, ev *events.S3Event, pc Client, log *log &s3.GetObjectInput{ Bucket: aws.String(labels["bucket"]), Key: aws.String(labels["key"]), - ExpectedBucketOwner: aws.String(labels["bucket_owner"]), }) if err != nil { - return fmt.Errorf("failed to get object %s from bucket %s on account %s, %s", labels["key"], labels["bucket"], labels["bucket_owner"], err) + return fmt.Errorf("failed to get object %s from bucket %s on account %s, %s", labels["key"], labels["bucket"], err) } err = parseS3Log(ctx, batch, labels, obj.Body, log) if err != nil { From b71d832e523f149b48a4d1121bdbf2e3b3a7ca52 Mon Sep 17 00:00:00 2001 From: Milen Pankov <123647381+milenpankov-bosch@users.noreply.github.com> Date: Fri, 13 Dec 2024 14:05:33 +0200 Subject: [PATCH 2/3] Update tools/lambda-promtail/lambda-promtail/s3.go Co-authored-by: Christian Haudum --- tools/lambda-promtail/lambda-promtail/s3.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/lambda-promtail/lambda-promtail/s3.go b/tools/lambda-promtail/lambda-promtail/s3.go index 14949ebc1b3eb..0edeeb853d05a 100644 --- a/tools/lambda-promtail/lambda-promtail/s3.go +++ b/tools/lambda-promtail/lambda-promtail/s3.go @@ -293,7 +293,7 @@ func processS3Event(ctx context.Context, ev *events.S3Event, pc Client, log *log Key: aws.String(labels["key"]), }) if err != nil { - return fmt.Errorf("failed to get object %s from bucket %s on account %s, %s", labels["key"], labels["bucket"], err) + return fmt.Errorf("failed to get object %s from bucket %s, %s", labels["key"], labels["bucket"], err) } err = parseS3Log(ctx, batch, labels, obj.Body, log) if err != nil { From ad0ec5ce2156b758e47f6da971b05728341463c6 Mon Sep 17 00:00:00 2001 From: Milen Pankov Date: Fri, 13 Dec 2024 14:43:15 +0200 Subject: [PATCH 3/3] lambda-promtail: remove s3 bucket_owner label from tests --- .../lambda-promtail/s3_test.go | 53 ------------------- 1 file changed, 53 deletions(-) diff --git a/tools/lambda-promtail/lambda-promtail/s3_test.go b/tools/lambda-promtail/lambda-promtail/s3_test.go index 00c58d7ecfb2c..8ff569e9bf5d9 100644 --- a/tools/lambda-promtail/lambda-promtail/s3_test.go +++ b/tools/lambda-promtail/lambda-promtail/s3_test.go @@ -33,9 +33,6 @@ func Test_getLabels(t *testing.T) { S3: events.S3Entity{ Bucket: events.S3Bucket{ Name: "elb_logs_test", - OwnerIdentity: events.S3UserIdentity{ - PrincipalID: "test", - }, }, Object: events.S3Object{ Key: "my-bucket/AWSLogs/123456789012/elasticloadbalancing/us-east-1/2022/01/24/123456789012_elasticloadbalancing_us-east-1_app.my-loadbalancer.b13ea9d19f16d015_20220124T0000Z_0.0.0.0_2et2e1mx.log.gz", @@ -46,7 +43,6 @@ func Test_getLabels(t *testing.T) { want: map[string]string{ "account_id": "123456789012", "bucket": "elb_logs_test", - "bucket_owner": "test", "bucket_region": "us-east-1", "day": "24", "key": "my-bucket/AWSLogs/123456789012/elasticloadbalancing/us-east-1/2022/01/24/123456789012_elasticloadbalancing_us-east-1_app.my-loadbalancer.b13ea9d19f16d015_20220124T0000Z_0.0.0.0_2et2e1mx.log.gz", @@ -67,9 +63,6 @@ func Test_getLabels(t *testing.T) { S3: events.S3Entity{ Bucket: events.S3Bucket{ Name: "elb_logs_test", - OwnerIdentity: events.S3UserIdentity{ - PrincipalID: "test", - }, }, Object: events.S3Object{ Key: "my-bucket/prefix/AWSLogs/123456789012/elasticloadbalancing/us-east-2/2016/05/01/123456789012_elasticloadbalancing_us-east-2_net.my-loadbalancer.1234567890abcdef_201605010000Z_2soosksi.log.gz", @@ -80,7 +73,6 @@ func Test_getLabels(t *testing.T) { want: map[string]string{ "account_id": "123456789012", "bucket": "elb_logs_test", - "bucket_owner": "test", "bucket_region": "us-east-2", "day": "01", "key": "my-bucket/prefix/AWSLogs/123456789012/elasticloadbalancing/us-east-2/2016/05/01/123456789012_elasticloadbalancing_us-east-2_net.my-loadbalancer.1234567890abcdef_201605010000Z_2soosksi.log.gz", @@ -101,9 +93,6 @@ func Test_getLabels(t *testing.T) { S3: events.S3Entity{ Bucket: events.S3Bucket{ Name: "s3_guardduty_test", - OwnerIdentity: events.S3UserIdentity{ - PrincipalID: "test", - }, }, Object: events.S3Object{ Key: "AWSLogs/123456789012/GuardDuty/us-east-1/2024/05/30/07a3f2ce-1485-3031-b842-e1f324c4a48d.jsonl.gz", @@ -114,7 +103,6 @@ func Test_getLabels(t *testing.T) { want: map[string]string{ "account_id": "123456789012", "bucket": "s3_guardduty_test", - "bucket_owner": "test", "bucket_region": "us-east-1", "day": "30", "key": "AWSLogs/123456789012/GuardDuty/us-east-1/2024/05/30/07a3f2ce-1485-3031-b842-e1f324c4a48d.jsonl.gz", @@ -133,9 +121,6 @@ func Test_getLabels(t *testing.T) { S3: events.S3Entity{ Bucket: events.S3Bucket{ Name: "vpc_logs_test", - OwnerIdentity: events.S3UserIdentity{ - PrincipalID: "test", - }, }, Object: events.S3Object{ Key: "my-bucket/AWSLogs/123456789012/vpcflowlogs/us-east-1/2022/01/24/123456789012_vpcflowlogs_us-east-1_fl-1234abcd_20180620T1620Z_fe123456.log.gz", @@ -146,7 +131,6 @@ func Test_getLabels(t *testing.T) { want: map[string]string{ "account_id": "123456789012", "bucket": "vpc_logs_test", - "bucket_owner": "test", "bucket_region": "us-east-1", "day": "24", "key": "my-bucket/AWSLogs/123456789012/vpcflowlogs/us-east-1/2022/01/24/123456789012_vpcflowlogs_us-east-1_fl-1234abcd_20180620T1620Z_fe123456.log.gz", @@ -166,9 +150,6 @@ func Test_getLabels(t *testing.T) { S3: events.S3Entity{ Bucket: events.S3Bucket{ Name: "vpc_logs_test", - OwnerIdentity: events.S3UserIdentity{ - PrincipalID: "test", - }, }, Object: events.S3Object{ Key: "my-bucket/AWSLogs/123456789012/vpcflowlogs/us-gov-east-1/2022/01/24/123456789012_vpcflowlogs_us-gov-east-1_fl-1234abcd_20180620T1620Z_fe123456.log.gz", @@ -179,7 +160,6 @@ func Test_getLabels(t *testing.T) { want: map[string]string{ "account_id": "123456789012", "bucket": "vpc_logs_test", - "bucket_owner": "test", "bucket_region": "us-gov-east-1", "day": "24", "key": "my-bucket/AWSLogs/123456789012/vpcflowlogs/us-gov-east-1/2022/01/24/123456789012_vpcflowlogs_us-gov-east-1_fl-1234abcd_20180620T1620Z_fe123456.log.gz", @@ -199,9 +179,6 @@ func Test_getLabels(t *testing.T) { S3: events.S3Entity{ Bucket: events.S3Bucket{ Name: "cloudtrail_digest_logs_test", - OwnerIdentity: events.S3UserIdentity{ - PrincipalID: "test", - }, }, Object: events.S3Object{ Key: "my-bucket/AWSLogs/123456789012/CloudTrail-Digest/us-east-1/2022/01/24/123456789012_CloudTrail-Digest_us-east-1_20220124T0000Z_4jhzXFO2Jlvu2b3y.json.gz", @@ -212,7 +189,6 @@ func Test_getLabels(t *testing.T) { want: map[string]string{ "account_id": "123456789012", "bucket": "cloudtrail_digest_logs_test", - "bucket_owner": "test", "bucket_region": "us-east-1", "day": "24", "key": "my-bucket/AWSLogs/123456789012/CloudTrail-Digest/us-east-1/2022/01/24/123456789012_CloudTrail-Digest_us-east-1_20220124T0000Z_4jhzXFO2Jlvu2b3y.json.gz", @@ -232,9 +208,6 @@ func Test_getLabels(t *testing.T) { S3: events.S3Entity{ Bucket: events.S3Bucket{ Name: "cloudtrail_logs_test", - OwnerIdentity: events.S3UserIdentity{ - PrincipalID: "test", - }, }, Object: events.S3Object{ Key: "my-bucket/AWSLogs/123456789012/CloudTrail/us-east-1/2022/01/24/123456789012_CloudTrail_us-east-1_20220124T0000Z_4jhzXFO2Jlvu2b3y.json.gz", @@ -245,7 +218,6 @@ func Test_getLabels(t *testing.T) { want: map[string]string{ "account_id": "123456789012", "bucket": "cloudtrail_logs_test", - "bucket_owner": "test", "bucket_region": "us-east-1", "day": "24", "key": "my-bucket/AWSLogs/123456789012/CloudTrail/us-east-1/2022/01/24/123456789012_CloudTrail_us-east-1_20220124T0000Z_4jhzXFO2Jlvu2b3y.json.gz", @@ -265,9 +237,6 @@ func Test_getLabels(t *testing.T) { S3: events.S3Entity{ Bucket: events.S3Bucket{ Name: "cloudtrail_logs_test", - OwnerIdentity: events.S3UserIdentity{ - PrincipalID: "test", - }, }, Object: events.S3Object{ Key: "my-bucket/AWSLogs/123456789012/CloudTrail/us-gov-east-1/2022/01/24/123456789012_CloudTrail_us-gov-east-1_20220124T0000Z_4jhzXFO2Jlvu2b3y.json.gz", @@ -278,7 +247,6 @@ func Test_getLabels(t *testing.T) { want: map[string]string{ "account_id": "123456789012", "bucket": "cloudtrail_logs_test", - "bucket_owner": "test", "bucket_region": "us-gov-east-1", "day": "24", "key": "my-bucket/AWSLogs/123456789012/CloudTrail/us-gov-east-1/2022/01/24/123456789012_CloudTrail_us-gov-east-1_20220124T0000Z_4jhzXFO2Jlvu2b3y.json.gz", @@ -298,9 +266,6 @@ func Test_getLabels(t *testing.T) { S3: events.S3Entity{ Bucket: events.S3Bucket{ Name: "cloudtrail_logs_test", - OwnerIdentity: events.S3UserIdentity{ - PrincipalID: "test", - }, }, Object: events.S3Object{ Key: "my-bucket/AWSLogs/o-test123456/123456789012/CloudTrail/us-east-1/2022/01/24/123456789012_CloudTrail_us-east-1_20220124T0000Z_4jhzXFO2Jlvu2b3y.json.gz", @@ -311,7 +276,6 @@ func Test_getLabels(t *testing.T) { want: map[string]string{ "account_id": "123456789012", "bucket": "cloudtrail_logs_test", - "bucket_owner": "test", "bucket_region": "us-east-1", "day": "24", "key": "my-bucket/AWSLogs/o-test123456/123456789012/CloudTrail/us-east-1/2022/01/24/123456789012_CloudTrail_us-east-1_20220124T0000Z_4jhzXFO2Jlvu2b3y.json.gz", @@ -332,9 +296,6 @@ func Test_getLabels(t *testing.T) { S3: events.S3Entity{ Bucket: events.S3Bucket{ Name: "cloudfront_logs_test", - OwnerIdentity: events.S3UserIdentity{ - PrincipalID: "test", - }, }, Object: events.S3Object{ Key: "my/bucket/prefix/E2K2LNL5N3WR51.2022-07-18-12.a10a8496.gz", @@ -344,7 +305,6 @@ func Test_getLabels(t *testing.T) { }, want: map[string]string{ "bucket": "cloudfront_logs_test", - "bucket_owner": "test", "bucket_region": "us-east-1", "day": "18", "key": "my/bucket/prefix/E2K2LNL5N3WR51.2022-07-18-12.a10a8496.gz", @@ -364,9 +324,6 @@ func Test_getLabels(t *testing.T) { S3: events.S3Entity{ Bucket: events.S3Bucket{ Name: "waf_logs_test", - OwnerIdentity: events.S3UserIdentity{ - PrincipalID: "test", - }, }, Object: events.S3Object{ Key: "prefix/AWSLogs/11111111111/WAFLogs/us-east-1/TEST-WEBACL/2021/10/28/19/50/11111111111_waflogs_us-east-1_TEST-WEBACL_20211028T1950Z_e0ca43b5.log.gz", @@ -376,7 +333,6 @@ func Test_getLabels(t *testing.T) { }, want: map[string]string{ "account_id": "11111111111", - "bucket_owner": "test", "bucket_region": "us-east-1", "bucket": "waf_logs_test", "day": "28", @@ -399,9 +355,6 @@ func Test_getLabels(t *testing.T) { S3: events.S3Entity{ Bucket: events.S3Bucket{ Name: "waf_logs_test", - OwnerIdentity: events.S3UserIdentity{ - PrincipalID: "test", - }, }, Object: events.S3Object{ Key: "prefix/AWSLogs/11111111111/WAFLogs/us-gov-east-1/TEST-WEBACL/2021/10/28/19/50/11111111111_waflogs_us-gov-east-1_TEST-WEBACL_20211028T1950Z_e0ca43b5.log.gz", @@ -411,7 +364,6 @@ func Test_getLabels(t *testing.T) { }, want: map[string]string{ "account_id": "11111111111", - "bucket_owner": "test", "bucket_region": "us-gov-east-1", "bucket": "waf_logs_test", "day": "28", @@ -434,9 +386,6 @@ func Test_getLabels(t *testing.T) { S3: events.S3Entity{ Bucket: events.S3Bucket{ Name: "missing_type", - OwnerIdentity: events.S3UserIdentity{ - PrincipalID: "test", - }, }, Object: events.S3Object{ Key: "some-object.json", @@ -446,7 +395,6 @@ func Test_getLabels(t *testing.T) { }, want: map[string]string{ "bucket": "missing_type", - "bucket_owner": "test", "bucket_region": "us-east-1", "key": "some-object.json", }, @@ -644,7 +592,6 @@ func Test_parseS3Log(t *testing.T) { }, labels: map[string]string{ "bucket": "missing_parser", - "bucket_owner": "test", "bucket_region": "us-east-1", "key": "some-object.json", "type": "",