feat(logs): AWSX-1389 Moving s3 access logs identification to backend

aws/logs_monitoring/steps/handlers/s3_handler.py

@@ -57,7 +57,6 @@ def handle(self, event):
         event = self._extract_event(event)
         self._set_source(event)
         add_service_tag(self.metadata)
-        self._set_host()
         self._add_s3_tags_from_cache()
         self._extract_data()
         yield from self._get_structured_lines_for_s3_handler()
@@ -83,18 +82,6 @@ def _set_source(self, event):
             self.data_store.source = str(AwsEventSource.TRANSITGATEWAY)
         self.metadata[DD_SOURCE] = self.data_store.source
 
-    def _set_host(self):
-        hostname = self._parse_service_arn()
-        if hostname:
-            self.metadata[DD_HOST] = hostname
-
-    def _parse_service_arn(self):
-        src = AwsEventSource._value2member_map_.get(self.data_store.source)
-        match src:
-            case AwsEventSource.S3:
-                # For S3 access logs we use the bucket name to rebuild the arn
-                return self._get_s3_arn()
-
     def _get_s3_arn(self):
         if not self.data_store.bucket:
             return None

aws/logs_monitoring/tests/test_cloudtrail_s3.py

@@ -1,11 +1,15 @@
-from unittest.mock import MagicMock, patch
+import copy
+import gzip
+import io
+import json
 import os
 import sys
 import unittest
-import json
-import copy
-import io
-import gzip
+from unittest.mock import MagicMock, patch
+
+import lambda_function
+from caching.cache_layer import CacheLayer
+from steps.parsing import parse
 
 sys.modules["trace_forwarder.connection"] = MagicMock()
 sys.modules["datadog_lambda.wrapper"] = MagicMock()
@@ -22,10 +26,6 @@
     },
 )
 env_patch.start()
-import lambda_function
-from steps.parsing import parse
-from caching.cache_layer import CacheLayer
-
 env_patch.stop()
 
 

aws/logs_monitoring/tests/test_customized_log_group.py

@@ -1,7 +1,8 @@
 import unittest
+
 from customized_log_group import (
-    is_lambda_customized_log_group,
     get_lambda_function_name_from_logstream_name,
+    is_lambda_customized_log_group,
     is_step_functions_log_group,
 )
 

aws/logs_monitoring/tests/test_s3_handler.py

@@ -140,7 +140,6 @@ def test_s3_handler(self):
             ],
         )
         self.assertEqual(self.s3_handler.metadata["ddsource"], "s3")
-        self.assertEqual(self.s3_handler.metadata["host"], "arn:aws:s3:::my-bucket")
 
     def test_s3_handler_with_multiline_regex(self):
         event = {
@@ -207,36 +206,12 @@ def test_s3_handler_with_sns(self):
             ],
         )
         self.assertEqual(self.s3_handler.metadata["ddsource"], "s3")
-        self.assertEqual(self.s3_handler.metadata["host"], "arn:aws:s3:::my-bucket")
-
-    @patch("steps.handlers.s3_handler.S3EventHandler._get_s3_client")
-    def test_s3_tags_not_added_to_metadata(self, mock_get_s3_client):
-        mock_get_s3_client.side_effect = MagicMock()
-        cache_layer = CacheLayer("")
-        cache_layer._s3_tags_cache.get = MagicMock(return_value=["s3_tag:tag_value"])
-        self.s3_handler.cache_layer = cache_layer
-        event = {
-            "Records": [
-                {
-                    "s3": {
-                        "bucket": {"name": "mybucket"},
-                        "object": {"key": "mykey"},
-                    }
-                }
-            ]
-        }
-
-        _ = list(self.s3_handler.handle(event))
-
-        assert "s3_tag:tag_value" not in self.s3_handler.metadata["ddtags"]
 
     @patch("caching.cloudwatch_log_group_cache.CloudwatchLogGroupTagsCache.__init__")
-    @patch("steps.handlers.s3_handler.S3EventHandler._parse_service_arn")
     @patch("steps.handlers.s3_handler.S3EventHandler._get_s3_client")
     def test_s3_tags_added_to_metadata(
         self,
         mock_get_s3_client,
-        mock_parse_service_arn,
         mock_cache_init,
     ):
         mock_get_s3_client.side_effect = MagicMock()
@@ -255,7 +230,6 @@ def test_s3_tags_added_to_metadata(
             ]
         }
 
-        mock_parse_service_arn.return_value = ""
         _ = list(self.s3_handler.handle(event))
 
         assert "s3_tag:tag_value" in self.s3_handler.metadata["ddtags"]