diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md index 55a966b..5834e60 100644 --- a/THIRD_PARTY_NOTICES.md +++ b/THIRD_PARTY_NOTICES.md @@ -1,31 +1,14 @@ # Third Party Notices The aws_s3_log_ingestion_lambda uses source code from third party libraries which carry their own copyright notices and license terms. These notices are provided below. - the event that a required notice is missing or incorrect, please notify us by e-mailing open-source@newrelic.com. -r any licenses that require the disclosure of source code, the source code can be found at https://github.com/newrelic/aws_s3_log_ingestion_lambda. +In the event that a required notice is missing or incorrect, please notify us by e-mailing open-source@newrelic.com. +For any licenses that require the disclosure of source code, the source code can be found at https://github.com/newrelic/aws_s3_log_ingestion_lambda. # Content **[dependencies](#dependencies)** # Dependencies | Name | Version | License | |-------------------|------------|--------------------------------------------------------------| -| Pympler | 0.8 | Apache License, Version 2.0 | | aiohttp | 3.6.2 | Apache 2 | -| async-timeout | 3.0.1 | Apache 2 | -| attrs | 19.3.0 | MIT | -| boto | 2.49.0 | MIT | | boto3 | 1.14.40 | Apache License 2.0 | -| botocore | 1.17.40 | Apache License 2.0 | -| certifi | 2020.6.20 | MPL-2.0 | -| chardet | 3.0.4 | LGPL | -| docutils | 0.15.2 | public domain, Python, 2-Clause BSD, GPL 3 (see COPYING.txt) | -| idna | 2.10 | BSD-like | -| jmespath | 0.10.0 | MIT | -| multidict | 4.7.6 | Apache 2 | | python-dateutil | 2.8.1 | Dual License | -| requests | 2.24.0 | Apache 2.0 | -| s3transfer | 0.3.3 | Apache License 2.0 | -| six | 1.15.0 | MIT | | smart-open | 2.1.0 | MIT | -| typing-extensions | 3.7.4.2 | GPL-compatible, see: https://docs.python.org/3/license.html | -| urllib3 | 1.25.10 | MIT | -| yarl | 1.5.1 | Apache 2 | diff --git a/src/handler.py b/src/handler.py index 4cdb27f..e173708 100644 --- a/src/handler.py +++ b/src/handler.py @@ -1,8 +1,8 @@ import json -import sys +from sys import getsizeof import urllib.parse import boto3 -import gzip +from gzip import compress import os from urllib import request import aiohttp @@ -10,7 +10,7 @@ import time import logging from smart_open import open -import re +from re import search from dateutil import parser @@ -95,7 +95,7 @@ def _is_ignore_log_file(key=None, regex_pattern=None): if not regex_pattern: regex_pattern = _get_optional_env("S3_IGNORE_PATTERN", "$^") - return bool(re.search(regex_pattern, key)) + return bool(search(regex_pattern, key)) def _isCloudTrail(key=None, regex_pattern=None): @@ -106,7 +106,7 @@ def _isCloudTrail(key=None, regex_pattern=None): regex_pattern = _get_optional_env( "S3_CLOUDTRAIL_LOG_PATTERN", ".*CloudTrail.*\.json.gz$") - return bool(re.search(regex_pattern, key)) + return bool(search(regex_pattern, key)) def _convert_float(s): try: @@ -173,9 +173,9 @@ def _compress_payload(data): This method usually returns a list of one element, but can be bigger if the payload size is too big """ - logger.debug(f"uncompressed size: {sys.getsizeof(json.dumps(data).encode())}") - payload = gzip.compress(json.dumps(data).encode()) - logger.debug(f"compressed size: {sys.getsizeof(payload)}") + logger.debug(f"uncompressed size: {getsizeof(json.dumps(data).encode())}") + payload = compress(json.dumps(data).encode()) + logger.debug(f"compressed size: {getsizeof(payload)}") return payload @@ -300,7 +300,7 @@ async def _fetch_data_from_s3(bucket, key, context): log_lines = cloudtrail_events for index, log in enumerate(log_lines): - log_batch_size += sys.getsizeof(str(log)) + log_batch_size += getsizeof(str(log)) if index % 500 == 0: logger.debug(f"index: {index}") logger.debug(f"log_batch_size: {log_batch_size}") diff --git a/src/requirements.txt b/src/requirements.txt index f2053b4..aba5040 100644 --- a/src/requirements.txt +++ b/src/requirements.txt @@ -1,20 +1,4 @@ aiohttp==3.8.4 -async-timeout==4.0.2 -attrs==19.3.0 -boto==2.49.0 boto3==1.24.59 -botocore==1.27.59 -certifi==2022.12.7 -chardet==3.0.4 -docutils==0.15.2 -idna==2.10 -jmespath==0.10.0 -multidict==4.7.6 python-dateutil==2.8.1 -requests==2.28.1 -s3transfer==0.6.0 -six==1.15.0 -smart-open==2.1.0 -typing-extensions==3.7.4.2 -urllib3==1.26.15 -yarl==1.5.1 +smart-open==6.3.0 \ No newline at end of file