From abb72ab4c2f6134a8022e6727208bd684f60984c Mon Sep 17 00:00:00 2001 From: Thomas Parslow Date: Fri, 26 Nov 2021 12:19:46 +0000 Subject: [PATCH] Fix memory leak Creating a new session each time S3Storage is instantiated creates a memory leak. It seems that S3Storage can get created a bunch of times (I'm seeing it get created again and again as my app runs) and a boto3's Session takes loads of memory (see https://github.com/boto/boto3/issues/1670) so my app eventually runs out of memory. This should fix the issue while still avoiding using the same session across different threads. --- django_s3_storage/storage.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/django_s3_storage/storage.py b/django_s3_storage/storage.py index ac81a57..847f7d0 100644 --- a/django_s3_storage/storage.py +++ b/django_s3_storage/storage.py @@ -93,10 +93,17 @@ def open(self, mode="rb"): if self.closed: self.file = self._storage.open(self.name, mode).file return super(S3File, self).open(mode) + + +class _LocalSession(local): + def __init__(self): + self.session = boto3.session.Session() + + +local_session = _LocalSession() class _Local(local): - """ Thread-local connection manager. @@ -116,8 +123,7 @@ def __init__(self, storage): connection_kwargs["aws_session_token"] = storage.settings.AWS_SESSION_TOKEN if storage.settings.AWS_S3_ENDPOINT_URL: connection_kwargs["endpoint_url"] = storage.settings.AWS_S3_ENDPOINT_URL - self.session = boto3.session.Session() - self.s3_connection = self.session.client("s3", config=Config( + self.s3_connection = local_session.session.client("s3", config=Config( s3={"addressing_style": storage.settings.AWS_S3_ADDRESSING_STYLE}, signature_version=storage.settings.AWS_S3_SIGNATURE_VERSION, ), **connection_kwargs)