Fix: MissingContentMD5 due to changed defaults in boto3, added aws_checksum_validation option

karton/core/asyncio/backend.py

@@ -4,6 +4,7 @@
 from typing import IO, Any, Dict, List, Optional, Tuple, Union
 
 import aioboto3
+from aiobotocore.config import AioConfig
 from aiobotocore.credentials import ContainerProvider, InstanceMetadataProvider
 from aiobotocore.session import ClientCreatorContext, get_session
 from aiobotocore.utils import InstanceMetadataFetcher
@@ -50,10 +51,22 @@ def s3(self) -> ClientCreatorContext:
         if not self._s3_session:
             raise RuntimeError("Call connect() first before using KartonAsyncBackend")
         endpoint = self.config.get("s3", "address")
+        if not self.config.getboolean("s3", "aws_checksum_validation"):
+            # This default is more compliant with non-AWS S3 providers
+            # See also:
+            # - https://github.com/boto/boto3/issues/4392
+            # - https://github.com/fsspec/s3fs/issues/931
+            botocore_config = AioConfig(
+                request_checksum_calculation="when_required",
+                response_checksum_validation="when_required",
+            )
+        else:
+            botocore_config = AioConfig()
         if self._s3_iam_auth:
             return self._s3_session.client(
                 "s3",
                 endpoint_url=endpoint,
+                config=botocore_config,
             )
         else:
             access_key = self.config.get("s3", "access_key")
@@ -63,6 +76,7 @@ def s3(self) -> ClientCreatorContext:
                 endpoint_url=endpoint,
                 aws_access_key_id=access_key,
                 aws_secret_access_key=secret_key,
+                config=botocore_config,
             )
 
     async def connect(self):

karton/core/backend.py

@@ -9,6 +9,7 @@
 from typing import IO, Any, Dict, Iterable, Iterator, List, Optional, Set, Tuple, Union
 
 import boto3
+import botocore.config
 from botocore.credentials import (
     ContainerProvider,
     InstanceMetadataFetcher,
@@ -256,6 +257,17 @@ def __init__(
         access_key = config.get("s3", "access_key")
         secret_key = config.get("s3", "secret_key")
         iam_auth = config.getboolean("s3", "iam_auth")
+        if not config.getboolean("s3", "aws_checksum_validation"):
+            # This default is more compliant with non-AWS S3 providers
+            # See also:
+            # - https://github.com/boto/boto3/issues/4392
+            # - https://github.com/fsspec/s3fs/issues/931
+            botocore_config = botocore.config.Config(
+                request_checksum_calculation="when_required",
+                response_checksum_validation="when_required",
+            )
+        else:
+            botocore_config = botocore.config.Config()
 
         if not endpoint:
             raise RuntimeError("Attempting to get S3 client without an endpoint set")
@@ -267,7 +279,7 @@ def __init__(
             )
 
         if iam_auth:
-            s3_client = self.iam_auth_s3(endpoint)
+            s3_client = self.iam_auth_s3(endpoint, botocore_config=botocore_config)
             if s3_client:
                 self.s3 = s3_client
                 return
@@ -282,9 +294,10 @@ def __init__(
             endpoint_url=endpoint,
             aws_access_key_id=access_key,
             aws_secret_access_key=secret_key,
+            config=botocore_config,
         )
 
-    def iam_auth_s3(self, endpoint: str):
+    def iam_auth_s3(self, endpoint: str, botocore_config: botocore.config.Config):
         boto_session = get_session()
         iam_providers = [
             ContainerProvider(),
@@ -300,6 +313,7 @@ def iam_auth_s3(self, endpoint: str):
                 return boto3.Session(botocore_session=boto_session).client(
                     "s3",
                     endpoint_url=endpoint,
+                    config=botocore_config,
                 )
 
     @staticmethod