Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 12 additions & 1 deletion botocore/args.py
Original file line number Diff line number Diff line change
Expand Up @@ -170,8 +170,19 @@ def get_client_args(
proxies_config=new_config.proxies_config,
)

serializer_kwargs = {'timestamp_precision': 'second'}
event_name = f'creating-serializer.{service_name}'
event_emitter.emit(
event_name,
protocol_name=protocol,
service_model=service_model,
serializer_kwargs=serializer_kwargs,
)

serializer = botocore.serialize.create_serializer(
protocol, parameter_validation
protocol,
parameter_validation,
timestamp_precision=serializer_kwargs['timestamp_precision'],
)
response_parser = botocore.parsers.create_parser(protocol)

Expand Down
9 changes: 9 additions & 0 deletions botocore/handlers.py
Original file line number Diff line number Diff line change
Expand Up @@ -1069,6 +1069,11 @@ def remove_bedrock_runtime_invoke_model_with_bidirectional_stream(
del class_attributes['invoke_model_with_bidirectional_stream']


def customize_bedrock_agentcore_serializer(serializer_kwargs, **kwargs):
"""Event handler to enable millisecond precision for bedrock-agentcore."""
serializer_kwargs['timestamp_precision'] = 'millisecond'


def add_retry_headers(request, **kwargs):
retries_context = request.context.get('retries')
if not retries_context:
Expand Down Expand Up @@ -1471,6 +1476,10 @@ def get_bearer_auth_supported_services():
'creating-client-class.bedrock-runtime',
remove_bedrock_runtime_invoke_model_with_bidirectional_stream,
),
(
'creating-serializer.bedrock-agentcore',
customize_bedrock_agentcore_serializer,
),
('after-call.iam', json_decode_policies),
('after-call.ec2.GetConsoleOutput', decode_console_output),
('after-call.cloudformation.GetTemplate', json_decode_template_body),
Expand Down
52 changes: 44 additions & 8 deletions botocore/serialize.py
Original file line number Diff line number Diff line change
Expand Up @@ -67,9 +67,22 @@
HOST_PREFIX_RE = re.compile(r"^[A-Za-z0-9\.\-]+$")


def create_serializer(protocol_name, include_validation=True):
def create_serializer(
protocol_name, include_validation=True, timestamp_precision='second'
):
"""Create a serializer for the given protocol.

:param protocol_name: The protocol name to create a serializer for.
:param include_validation: Whether to include parameter validation.
:param timestamp_precision: Timestamp precision level.
- 'second': Standard second precision (default)
- 'millisecond': Millisecond precision for high-accuracy timestamps
:return: A serializer instance for the given protocol.
"""
# TODO: Unknown protocols.
serializer = SERIALIZERS[protocol_name]()
serializer = SERIALIZERS[protocol_name](
timestamp_precision=timestamp_precision
)
if include_validation:
validator = validate.ParamValidator()
serializer = validate.ParamValidationDecorator(validator, serializer)
Expand All @@ -85,6 +98,11 @@ class Serializer:
MAP_TYPE = dict
DEFAULT_ENCODING = 'utf-8'

def __init__(self, timestamp_precision='second'):
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is it possible to pass timestamp_precision to the serialize_to_request method instead? I think it would be a bit cleaner if serializer objects don't need to be tied to a specific timestamp_precision value. I haven't looked into if it's possible to grab the value at the method call sites and I don't feel super strongly about it so feel free to merge if we think this is the best place

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I like that idea better in theory, but it would be a hackey shim. Right now our serialization logic just passes in the value and shape. At this point in the code, we have access to the following (assuming JSON parser):

  • self (the whole serializer object) - storing something here wouldn't be an option for setting it on a method level due to race conditions that could occur
  • The "key" that it will be serialized to
  • The modeled shape
  • The DateTime object that we are serializing

That leaves us with two options:

  1. Add this to the operation model somehow, probably using service-2.sdk-extras.json or some sort of model pre-processing. That would get pretty difficult to maintain as these services grow
  2. Find some way to pass around an operation_context of sorts that contains it, and add the precision to the context in serialize_to_request.
    2a. We could pass around a dictionary to these methods, which gets messy since these are generally inside a structure (maybe nested a few layers), so we have to make sure that persists.
    2b. contextvars would be the other solution I could think of here, but I'd be concerned about behavior if for some reason the context var didn't exist. Maybe that's a non-issue since we would have just created the context in the serialize_to_request method.

Based on all of this, do you have a preferred solution? I'm open to chat here, but right now I'd say we should either go with the current solution or 2b.

if timestamp_precision is None:
timestamp_precision = 'second'
self._timestamp_precision = timestamp_precision

def serialize_to_request(self, parameters, operation_model):
"""Serialize parameters into an HTTP request.

Expand Down Expand Up @@ -139,18 +157,36 @@ def _create_default_request(self):
# Some extra utility methods subclasses can use.

def _timestamp_iso8601(self, value):
if value.microsecond > 0:
timestamp_format = ISO8601_MICRO
"""Return ISO8601 timestamp with precision based on timestamp_precision."""
# Smithy's standard is milliseconds, so we truncate the timestamp if the millisecond flag is set to true
if self._timestamp_precision == 'millisecond':
milliseconds = value.microsecond // 1000
return (
value.strftime('%Y-%m-%dT%H:%M:%S') + f'.{milliseconds:03d}Z'
)
else:
timestamp_format = ISO8601
return value.strftime(timestamp_format)
# Otherwise we continue supporting microseconds in iso8601 for legacy reasons
if value.microsecond > 0:
timestamp_format = ISO8601_MICRO
else:
timestamp_format = ISO8601
return value.strftime(timestamp_format)

def _timestamp_unixtimestamp(self, value):
return int(calendar.timegm(value.timetuple()))
"""Return unix timestamp with precision based on timestamp_precision."""
# As of the addition of the precision flag, we support millisecond precision here as well
if self._timestamp_precision == 'millisecond':
base_timestamp = calendar.timegm(value.timetuple())
milliseconds = (value.microsecond // 1000) / 1000.0
return base_timestamp + milliseconds
else:
return int(calendar.timegm(value.timetuple()))

def _timestamp_rfc822(self, value):
"""Return RFC822 timestamp (always second precision - RFC doesn't support sub-second)."""
# RFC 2822 doesn't support sub-second precision, so always use second precision format
if isinstance(value, datetime.datetime):
value = self._timestamp_unixtimestamp(value)
value = int(calendar.timegm(value.timetuple()))
return formatdate(value, usegmt=True)

def _convert_timestamp_to_str(self, value, timestamp_format=None):
Expand Down
86 changes: 86 additions & 0 deletions tests/unit/test_serialize.py
Original file line number Diff line number Diff line change
Expand Up @@ -616,3 +616,89 @@ def test_restxml_serializes_unicode(self):
self.serialize_to_request(params)
except UnicodeEncodeError:
self.fail("RestXML serializer failed to serialize unicode text.")


class TestTimestampPrecisionParameter(unittest.TestCase):
def setUp(self):
self.model = {
'metadata': {'protocol': 'query', 'apiVersion': '2014-01-01'},
'documentation': '',
'operations': {
'TestOperation': {
'name': 'TestOperation',
'http': {
'method': 'POST',
'requestUri': '/',
},
'input': {'shape': 'InputShape'},
}
},
'shapes': {
'InputShape': {
'type': 'structure',
'members': {
'UnixTimestamp': {'shape': 'UnixTimestampType'},
'IsoTimestamp': {'shape': 'IsoTimestampType'},
},
},
'IsoTimestampType': {
'type': 'timestamp',
"timestampFormat": "iso8601",
},
'UnixTimestampType': {
'type': 'timestamp',
"timestampFormat": "unixTimestamp",
},
},
}
self.service_model = ServiceModel(self.model)

def serialize_to_request(self, input_params, timestamp_precision='second'):
request_serializer = serialize.create_serializer(
self.service_model.metadata['protocol'],
timestamp_precision=timestamp_precision,
)
return request_serializer.serialize_to_request(
input_params, self.service_model.operation_model('TestOperation')
)

def test_second_precision_maintains_existing_behavior(self):
test_datetime = datetime.datetime(2024, 1, 1, 12, 0, 0, 123456)
request = self.serialize_to_request(
{'UnixTimestamp': test_datetime, 'IsoTimestamp': test_datetime}
)
# To maintain backwards compatibility, unix should not include milliseconds by default
self.assertEqual(1704110400, request['body']['UnixTimestamp'])

# ISO always supported microseconds, so we need to continue supporting this
self.assertEqual(
'2024-01-01T12:00:00.123456Z',
request['body']['IsoTimestamp'],
)

def test_millisecond_precision_serialization(self):
test_datetime = datetime.datetime(2024, 1, 1, 12, 0, 0, 123456)

# Check that millisecond precision is used when it is opted in to via the input param
request = self.serialize_to_request(
{'UnixTimestamp': test_datetime, 'IsoTimestamp': test_datetime},
'millisecond',
)
self.assertEqual(1704110400.123, request['body']['UnixTimestamp'])
self.assertEqual(
'2024-01-01T12:00:00.123Z',
request['body']['IsoTimestamp'],
)

def test_millisecond_precision_with_zero_microseconds(self):
test_datetime = datetime.datetime(2024, 1, 1, 12, 0, 0, 0)

request = self.serialize_to_request(
{'UnixTimestamp': test_datetime, 'IsoTimestamp': test_datetime},
'millisecond',
)
self.assertEqual(1704110400.0, request['body']['UnixTimestamp'])
self.assertEqual(
'2024-01-01T12:00:00.000Z',
request['body']['IsoTimestamp'],
)
Loading