From a118a13897f5ca41f287a4478813832dbe266f87 Mon Sep 17 00:00:00 2001 From: Swathi Pillalamarri Date: Fri, 1 Oct 2021 01:52:07 -0700 Subject: [PATCH 1/2] check avro type leaks --- .../avroserializer/_avro_serializer.py | 40 +- .../_schema_registry_avro_serializer.py | 21 +- .../serializer/avroserializer/exceptions.py | 40 ++ ...serializer_with_auto_register_schemas.yaml | 8 +- ...ializer_without_auto_register_schemas.yaml | 8 +- ...zer.test_parse_error_schema_as_record.yaml | 51 ++ ...serializer.test_parse_invalid_aliases.yaml | 52 ++ ...vro_serializer.test_parse_record_name.yaml | 100 ++++ ...t_avro_serializer.test_serialize_enum.yaml | 50 ++ ...avro_serializer.test_serialize_record.yaml | 53 ++ ...avro_serializer_parse_invalid_aliases.yaml | 41 ++ .../tests/test_avro_serializer.py | 470 +++++++++++++++++- 12 files changed, 908 insertions(+), 26 deletions(-) create mode 100644 sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avroserializer/exceptions.py create mode 100644 sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer.test_parse_error_schema_as_record.yaml create mode 100644 sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer.test_parse_invalid_aliases.yaml create mode 100644 sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer.test_parse_record_name.yaml create mode 100644 sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer.test_serialize_enum.yaml create mode 100644 sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer.test_serialize_record.yaml create mode 100644 sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer.test_sr_avro_serializer_parse_invalid_aliases.yaml diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avroserializer/_avro_serializer.py b/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avroserializer/_avro_serializer.py index a190bffe5467..00d769d2c465 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avroserializer/_avro_serializer.py +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avroserializer/_avro_serializer.py @@ -26,7 +26,9 @@ from typing import BinaryIO, Union, TypeVar, Dict from io import BytesIO import avro -from avro.io import DatumWriter, DatumReader, BinaryDecoder, BinaryEncoder +from avro.schema import SchemaParseException, AvroException, InvalidName +from avro.io import DatumWriter, DatumReader, BinaryDecoder, BinaryEncoder, AvroTypeException +from .exceptions import SchemaParseException as AzureSchemaParseException ObjectType = TypeVar("ObjectType") @@ -43,17 +45,17 @@ def __init__(self, codec=None): def serialize( self, - data, # type: ObjectType - schema, # type: Union[str, bytes, avro.schema.Schema] + data, # type: Mapping[str, Any] + schema, # type: Union[str, avro.schema.Schema] ): # type: (ObjectType, Union[str, bytes, avro.schema.Schema]) -> bytes """Convert the provided value to it's binary representation and write it to the stream. Schema must be a Avro RecordSchema: https://avro.apache.org/docs/1.10.0/gettingstartedpython.html#Defining+a+schema - :param data: An object to serialize - :type data: ObjectType + :param data: A data to serialize + :type data: Mapping[str, Any] :param schema: An Avro RecordSchema - :type schema: Union[str, bytes, avro.schema.Schema] + :type schema: Union[str, avro.schema.Schema] :returns: Encoded bytes :rtype: bytes """ @@ -61,7 +63,10 @@ def serialize( raise ValueError("Schema is required in Avro serializer.") if not isinstance(schema, avro.schema.Schema): - schema = avro.schema.parse(schema) + try: + schema = avro.schema.parse(schema) + except (SchemaParseException, AvroTypeException, AvroException, InvalidName) as e: + raise AzureSchemaParseException(e) try: writer = self._schema_writer_cache[str(schema)] @@ -71,22 +76,25 @@ def serialize( stream = BytesIO() with stream: - writer.write(data, BinaryEncoder(stream)) + try: + writer.write(data, BinaryEncoder(stream)) + except AvroTypeException as e: + raise ValueError(e) encoded_data = stream.getvalue() return encoded_data def deserialize( self, data, # type: Union[bytes, BinaryIO] - schema, # type: Union[str, bytes, avro.schema.Schema] + schema, # type: str ): - # type: (Union[bytes, BinaryIO], Union[str, bytes, avro.schema.Schema]) -> ObjectType + # type: (Union[bytes, BinaryIO], str) -> ObjectType """Read the binary representation into a specific type. Return type will be ignored, since the schema is deduced from the provided bytes. :param data: A stream of bytes or bytes directly :type data: BinaryIO or bytes :param schema: An Avro RecordSchema - :type schema: Union[str, bytes, avro.schema.Schema] + :type schema: str :returns: An instantiated object :rtype: ObjectType """ @@ -94,7 +102,10 @@ def deserialize( data = BytesIO(data) if not isinstance(schema, avro.schema.Schema): - schema = avro.schema.parse(schema) + try: + schema = avro.schema.parse(schema) + except (SchemaParseException, AvroTypeException, AvroException, InvalidName) as e: + raise AzureSchemaParseException(e) try: reader = self._schema_reader_cache[str(schema)] @@ -104,6 +115,9 @@ def deserialize( with data: bin_decoder = BinaryDecoder(data) - decoded_data = reader.read(bin_decoder) + try: + decoded_data = reader.read(bin_decoder) + except AvroTypeException as e: + raise ValueError(e) return decoded_data diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avroserializer/_schema_registry_avro_serializer.py b/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avroserializer/_schema_registry_avro_serializer.py index f316fa9d1f07..57f7e51a3192 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avroserializer/_schema_registry_avro_serializer.py +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avroserializer/_schema_registry_avro_serializer.py @@ -30,7 +30,10 @@ from io import BytesIO from typing import Any, Dict, Mapping import avro +from avro.schema import SchemaParseException, AvroException, InvalidName, PRIMITIVE_TYPES +from avro.io import AvroTypeException +from .exceptions import SchemaParseException as AzureSchemaParseException from ._constants import SCHEMA_ID_START_INDEX, SCHEMA_ID_LENGTH, DATA_START_INDEX from ._avro_serializer import AvroObjectSerializer @@ -121,10 +124,12 @@ def serialize(self, value, **kwargs): Encode data with the given schema. The returns bytes are consisted of: The first 4 bytes denoting record format identifier. The following 32 bytes denoting schema id returned by schema registry service. The remaining bytes are the real data payload. + Schema must be a Avro RecordSchema: + https://avro.apache.org/docs/1.10.0/gettingstartedpython.html#Defining+a+schema :param value: The data to be encoded. :type value: Mapping[str, Any] - :keyword schema: Required. The schema used to encode the data. + :keyword schema: Required. The Avro RecordSchema used to encode the data. :paramtype schema: str :rtype: bytes """ @@ -135,12 +140,22 @@ def serialize(self, value, **kwargs): try: cached_schema = self._user_input_schema_cache[raw_input_schema] except KeyError: - parsed_schema = avro.schema.parse(raw_input_schema) + try: + parsed_schema = avro.schema.parse(raw_input_schema) + except (SchemaParseException, AvroTypeException, AvroException, InvalidName) as e: + raise AzureSchemaParseException(e) self._user_input_schema_cache[raw_input_schema] = parsed_schema cached_schema = parsed_schema + # TODO: check if there's a more pythonic way, if we need it at all + if cached_schema.type in PRIMITIVE_TYPES: + raise ValueError("Expected schema of type `Record`, got instead primitive: {}".format(raw_input_schema)) + record_format_identifier = b"\0\0\0\0" - schema_id = self._get_schema_id(cached_schema.fullname, str(cached_schema), **kwargs) + try: + schema_id = self._get_schema_id(cached_schema.fullname, str(cached_schema), **kwargs) + except AttributeError: + raise ValueError("Expected schema of type `Record`, got instead schema of type: {}".format(cached_schema.type)) data_bytes = self._avro_serializer.serialize(value, cached_schema) stream = BytesIO() diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avroserializer/exceptions.py b/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avroserializer/exceptions.py new file mode 100644 index 000000000000..9b0177270b54 --- /dev/null +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avroserializer/exceptions.py @@ -0,0 +1,40 @@ +# -------------------------------------------------------------------------- +# +# Copyright (c) Microsoft Corporation. All rights reserved. +# +# The MIT License (MIT) +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the ""Software""), to +# deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +# -------------------------------------------------------------------------- +from azure.core.exceptions import AzureError + +class SchemaParseException(AzureError): + """Error parsing a JSON schema. + + :ivar message: The error message. + :vartype message: str + :ivar error: The error condition, if available. + :vartype error: str + """ + + def __init__(self, message, **kwargs): + self.message = message + self.error = kwargs.get("error") + super(SchemaParseException, self).__init__(self.message, **kwargs) diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer.test_basic_sr_avro_serializer_with_auto_register_schemas.yaml b/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer.test_basic_sr_avro_serializer_with_auto_register_schemas.yaml index 90dc88aa96ec..70ece0e6bfd6 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer.test_basic_sr_avro_serializer_with_auto_register_schemas.yaml +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer.test_basic_sr_avro_serializer_with_auto_register_schemas.yaml @@ -23,12 +23,12 @@ interactions: uri: https://fake_resource.servicebus.windows.net/$schemagroups/fakegroup/schemas/example.avro.User?api-version=2017-04 response: body: - string: '{"id":"f666e373299048fabaa4296f5dbfed46"}' + string: '{"id":"7b4eff1c25d9438a975ff7a3d985a5c6"}' headers: content-type: - application/json date: - - Tue, 28 Sep 2021 22:27:25 GMT + - Fri, 01 Oct 2021 08:50:24 GMT location: - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/fakegroup/schemas/example.avro.User/versions/1?api-version=2017-04 server: @@ -38,9 +38,9 @@ interactions: transfer-encoding: - chunked x-schema-id: - - f666e373299048fabaa4296f5dbfed46 + - 7b4eff1c25d9438a975ff7a3d985a5c6 x-schema-id-location: - - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/getschemabyid/f666e373299048fabaa4296f5dbfed46?api-version=2017-04 + - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/getschemabyid/7b4eff1c25d9438a975ff7a3d985a5c6?api-version=2017-04 x-schema-type: - Avro x-schema-version: diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer.test_basic_sr_avro_serializer_without_auto_register_schemas.yaml b/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer.test_basic_sr_avro_serializer_without_auto_register_schemas.yaml index 7dce4b62fbfe..c0d20d3231c8 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer.test_basic_sr_avro_serializer_without_auto_register_schemas.yaml +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer.test_basic_sr_avro_serializer_without_auto_register_schemas.yaml @@ -23,12 +23,12 @@ interactions: uri: https://fake_resource.servicebus.windows.net/$schemagroups/fakegroup/schemas/example.avro.User?api-version=2017-04 response: body: - string: '{"id":"f666e373299048fabaa4296f5dbfed46"}' + string: '{"id":"7b4eff1c25d9438a975ff7a3d985a5c6"}' headers: content-type: - application/json date: - - Tue, 28 Sep 2021 22:27:26 GMT + - Fri, 01 Oct 2021 08:50:25 GMT location: - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/fakegroup/schemas/example.avro.User/versions/1?api-version=2017-04 server: @@ -38,9 +38,9 @@ interactions: transfer-encoding: - chunked x-schema-id: - - f666e373299048fabaa4296f5dbfed46 + - 7b4eff1c25d9438a975ff7a3d985a5c6 x-schema-id-location: - - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/getschemabyid/f666e373299048fabaa4296f5dbfed46?api-version=2017-04 + - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/getschemabyid/7b4eff1c25d9438a975ff7a3d985a5c6?api-version=2017-04 x-schema-type: - Avro x-schema-version: diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer.test_parse_error_schema_as_record.yaml b/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer.test_parse_error_schema_as_record.yaml new file mode 100644 index 000000000000..99f635837834 --- /dev/null +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer.test_parse_error_schema_as_record.yaml @@ -0,0 +1,51 @@ +interactions: +- request: + body: '"{\"type\": \"error\", \"name\": \"User\", \"namespace\": \"example.avro.error\", + \"fields\": [{\"type\": \"string\", \"name\": \"name\"}]}"' + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '140' + Content-Type: + - application/json + User-Agent: + - azsdk-python-azureschemaregistry/1.0.0b1 Python/3.9.0 (Windows-10-10.0.19041-SP0) + X-Schema-Type: + - Avro + method: PUT + uri: https://fake_resource.servicebus.windows.net/$schemagroups/fakegroup/schemas/example.avro.error.User?api-version=2017-04 + response: + body: + string: '{"id":"4e9f5888b6f8411abaa4ce6c110b2fb3"}' + headers: + content-type: + - application/json + date: + - Fri, 01 Oct 2021 08:50:26 GMT + location: + - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/fakegroup/schemas/example.avro.error.User/versions/1?api-version=2017-04 + server: + - Microsoft-HTTPAPI/2.0 + strict-transport-security: + - max-age=31536000 + transfer-encoding: + - chunked + x-schema-id: + - 4e9f5888b6f8411abaa4ce6c110b2fb3 + x-schema-id-location: + - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/getschemabyid/4e9f5888b6f8411abaa4ce6c110b2fb3?api-version=2017-04 + x-schema-type: + - Avro + x-schema-version: + - '1' + x-schema-versions-location: + - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/swathip-test-schema/schemas/example.avro.error.User/versions?api-version=2017-04 + status: + code: 200 + message: OK +version: 1 diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer.test_parse_invalid_aliases.yaml b/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer.test_parse_invalid_aliases.yaml new file mode 100644 index 000000000000..559ecefc7874 --- /dev/null +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer.test_parse_invalid_aliases.yaml @@ -0,0 +1,52 @@ +interactions: +- request: + body: '"{\"type\": \"record\", \"aliases\": [\"Resu\"], \"name\": \"User_aliases\", + \"namespace\": \"example.avro\", \"fields\": [{\"type\": \"string\", \"name\": + \"name\"}]}"' + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '168' + Content-Type: + - application/json + User-Agent: + - azsdk-python-azureschemaregistry/1.0.0b1 Python/3.9.0 (Windows-10-10.0.19041-SP0) + X-Schema-Type: + - Avro + method: PUT + uri: https://fake_resource.servicebus.windows.net/$schemagroups/fakegroup/schemas/example.avro.User_aliases?api-version=2017-04 + response: + body: + string: '{"id":"83a1c5b1e3f049acae089ed34eb5191c"}' + headers: + content-type: + - application/json + date: + - Fri, 01 Oct 2021 07:29:00 GMT + location: + - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/fakegroup/schemas/example.avro.User_aliases/versions/1?api-version=2017-04 + server: + - Microsoft-HTTPAPI/2.0 + strict-transport-security: + - max-age=31536000 + transfer-encoding: + - chunked + x-schema-id: + - 83a1c5b1e3f049acae089ed34eb5191c + x-schema-id-location: + - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/getschemabyid/83a1c5b1e3f049acae089ed34eb5191c?api-version=2017-04 + x-schema-type: + - Avro + x-schema-version: + - '1' + x-schema-versions-location: + - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/swathip-test-schema/schemas/example.avro.User_aliases/versions?api-version=2017-04 + status: + code: 200 + message: OK +version: 1 diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer.test_parse_record_name.yaml b/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer.test_parse_record_name.yaml new file mode 100644 index 000000000000..61b2c9b7873d --- /dev/null +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer.test_parse_record_name.yaml @@ -0,0 +1,100 @@ +interactions: +- request: + body: '"{\"type\": \"record\", \"name\": \"User.avro\", \"namespace\": \"User\", + \"fields\": [{\"type\": \"string\", \"name\": \"name\"}]}"' + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '132' + Content-Type: + - application/json + User-Agent: + - azsdk-python-azureschemaregistry/1.0.0b1 Python/3.9.0 (Windows-10-10.0.19041-SP0) + X-Schema-Type: + - Avro + method: PUT + uri: https://fake_resource.servicebus.windows.net/$schemagroups/fakegroup/schemas/User.avro?api-version=2017-04 + response: + body: + string: '{"id":"7d036bfe525b499c8f2f83dd7c83088c"}' + headers: + content-type: + - application/json + date: + - Fri, 01 Oct 2021 08:50:28 GMT + location: + - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/fakegroup/schemas/User.avro/versions/1?api-version=2017-04 + server: + - Microsoft-HTTPAPI/2.0 + strict-transport-security: + - max-age=31536000 + transfer-encoding: + - chunked + x-schema-id: + - 7d036bfe525b499c8f2f83dd7c83088c + x-schema-id-location: + - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/getschemabyid/7d036bfe525b499c8f2f83dd7c83088c?api-version=2017-04 + x-schema-type: + - Avro + x-schema-version: + - '1' + x-schema-versions-location: + - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/swathip-test-schema/schemas/User.avro/versions?api-version=2017-04 + status: + code: 200 + message: OK +- request: + body: '"{\"type\": \"record\", \"name\": \"User\", \"fields\": [{\"type\": \"string\", + \"name\": \"name\"}]}"' + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '102' + Content-Type: + - application/json + User-Agent: + - azsdk-python-azureschemaregistry/1.0.0b1 Python/3.9.0 (Windows-10-10.0.19041-SP0) + X-Schema-Type: + - Avro + method: PUT + uri: https://fake_resource.servicebus.windows.net/$schemagroups/fakegroup/schemas/User?api-version=2017-04 + response: + body: + string: '{"id":"db1e563051b14b4f972c5d80012d0d13"}' + headers: + content-type: + - application/json + date: + - Fri, 01 Oct 2021 08:50:28 GMT + location: + - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/fakegroup/schemas/User/versions/1?api-version=2017-04 + server: + - Microsoft-HTTPAPI/2.0 + strict-transport-security: + - max-age=31536000 + transfer-encoding: + - chunked + x-schema-id: + - db1e563051b14b4f972c5d80012d0d13 + x-schema-id-location: + - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/getschemabyid/db1e563051b14b4f972c5d80012d0d13?api-version=2017-04 + x-schema-type: + - Avro + x-schema-version: + - '1' + x-schema-versions-location: + - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/swathip-test-schema/schemas/User/versions?api-version=2017-04 + status: + code: 200 + message: OK +version: 1 diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer.test_serialize_enum.yaml b/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer.test_serialize_enum.yaml new file mode 100644 index 000000000000..94ee8807b1f5 --- /dev/null +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer.test_serialize_enum.yaml @@ -0,0 +1,50 @@ +interactions: +- request: + body: '"{\"type\": \"enum\", \"name\": \"Suit\", \"symbols\": [\"SPADES\", \"HEARTS\"]}"' + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '81' + Content-Type: + - application/json + User-Agent: + - azsdk-python-azureschemaregistry/1.0.0b1 Python/3.9.0 (Windows-10-10.0.19041-SP0) + X-Schema-Type: + - Avro + method: PUT + uri: https://fake_resource.servicebus.windows.net/$schemagroups/fakegroup/schemas/Suit?api-version=2017-04 + response: + body: + string: '{"id":"1e8423d98b774ebc8a0f4be3889baf52"}' + headers: + content-type: + - application/json + date: + - Fri, 01 Oct 2021 08:50:30 GMT + location: + - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/fakegroup/schemas/Suit/versions/1?api-version=2017-04 + server: + - Microsoft-HTTPAPI/2.0 + strict-transport-security: + - max-age=31536000 + transfer-encoding: + - chunked + x-schema-id: + - 1e8423d98b774ebc8a0f4be3889baf52 + x-schema-id-location: + - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/getschemabyid/1e8423d98b774ebc8a0f4be3889baf52?api-version=2017-04 + x-schema-type: + - Avro + x-schema-version: + - '1' + x-schema-versions-location: + - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/swathip-test-schema/schemas/Suit/versions?api-version=2017-04 + status: + code: 200 + message: OK +version: 1 diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer.test_serialize_record.yaml b/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer.test_serialize_record.yaml new file mode 100644 index 000000000000..198ff9db378b --- /dev/null +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer.test_serialize_record.yaml @@ -0,0 +1,53 @@ +interactions: +- request: + body: '"{\"type\": \"record\", \"name\": \"User\", \"namespace\": \"example.avro.populatedrecord\", + \"fields\": [{\"type\": \"string\", \"name\": \"name\"}, {\"type\": \"int\", + \"name\": \"age\"}, {\"type\": \"boolean\", \"name\": \"married\"}, {\"type\": + \"float\", \"name\": \"height\"}, {\"type\": \"bytes\", \"name\": \"randb\"}]}"' + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '328' + Content-Type: + - application/json + User-Agent: + - azsdk-python-azureschemaregistry/1.0.0b1 Python/3.9.0 (Windows-10-10.0.19041-SP0) + X-Schema-Type: + - Avro + method: PUT + uri: https://fake_resource.servicebus.windows.net/$schemagroups/fakegroup/schemas/example.avro.populatedrecord.User?api-version=2017-04 + response: + body: + string: '{"id":"710ccdfdc2174135b1e971e47bc3f595"}' + headers: + content-type: + - application/json + date: + - Fri, 01 Oct 2021 08:50:31 GMT + location: + - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/fakegroup/schemas/example.avro.populatedrecord.User/versions/6?api-version=2017-04 + server: + - Microsoft-HTTPAPI/2.0 + strict-transport-security: + - max-age=31536000 + transfer-encoding: + - chunked + x-schema-id: + - 710ccdfdc2174135b1e971e47bc3f595 + x-schema-id-location: + - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/getschemabyid/710ccdfdc2174135b1e971e47bc3f595?api-version=2017-04 + x-schema-type: + - Avro + x-schema-version: + - '6' + x-schema-versions-location: + - https://swathip-test-eventhubs.servicebus.windows.net:443/$schemagroups/swathip-test-schema/schemas/example.avro.populatedrecord.User/versions?api-version=2017-04 + status: + code: 200 + message: OK +version: 1 diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer.test_sr_avro_serializer_parse_invalid_aliases.yaml b/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer.test_sr_avro_serializer_parse_invalid_aliases.yaml new file mode 100644 index 000000000000..efa4e09532c6 --- /dev/null +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/recordings/test_avro_serializer.test_sr_avro_serializer_parse_invalid_aliases.yaml @@ -0,0 +1,41 @@ +interactions: +- request: + body: '"{\"type\": \"record\", \"name\": \"User\", \"namespace\": \"example.avro\", + \"fields\": [{\"type\": \"string\", \"name\": \"name\"}]}"' + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '135' + Content-Type: + - application/json + User-Agent: + - azsdk-python-azureschemaregistry/1.0.0b1 Python/3.9.0 (Windows-10-10.0.19041-SP0) + X-Schema-Type: + - Avro + method: POST + uri: https://fake_resource.servicebus.windows.net/$schemagroups/fakegroup/schemas/example.avro.User?api-version=2017-04 + response: + body: + string: '{"Code":404,"Detail":"Schema fakegroup\/example.avro.User does not + exist. TrackingId:08d40854-90ff-4062-81f1-af9ca09ff70e_G8, SystemTracker:swathip-test-eventhubs.servicebus.windows.net:$schemagroups\/fakegroup\/schemas\/example.avro.User, + Timestamp:2021-09-30T07:27:40"}' + headers: + content-type: + - application/json + date: + - Thu, 30 Sep 2021 07:27:39 GMT + server: + - Microsoft-HTTPAPI/2.0 + strict-transport-security: + - max-age=31536000 + transfer-encoding: + - chunked + status: + code: 404 + message: Not Found +version: 1 diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/test_avro_serializer.py b/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/test_avro_serializer.py index bf2bca41907f..1a047d919e00 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/test_avro_serializer.py +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/tests/test_avro_serializer.py @@ -22,10 +22,12 @@ import uuid import avro import avro.io +import json from io import BytesIO from azure.schemaregistry import SchemaRegistryClient from azure.schemaregistry.serializer.avroserializer import SchemaRegistryAvroSerializer +from azure.schemaregistry.serializer.avroserializer.exceptions import SchemaParseException from azure.schemaregistry.serializer.avroserializer._avro_serializer import AvroObjectSerializer from azure.identity import ClientSecretCredential from azure.core.exceptions import ClientAuthenticationError, ServiceRequestError, HttpResponseError @@ -67,12 +69,14 @@ def test_raw_avro_serializer_negative(self): raw_avro_object_serializer = AvroObjectSerializer() dict_data_wrong_type = {"name": u"Ben", "favorite_number": u"something", "favorite_color": u"red"} - with pytest.raises(avro.io.AvroTypeException): + with pytest.raises(ValueError) as e: raw_avro_object_serializer.serialize(dict_data_wrong_type, schema) + assert "is not an example of" in str(e.value) dict_data_missing_required_field = {"favorite_number": 7, "favorite_color": u"red"} - with pytest.raises(avro.io.AvroTypeException): + with pytest.raises(ValueError) as e: raw_avro_object_serializer.serialize(dict_data_missing_required_field, schema) + assert "is not an example of" in str(e.value) @SchemaRegistryPowerShellPreparer() def test_basic_sr_avro_serializer_with_auto_register_schemas(self, schemaregistry_fully_qualified_namespace, schemaregistry_group, **kwargs): @@ -123,3 +127,465 @@ def test_basic_sr_avro_serializer_without_auto_register_schemas(self, schemaregi assert decoded_data["favorite_color"] == u"red" sr_avro_serializer.close() + + ################################################################# + ######################### PARSE SCHEMAS ######################### + ################################################################# + + @SchemaRegistryPowerShellPreparer() + def test_parse_invalid_json_string(self, schemaregistry_fully_qualified_namespace, schemaregistry_group, **kwargs): + sr_client = self.create_basic_client(SchemaRegistryClient, endpoint=schemaregistry_fully_qualified_namespace) + sr_avro_serializer = SchemaRegistryAvroSerializer(client=sr_client, group_name=schemaregistry_group, auto_register_schemas=True) + invalid_schema = { + "name":"User", + "type":"record", + "namespace":"example.avro", + "fields":[{"name":"name","type":"string"}] + } + invalid_schema_string = "{}".format(invalid_schema) + with pytest.raises(SchemaParseException): # caught avro SchemaParseException + sr_avro_serializer.serialize({"name": u"Ben"}, schema=invalid_schema_string) + + ######################### UNION SCHEMA ######################### + + @SchemaRegistryPowerShellPreparer() + def test_parse_invalid_union_item(self, schemaregistry_fully_qualified_namespace, schemaregistry_group, **kwargs): + sr_client = self.create_basic_client(SchemaRegistryClient, endpoint=schemaregistry_fully_qualified_namespace) + sr_avro_serializer = SchemaRegistryAvroSerializer(client=sr_client, group_name=schemaregistry_group, auto_register_schemas=True) + + schema_invalid_union_item = [""" + "name":"User", + "namespace":"example.avro", + "type":"record", + "fields":[{"name":"name","type":"string"}] + """] + with pytest.raises(TypeError) as e: + sr_avro_serializer.serialize({}, schema=schema_invalid_union_item) + assert str(e.value) == "unhashable type: 'list'" + + ######################### PRIMITIVES ######################### + + @SchemaRegistryPowerShellPreparer() + def test_parse_primitive_types(self, schemaregistry_fully_qualified_namespace, schemaregistry_group, **kwargs): + sr_client = self.create_basic_client(SchemaRegistryClient, endpoint=schemaregistry_fully_qualified_namespace) + sr_avro_serializer = SchemaRegistryAvroSerializer(client=sr_client, group_name=schemaregistry_group, auto_register_schemas=True) + + primitive_string = "string" + with pytest.raises(SchemaParseException) as e: + sr_avro_serializer.serialize("hello", schema=primitive_string) + + int_type = """{"type": "int", "fake_prop": "string"}""" + # Strange schema, valid for parsing but not for serializing. Catch early by checking for primitive + # before a bad schema gets registered. Otherwise, will be caught when writing data. + with pytest.raises(ValueError) as e: + sr_avro_serializer.serialize({"fake_prop": "hello"}, schema=int_type) + + null_type = """{"type": "null"}""" + with pytest.raises(ValueError) as e: # was HttpResponseError: Bad Request to service, schema fails service side validation + sr_avro_serializer.serialize(None, schema=null_type) + + ######################### type fixed ######################### + + @SchemaRegistryPowerShellPreparer() + def test_parse_fixed_types(self, schemaregistry_fully_qualified_namespace, schemaregistry_group, **kwargs): + sr_client = self.create_basic_client(SchemaRegistryClient, endpoint=schemaregistry_fully_qualified_namespace) + sr_avro_serializer = SchemaRegistryAvroSerializer(client=sr_client, group_name=schemaregistry_group, auto_register_schemas=True) + + # should give warning from IgnoredLogicalType error since precision < 0 + #fixed_type_ignore_logical_type_error = """{"type": "fixed", "size": 4, "namespace":"example.avro", "name":"User", "precision": -1}""" + #sr_avro_serializer.serialize({}, schema=fixed_type_ignore_logical_type_error) + + schema_no_size = """{"type": "fixed", "name":"User"}""" + with pytest.raises(SchemaParseException): # caught AvroException + sr_avro_serializer.serialize({}, schema=schema_no_size) + + schema_no_name = """{"type": "fixed", "size": 3}""" + with pytest.raises(SchemaParseException): # caught SchemaParseException + sr_avro_serializer.serialize({}, schema=schema_no_name) + + schema_wrong_name = """{"type": "fixed", "name": 1, "size": 3}""" + with pytest.raises(SchemaParseException): # caught SchemaParseException + sr_avro_serializer.serialize({}, schema=schema_wrong_name) + + schema_wrong_namespace = """{"type": "fixed", "name": "User", "size": 3, "namespace": 1}""" + with pytest.raises(SchemaParseException): # caught SchemaParseException + sr_avro_serializer.serialize({}, schema=schema_wrong_namespace) + + ######################### type enum ######################### + + @SchemaRegistryPowerShellPreparer() + def test_parse_enum_types(self, schemaregistry_fully_qualified_namespace, schemaregistry_group, **kwargs): + sr_client = self.create_basic_client(SchemaRegistryClient, endpoint=schemaregistry_fully_qualified_namespace) + sr_avro_serializer = SchemaRegistryAvroSerializer(client=sr_client, group_name=schemaregistry_group, auto_register_schemas=True) + + # THE FOLLOWING SHOULD RAISE AVRO SCHEMAPARSEEXCEPTION ERRORS + # IT HAS BEEN FIXED IN AVRO GITHUB REPO, BUT HAS NOT BEEN RELEASED YET + #schema_enum_wrong_no_symbols_type = """{ + # "type": "enum", + # "name": "Suit" + #}""" + #with pytest.raises(SchemaParseException): # caught SchemaParseException + # sr_avro_serializer.serialize({}, schema=schema_enum_wrong_no_symbols_type) + + #schema_enum_wrong_symbol_item_types = """{ + # "type": "enum", + # "name": "Suit", + # "symbols": [1] + #}""" + #with pytest.raises(SchemaParseException): # caught SchemaParseException + # sr_avro_serializer.serialize({}, schema=schema_enum_wrong_symbol_item_types) + + schema_enum_invalid_symbol_item_names = """{ + "type": "enum", + "name": "Suit", + "symbols": ["9abc"] + }""" + with pytest.raises(SchemaParseException): # caught SchemaParseException + sr_avro_serializer.serialize({}, schema=schema_enum_invalid_symbol_item_names) + + schema_enum_duplicate_symbols = """{ + "type": "enum", + "name": "Suit", + "symbols": ["abc", "abc"] + }""" + with pytest.raises(SchemaParseException): # caught SchemaParseException + sr_avro_serializer.serialize({}, schema=schema_enum_duplicate_symbols) + + ################################################################### + + ## MAP, ARRAY, ERRORUNION schema except types that are already accounted for. ## + ## Test in the future when these are implemented ## + + ######################### type unspecified ######################### + + @SchemaRegistryPowerShellPreparer() + def test_parse_invalid_type(self, schemaregistry_fully_qualified_namespace, schemaregistry_group, **kwargs): + sr_client = self.create_basic_client(SchemaRegistryClient, endpoint=schemaregistry_fully_qualified_namespace) + sr_avro_serializer = SchemaRegistryAvroSerializer(client=sr_client, group_name=schemaregistry_group, auto_register_schemas=True) + + schema_no_type = """{ + "name": "User", + "namespace":"example.avro", + "fields":[{"name":"name","type":"string"}] + }""" + with pytest.raises(SchemaParseException): # caught avro SchemaParseException + sr_avro_serializer.serialize({"name": u"Ben"}, schema=schema_no_type) + + schema_wrong_type_type = """{ + "name":"User", + "type":1, + "namespace":"example.avro", + "fields":[{"name":"name","type":"string"}] + }""" + with pytest.raises(SchemaParseException): + sr_avro_serializer.serialize({"name": u"Ben"}, schema=schema_wrong_type_type) + + ######################### RECORD SCHEMA ######################### + + @SchemaRegistryPowerShellPreparer() + def test_parse_record_name(self, schemaregistry_fully_qualified_namespace, schemaregistry_group, **kwargs): + sr_client = self.create_basic_client(SchemaRegistryClient, endpoint=schemaregistry_fully_qualified_namespace) + sr_avro_serializer = SchemaRegistryAvroSerializer(client=sr_client, group_name=schemaregistry_group, auto_register_schemas=True) + + schema_name_has_dot = """{ + "namespace": "thrownaway", + "name":"User.avro", + "type":"record", + "fields":[{"name":"name","type":"string"}] + }""" + encoded_schema = sr_avro_serializer.serialize({"name": u"Ben"}, schema=schema_name_has_dot) + schema_id = encoded_schema[4:36].decode("utf-8") + registered_schema = sr_client.get_schema(schema_id) + # TODO: change content to schema_definition below after release SR b3 + decoded_registered_schema = json.loads(registered_schema.schema_content) + + # ensure that namespace is saved as part of name before . in registered schema + # result, but might be bug in avro? + assert decoded_registered_schema["name"] == "User.avro" + assert decoded_registered_schema["namespace"] == "User" + + schema_name_no_namespace = """{ + "name":"User", + "type":"record", + "fields":[{"name":"name","type":"string"}] + }""" + encoded_schema = sr_avro_serializer.serialize({"name": u"Ben"}, schema=schema_name_no_namespace) + schema_id = encoded_schema[4:36].decode("utf-8") + registered_schema = sr_client.get_schema(schema_id) + # TODO: change content to schema_definition below after release SR b3 + decoded_registered_schema = json.loads(registered_schema.schema_content) + + # ensure that namespace is saved as part of name before . in registered schema + assert decoded_registered_schema["name"] == "User" + assert "namespace" not in decoded_registered_schema + + schema_invalid_fullname = """{ + "name":"abc", + "type":"record", + "namespace":"9example.avro", + "fields":[{"name":"name","type":"string"}] + }""" + with pytest.raises(SchemaParseException): + sr_avro_serializer.serialize({"name": u"Ben"}, schema=schema_invalid_fullname) + + schema_invalid_name_in_fullname = """{ + "name":"1abc", + "type":"record", + "fields":[{"name":"name","type":"string"}] + }""" + with pytest.raises(SchemaParseException): + sr_avro_serializer.serialize({"name": u"Ben"}, schema=schema_invalid_name_in_fullname) + + schema_invalid_name_reserved_type = """{ + "name":"record", + "type":"record", + "fields":[{"name":"name","type":"string"}] + }""" + with pytest.raises(SchemaParseException): + sr_avro_serializer.serialize({"name": u"Ben"}, schema=schema_invalid_name_in_fullname) + + schema_wrong_type_name = """{ + "name":1, + "type":"record", + "namespace":"example.avro", + "fields":[{"name":"name","type":"string"}] + }""" + with pytest.raises(SchemaParseException): + sr_avro_serializer.serialize({"name": u"Ben"}, schema=schema_wrong_type_name) + + schema_no_name = """{ + "namespace":"example.avro", + "type":"record", + "fields":[{"name":"name","type":"string"}] + }""" + with pytest.raises(SchemaParseException): + sr_avro_serializer.serialize({"name": u"Ben"}, schema=schema_no_name) + + @SchemaRegistryPowerShellPreparer() + def test_parse_error_schema_as_record(self, schemaregistry_fully_qualified_namespace, schemaregistry_group, **kwargs): + sr_client = self.create_basic_client(SchemaRegistryClient, endpoint=schemaregistry_fully_qualified_namespace) + sr_avro_serializer = SchemaRegistryAvroSerializer(client=sr_client, group_name=schemaregistry_group, auto_register_schemas=True) + + schema_error_type = """{ + "name":"User", + "namespace":"example.avro.error", + "type":"error", + "fields":[{"name":"name","type":"string"}] + }""" + encoded_data = sr_avro_serializer.serialize({"name": u"Ben"}, schema=schema_error_type) + schema_id = encoded_data[4:36].decode("utf-8") + registered_schema = sr_client.get_schema(schema_id) + # TODO: change content to schema_definition below after release SR b3 + decoded_registered_schema = json.loads(registered_schema.schema_content) + assert decoded_registered_schema["type"] == "error" + + @SchemaRegistryPowerShellPreparer() + def test_parse_record_fields(self, schemaregistry_fully_qualified_namespace, schemaregistry_group, **kwargs): + sr_client = self.create_basic_client(SchemaRegistryClient, endpoint=schemaregistry_fully_qualified_namespace) + sr_avro_serializer = SchemaRegistryAvroSerializer(client=sr_client, group_name=schemaregistry_group, auto_register_schemas=True) + + schema_no_fields = """{ + "name":"User", + "namespace":"example.avro", + "type":"record" + }""" + with pytest.raises(SchemaParseException): + sr_avro_serializer.serialize({"name": u"Ben"}, schema=schema_no_fields) + + schema_wrong_type_fields = """{ + "name":"User", + "namespace":"example.avro", + "type":"record" + "fields": "hello" + }""" + with pytest.raises(SchemaParseException): + sr_avro_serializer.serialize({"name": u"Ben"}, schema=schema_wrong_type_fields) + + schema_wrong_field_item_type = """{ + "name":"User", + "namespace":"example.avro", + "type":"record" + "fields": ["hello"] + }""" + with pytest.raises(SchemaParseException): + sr_avro_serializer.serialize({"name": u"Ben"}, schema=schema_wrong_field_item_type) + + schema_record_field_no_name= """{ + "name":"User", + "namespace":"example.avro", + "type":"record", + "fields":[{"type":"string"}] + }""" + with pytest.raises(SchemaParseException): + sr_avro_serializer.serialize({"name": u"Ben"}, schema=schema_record_field_no_name) + + schema_record_field_wrong_type_name= """{ + "name":"User", + "namespace":"example.avro", + "type":"record", + "fields":[{"name": 1, "type":"string"}] + }""" + with pytest.raises(SchemaParseException): + sr_avro_serializer.serialize({"name": u"Ben"}, schema=schema_record_field_wrong_type_name) + + schema_record_field_with_invalid_order = """{ + "name":"User", + "namespace":"example.avro.order", + "type":"record", + "fields":[{"name":"name","type":"string","order":"fake_order"}] + }""" + with pytest.raises(SchemaParseException): + sr_avro_serializer.serialize({"name": u"Ben"}, schema=schema_record_field_with_invalid_order) + + schema_record_duplicate_fields = """{ + "name":"User", + "namespace":"example.avro", + "type":"record", + "fields":[{"name":"name","type":"string"}, {"name":"name","type":"string"}] + }""" + with pytest.raises(SchemaParseException): + sr_avro_serializer.serialize({"name": u"Ben"}, schema=schema_record_duplicate_fields) + + schema_field_type_invalid = """{ + "name":"User", + "namespace":"example.avro", + "type":"record", + "fields":[{"name":"name","type":1}] + }""" + with pytest.raises(SchemaParseException): + sr_avro_serializer.serialize({"name": u"Ben"}, schema=schema_field_type_invalid) + + #@SchemaRegistryPowerShellPreparer() + #def test_parse_invalid_aliases(self, schemaregistry_fully_qualified_namespace, schemaregistry_group, **kwargs): + # sr_client = self.create_basic_client(SchemaRegistryClient, endpoint=schemaregistry_fully_qualified_namespace) + # sr_avro_serializer = SchemaRegistryAvroSerializer(client=sr_client, group_name=schemaregistry_group, auto_register_schemas=True) + + # NOT FAILING + # schema_wrong_type_aliases = """{ + # "name":"User_aliases", + # "type":"record", + # "namespace":"example.avro", + # "aliases":["Resu"], + # "fields":[{"name":"name","type":"string"}] + # }""" + # with pytest.raises(SchemaParseException): + # sr_avro_serializer.serialize({"name": u"Ben"}, schema=schema_wrong_type_aliases) + + ################################################################# + #################### SERIALIZE AND DESERIALIZE ################## + ################################################################# + + @SchemaRegistryPowerShellPreparer() + def test_serialize_primitive(self, schemaregistry_fully_qualified_namespace, schemaregistry_group, **kwargs): + sr_client = self.create_basic_client(SchemaRegistryClient, endpoint=schemaregistry_fully_qualified_namespace) + sr_avro_serializer = SchemaRegistryAvroSerializer(client=sr_client, group_name=schemaregistry_group, auto_register_schemas=True) + + schema_null = """{"type": "null"}""" + with pytest.raises(ValueError): # cannot serialize primitive + sr_avro_serializer.serialize(None, schema=schema_null) + + @SchemaRegistryPowerShellPreparer() + def test_serialize_fixed(self, schemaregistry_fully_qualified_namespace, schemaregistry_group, **kwargs): + sr_client = self.create_basic_client(SchemaRegistryClient, endpoint=schemaregistry_fully_qualified_namespace) + sr_avro_serializer = SchemaRegistryAvroSerializer(client=sr_client, group_name=schemaregistry_group, auto_register_schemas=True) + + schema_fixed = """{"type": "fixed", "name":"User", "size": 39}""" + # WHAT KIND OF VALUE SHOULD GO BELOW TO MAKE THIS WORK? + #sr_avro_serializer.serialize(b"\u00ff", schema=schema_fixed) + + @SchemaRegistryPowerShellPreparer() + def test_serialize_enum(self, schemaregistry_fully_qualified_namespace, schemaregistry_group, **kwargs): + sr_client = self.create_basic_client(SchemaRegistryClient, endpoint=schemaregistry_fully_qualified_namespace) + sr_avro_serializer = SchemaRegistryAvroSerializer(client=sr_client, group_name=schemaregistry_group, auto_register_schemas=True) + + schema_enum = """{ + "type": "enum", + "name": "Suit", + "symbols": ["SPADES", "HEARTS"] + }""" + data = "SPADES" + encoded_data = sr_avro_serializer.serialize(data, schema=schema_enum) + decoded_data = sr_avro_serializer.deserialize(encoded_data) + assert decoded_data == data + + @SchemaRegistryPowerShellPreparer() + def test_serialize_array(self, schemaregistry_fully_qualified_namespace, schemaregistry_group, **kwargs): + sr_client = self.create_basic_client(SchemaRegistryClient, endpoint=schemaregistry_fully_qualified_namespace) + sr_avro_serializer = SchemaRegistryAvroSerializer(client=sr_client, group_name=schemaregistry_group, auto_register_schemas=True) + + # mapping behaves similarly + schema_array = """{ + "type": "array", + "items": "string" + }""" + with pytest.raises(ValueError): # cannot serialize primitive + sr_avro_serializer.serialize(["hi"], schema=schema_array) + + @SchemaRegistryPowerShellPreparer() + def test_serialize_record(self, schemaregistry_fully_qualified_namespace, schemaregistry_group, **kwargs): + sr_client = self.create_basic_client(SchemaRegistryClient, endpoint=schemaregistry_fully_qualified_namespace) + sr_avro_serializer = SchemaRegistryAvroSerializer(client=sr_client, group_name=schemaregistry_group, auto_register_schemas=True) + + schema_record = """{ + "name":"User", + "namespace":"example.avro.populatedrecord", + "type":"record", + "fields":[ + {"name":"name","type":"string"}, + {"name":"age","type":"int"}, + {"name":"married","type":"boolean"}, + {"name":"height","type":"float"}, + {"name":"randb","type":"bytes"} + ] + }""" + # add below to schema later if possible + # {"name":"example.innerrec","type":"record","fields":[{"name":"a","type":"int"}]}, + # {"name":"innerenum","type":"enum","symbols":["FOO", "BAR"]}, + # {"name":"innerarray","type":"array","items":"int"}, + # {"name":"innermap","type":"map","values":"int"}, + # {"name":"innerfixed","type":"fixed","size":74} + data = { + "name": u"Ben", + "age": 3, + "married": False, + "height": 13.5, + "randb": b"\u00FF" + # "innerrec": {"a":1}, + # "innerenum": "FOO", + # "innerarray": [1], + # "innermap": {"a":1}, + # "innerfixed": "\u00ff" + } + + encoded_data = sr_avro_serializer.serialize(data, schema=schema_record) + decoded_data = sr_avro_serializer.deserialize(encoded_data) + assert decoded_data == data + + + # testing that "default" property in field does deserialize with default if field not in data + # below doesn't work, so what does "default" do? + #schema_record_without_default = """{ + # "name":"User", + # "namespace":"example.avro.withoutdefault", + # "type":"record", + # "fields":[ + # {"name":"name","type":"string"} + # ] + #}""" + #data = {"name":u"Ben"} + #encoded_data_with_envelope = sr_avro_serializer.serialize(data, schema=schema_record_without_default) + #encoded_data = encoded_data_with_envelope[36:] + + #schema_record_with_default = """{ + # "name":"User", + # "namespace":"example.avro.default", + # "type":"record", + # "fields":[ + # {"name":"name","type":"string"}, + # {"name":"age","type":"int","default":1} + # ] + #}""" + + #raw_avro_object_serializer = AvroObjectSerializer() + #decoded_data = raw_avro_object_serializer.deserialize(encoded_data, schema_record_with_default) + #assert decoded_data["age"] == 1 From ddc399fcf0f8c362c2b1887f4d515a322095c53f Mon Sep 17 00:00:00 2001 From: Swathi Pillalamarri Date: Fri, 1 Oct 2021 10:00:29 -0700 Subject: [PATCH 2/2] lint --- .../_schema_registry_avro_serializer.py | 42 ++++++++++++++----- 1 file changed, 32 insertions(+), 10 deletions(-) diff --git a/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avroserializer/_schema_registry_avro_serializer.py b/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avroserializer/_schema_registry_avro_serializer.py index 57f7e51a3192..c6722694e8ad 100644 --- a/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avroserializer/_schema_registry_avro_serializer.py +++ b/sdk/schemaregistry/azure-schemaregistry-avroserializer/azure/schemaregistry/serializer/avroserializer/_schema_registry_avro_serializer.py @@ -30,7 +30,12 @@ from io import BytesIO from typing import Any, Dict, Mapping import avro -from avro.schema import SchemaParseException, AvroException, InvalidName, PRIMITIVE_TYPES +from avro.schema import ( + SchemaParseException, + AvroException, + InvalidName, + PRIMITIVE_TYPES, +) from avro.io import AvroTypeException from .exceptions import SchemaParseException as AzureSchemaParseException @@ -56,16 +61,18 @@ def __init__(self, **kwargs): # type: (Any) -> None try: self._schema_group = kwargs.pop("group_name") - self._schema_registry_client = kwargs.pop("client") # type: "SchemaRegistryClient" + self._schema_registry_client = kwargs.pop( + "client" + ) # type: "SchemaRegistryClient" except KeyError as e: raise TypeError("'{}' is a required keyword.".format(e.args[0])) self._avro_serializer = AvroObjectSerializer(codec=kwargs.get("codec")) self._auto_register_schemas = kwargs.get("auto_register_schemas", False) self._auto_register_schema_func = ( - self._schema_registry_client.register_schema - if self._auto_register_schemas - else self._schema_registry_client.get_schema_id - ) + self._schema_registry_client.register_schema + if self._auto_register_schemas + else self._schema_registry_client.get_schema_id + ) self._user_input_schema_cache = {} def __enter__(self): @@ -142,20 +149,35 @@ def serialize(self, value, **kwargs): except KeyError: try: parsed_schema = avro.schema.parse(raw_input_schema) - except (SchemaParseException, AvroTypeException, AvroException, InvalidName) as e: + except ( + SchemaParseException, + AvroTypeException, + AvroException, + InvalidName, + ) as e: raise AzureSchemaParseException(e) self._user_input_schema_cache[raw_input_schema] = parsed_schema cached_schema = parsed_schema # TODO: check if there's a more pythonic way, if we need it at all if cached_schema.type in PRIMITIVE_TYPES: - raise ValueError("Expected schema of type `Record`, got instead primitive: {}".format(raw_input_schema)) + raise ValueError( + "Expected schema of type `Record`, got instead primitive: {}".format( + raw_input_schema + ) + ) record_format_identifier = b"\0\0\0\0" try: - schema_id = self._get_schema_id(cached_schema.fullname, str(cached_schema), **kwargs) + schema_id = self._get_schema_id( + cached_schema.fullname, str(cached_schema), **kwargs + ) except AttributeError: - raise ValueError("Expected schema of type `Record`, got instead schema of type: {}".format(cached_schema.type)) + raise ValueError( + "Expected schema of type `Record`, got instead schema of type: {}".format( + cached_schema.type + ) + ) data_bytes = self._avro_serializer.serialize(value, cached_schema) stream = BytesIO()