diff --git a/sdk/datafactory/Azure.ResourceManager.DataFactory/api/Azure.ResourceManager.DataFactory.netstandard2.0.cs b/sdk/datafactory/Azure.ResourceManager.DataFactory/api/Azure.ResourceManager.DataFactory.netstandard2.0.cs index 803c7f25889c..4949eda1f209 100644 --- a/sdk/datafactory/Azure.ResourceManager.DataFactory/api/Azure.ResourceManager.DataFactory.netstandard2.0.cs +++ b/sdk/datafactory/Azure.ResourceManager.DataFactory/api/Azure.ResourceManager.DataFactory.netstandard2.0.cs @@ -1099,6 +1099,7 @@ public AvroDataset(Azure.Core.Expressions.DataFactory.DataFactoryLinkedServiceRe public Azure.Core.Expressions.DataFactory.DataFactoryElement AvroCompressionCodec { get { throw null; } set { } } public int? AvroCompressionLevel { get { throw null; } set { } } public Azure.ResourceManager.DataFactory.Models.DatasetLocation DataLocation { get { throw null; } set { } } + public new Azure.Core.Expressions.DataFactory.DataFactoryElement Schema { get { throw null; } set { } } Azure.ResourceManager.DataFactory.Models.AvroDataset System.ClientModel.Primitives.IJsonModel.Create(ref System.Text.Json.Utf8JsonReader reader, System.ClientModel.Primitives.ModelReaderWriterOptions options) { throw null; } void System.ClientModel.Primitives.IJsonModel.Write(System.Text.Json.Utf8JsonWriter writer, System.ClientModel.Primitives.ModelReaderWriterOptions options) { } Azure.ResourceManager.DataFactory.Models.AvroDataset System.ClientModel.Primitives.IPersistableModel.Create(System.BinaryData data, System.ClientModel.Primitives.ModelReaderWriterOptions options) { throw null; } @@ -6486,6 +6487,7 @@ public JsonDataset(Azure.Core.Expressions.DataFactory.DataFactoryLinkedServiceRe public Azure.ResourceManager.DataFactory.Models.DatasetCompression Compression { get { throw null; } set { } } public Azure.ResourceManager.DataFactory.Models.DatasetLocation DataLocation { get { throw null; } set { } } public Azure.Core.Expressions.DataFactory.DataFactoryElement EncodingName { get { throw null; } set { } } + public new Azure.Core.Expressions.DataFactory.DataFactoryElement Schema { get { throw null; } set { } } Azure.ResourceManager.DataFactory.Models.JsonDataset System.ClientModel.Primitives.IJsonModel.Create(ref System.Text.Json.Utf8JsonReader reader, System.ClientModel.Primitives.ModelReaderWriterOptions options) { throw null; } void System.ClientModel.Primitives.IJsonModel.Write(System.Text.Json.Utf8JsonWriter writer, System.ClientModel.Primitives.ModelReaderWriterOptions options) { } Azure.ResourceManager.DataFactory.Models.JsonDataset System.ClientModel.Primitives.IPersistableModel.Create(System.BinaryData data, System.ClientModel.Primitives.ModelReaderWriterOptions options) { throw null; } diff --git a/sdk/datafactory/Azure.ResourceManager.DataFactory/assets.json b/sdk/datafactory/Azure.ResourceManager.DataFactory/assets.json index 3bec5ebcf58a..3a456ab61121 100644 --- a/sdk/datafactory/Azure.ResourceManager.DataFactory/assets.json +++ b/sdk/datafactory/Azure.ResourceManager.DataFactory/assets.json @@ -2,5 +2,5 @@ "AssetsRepo": "Azure/azure-sdk-assets", "AssetsRepoPrefixPath": "net", "TagPrefix": "net/datafactory/Azure.ResourceManager.DataFactory", - "Tag": "net/datafactory/Azure.ResourceManager.DataFactory_23a911dae9" + "Tag": "net/datafactory/Azure.ResourceManager.DataFactory_8bebc7d7e6" } diff --git a/sdk/datafactory/Azure.ResourceManager.DataFactory/src/Customized/AvroDataset.cs b/sdk/datafactory/Azure.ResourceManager.DataFactory/src/Customized/AvroDataset.cs new file mode 100644 index 000000000000..f2d249e51c5c --- /dev/null +++ b/sdk/datafactory/Azure.ResourceManager.DataFactory/src/Customized/AvroDataset.cs @@ -0,0 +1,193 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#nullable disable + +using System; +using System.Collections.Generic; +using System.ComponentModel; +using System.Text.Json; +using Azure.Core; +using Azure.Core.Expressions.DataFactory; + +namespace Azure.ResourceManager.DataFactory.Models +{ + /// Avro dataset. + [CodeGenSuppress(nameof(AvroDataset), typeof(string), typeof(string), typeof(DataFactoryElement>), + typeof(DataFactoryElement), typeof(Core.Expressions.DataFactory.DataFactoryLinkedServiceReference), typeof(IDictionary), + typeof(IList), typeof(string), typeof(IDictionary), typeof(DatasetLocation), typeof(DataFactoryElement), typeof(int))] + public partial class AvroDataset : DataFactoryDatasetProperties + { + /// Initializes a new instance of . + /// Type of dataset. + /// Dataset description. + /// Columns that define the structure of the dataset. Type: array (or Expression with resultType array), itemType: DatasetDataElement. + /// Columns that define the physical type schema of the dataset. Type: array (or Expression with resultType array), itemType: DatasetSchemaDataElement. + /// Linked service reference. + /// Parameters for dataset. + /// List of tags that can be used for describing the Dataset. + /// The folder that this Dataset is in. If not specified, Dataset will appear at the root level. + /// Additional Properties. + /// + /// The location of the avro storage. + /// Please note is the base class. According to the scenario, a derived class of the base class might need to be assigned here, or this property needs to be casted to one of the possible derived classes. + /// The available derived classes include , , , , , , , , , , , , and . + /// + /// The data avroCompressionCodec. Type: string (or Expression with resultType string). + /// + internal AvroDataset(string datasetType, string description, DataFactoryElement> structure, DataFactoryElement schema, Core.Expressions.DataFactory.DataFactoryLinkedServiceReference linkedServiceName, IDictionary parameters, IList annotations, DatasetFolder folder, IDictionary additionalProperties, DatasetLocation dataLocation, DataFactoryElement avroCompressionCodec, int? avroCompressionLevel) : base(datasetType, description, structure, null, linkedServiceName, parameters, annotations, folder, additionalProperties) + { + DataLocation = dataLocation; + AvroCompressionCodec = avroCompressionCodec; + AvroCompressionLevel = avroCompressionLevel; + DatasetType = datasetType ?? "Avro"; + Schema = schema; + } + + /// Columns that define the physical type schema of the dataset. Type: array (or Expression with resultType array), itemType: DatasetSchemaDataElement. + public new DataFactoryElement Schema { get; set; } + + internal static AvroDataset DeserializeAvroDataset(JsonElement element) + { + if (element.ValueKind == JsonValueKind.Null) + { + return null; + } + string type = default; + Optional description = default; + Optional>> structure = default; + Optional> schema = default; + Core.Expressions.DataFactory.DataFactoryLinkedServiceReference linkedServiceName = default; + Optional> parameters = default; + Optional> annotations = default; + Optional folder = default; + Optional location = default; + Optional> avroCompressionCodec = default; + Optional avroCompressionLevel = default; + IDictionary additionalProperties = default; + Dictionary additionalPropertiesDictionary = new Dictionary(); + foreach (var property in element.EnumerateObject()) + { + if (property.NameEquals("type"u8)) + { + type = property.Value.GetString(); + continue; + } + if (property.NameEquals("description"u8)) + { + description = property.Value.GetString(); + continue; + } + if (property.NameEquals("structure"u8)) + { + if (property.Value.ValueKind == JsonValueKind.Null) + { + continue; + } + structure = JsonSerializer.Deserialize>>(property.Value.GetRawText()); + continue; + } + if (property.NameEquals("schema"u8)) + { + if (property.Value.ValueKind == JsonValueKind.Null) + { + continue; + } + schema = JsonSerializer.Deserialize>(property.Value.GetRawText()); + continue; + } + if (property.NameEquals("linkedServiceName"u8)) + { + linkedServiceName = JsonSerializer.Deserialize(property.Value.GetRawText()); + continue; + } + if (property.NameEquals("parameters"u8)) + { + if (property.Value.ValueKind == JsonValueKind.Null) + { + continue; + } + Dictionary dictionary = new Dictionary(); + foreach (var property0 in property.Value.EnumerateObject()) + { + dictionary.Add(property0.Name, EntityParameterSpecification.DeserializeEntityParameterSpecification(property0.Value)); + } + parameters = dictionary; + continue; + } + if (property.NameEquals("annotations"u8)) + { + if (property.Value.ValueKind == JsonValueKind.Null) + { + continue; + } + List array = new List(); + foreach (var item in property.Value.EnumerateArray()) + { + if (item.ValueKind == JsonValueKind.Null) + { + array.Add(null); + } + else + { + array.Add(BinaryData.FromString(item.GetRawText())); + } + } + annotations = array; + continue; + } + if (property.NameEquals("folder"u8)) + { + if (property.Value.ValueKind == JsonValueKind.Null) + { + continue; + } + folder = DatasetFolder.DeserializeDatasetFolder(property.Value); + continue; + } + if (property.NameEquals("typeProperties"u8)) + { + if (property.Value.ValueKind == JsonValueKind.Null) + { + property.ThrowNonNullablePropertyIsNull(); + continue; + } + foreach (var property0 in property.Value.EnumerateObject()) + { + if (property0.NameEquals("location"u8)) + { + if (property0.Value.ValueKind == JsonValueKind.Null) + { + continue; + } + location = DatasetLocation.DeserializeDatasetLocation(property0.Value); + continue; + } + if (property0.NameEquals("avroCompressionCodec"u8)) + { + if (property0.Value.ValueKind == JsonValueKind.Null) + { + continue; + } + avroCompressionCodec = JsonSerializer.Deserialize>(property0.Value.GetRawText()); + continue; + } + if (property0.NameEquals("avroCompressionLevel"u8)) + { + if (property0.Value.ValueKind == JsonValueKind.Null) + { + continue; + } + avroCompressionLevel = property0.Value.GetInt32(); + continue; + } + } + continue; + } + additionalPropertiesDictionary.Add(property.Name, BinaryData.FromString(property.Value.GetRawText())); + } + additionalProperties = additionalPropertiesDictionary; + return new AvroDataset(type, description.Value, structure.Value, schema.Value, linkedServiceName, Optional.ToDictionary(parameters), Optional.ToList(annotations), folder.Value, additionalProperties, location.Value, avroCompressionCodec.Value, Optional.ToNullable(avroCompressionLevel)); + } + } +} diff --git a/sdk/datafactory/Azure.ResourceManager.DataFactory/src/Customized/JsonDataset.cs b/sdk/datafactory/Azure.ResourceManager.DataFactory/src/Customized/JsonDataset.cs new file mode 100644 index 000000000000..c8f69b9edd1e --- /dev/null +++ b/sdk/datafactory/Azure.ResourceManager.DataFactory/src/Customized/JsonDataset.cs @@ -0,0 +1,193 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#nullable disable + +using System; +using System.Collections.Generic; +using System.ComponentModel; +using System.Text.Json; +using Azure.Core; +using Azure.Core.Expressions.DataFactory; + +namespace Azure.ResourceManager.DataFactory.Models +{ + /// Json dataset. + [CodeGenSuppress(nameof(JsonDataset), typeof(string), typeof(string), typeof(DataFactoryElement>), + typeof(DataFactoryElement), typeof(Core.Expressions.DataFactory.DataFactoryLinkedServiceReference), typeof(IDictionary), + typeof(IList), typeof(string), typeof(IDictionary), typeof(DatasetLocation), typeof(DataFactoryElement), typeof(DatasetCompression))] + public partial class JsonDataset : DataFactoryDatasetProperties + { + /// Initializes a new instance of . + /// Type of dataset. + /// Dataset description. + /// Columns that define the structure of the dataset. Type: array (or Expression with resultType array), itemType: DatasetDataElement. + /// Columns that define the physical type schema of the dataset. Type: array (or Expression with resultType array), itemType: DatasetSchemaDataElement. + /// Linked service reference. + /// Parameters for dataset. + /// List of tags that can be used for describing the Dataset. + /// The folder that this Dataset is in. If not specified, Dataset will appear at the root level. + /// Additional Properties. + /// + /// The location of the json data storage. + /// Please note is the base class. According to the scenario, a derived class of the base class might need to be assigned here, or this property needs to be casted to one of the possible derived classes. + /// The available derived classes include , , , , , , , , , , , , and . + /// + /// The code page name of the preferred encoding. If not specified, the default value is UTF-8, unless BOM denotes another Unicode encoding. Refer to the name column of the table in the following link to set supported values: https://msdn.microsoft.com/library/system.text.encoding.aspx. Type: string (or Expression with resultType string). + /// The data compression method used for the json dataset. + internal JsonDataset(string datasetType, string description, DataFactoryElement> structure, DataFactoryElement schema, Core.Expressions.DataFactory.DataFactoryLinkedServiceReference linkedServiceName, IDictionary parameters, IList annotations, DatasetFolder folder, IDictionary additionalProperties, DatasetLocation dataLocation, DataFactoryElement encodingName, DatasetCompression compression) : base(datasetType, description, structure, null, linkedServiceName, parameters, annotations, folder, additionalProperties) + { + DataLocation = dataLocation; + EncodingName = encodingName; + Compression = compression; + Schema = schema; + DatasetType = datasetType ?? "Json"; + } + + /// Columns that define the physical type schema of the dataset. Type: array (or Expression with resultType array), itemType: DatasetSchemaDataElement. + public new DataFactoryElement Schema { get; set; } + + internal static JsonDataset DeserializeJsonDataset(JsonElement element) + { + if (element.ValueKind == JsonValueKind.Null) + { + return null; + } + string type = default; + Optional description = default; + Optional>> structure = default; + Optional> schema = default; + Core.Expressions.DataFactory.DataFactoryLinkedServiceReference linkedServiceName = default; + Optional> parameters = default; + Optional> annotations = default; + Optional folder = default; + Optional location = default; + Optional> encodingName = default; + Optional compression = default; + IDictionary additionalProperties = default; + Dictionary additionalPropertiesDictionary = new Dictionary(); + foreach (var property in element.EnumerateObject()) + { + if (property.NameEquals("type"u8)) + { + type = property.Value.GetString(); + continue; + } + if (property.NameEquals("description"u8)) + { + description = property.Value.GetString(); + continue; + } + if (property.NameEquals("structure"u8)) + { + if (property.Value.ValueKind == JsonValueKind.Null) + { + continue; + } + structure = JsonSerializer.Deserialize>>(property.Value.GetRawText()); + continue; + } + if (property.NameEquals("schema"u8)) + { + if (property.Value.ValueKind == JsonValueKind.Null) + { + continue; + } + schema = JsonSerializer.Deserialize>(property.Value.GetRawText()); + continue; + } + if (property.NameEquals("linkedServiceName"u8)) + { + linkedServiceName = JsonSerializer.Deserialize(property.Value.GetRawText()); + continue; + } + if (property.NameEquals("parameters"u8)) + { + if (property.Value.ValueKind == JsonValueKind.Null) + { + continue; + } + Dictionary dictionary = new Dictionary(); + foreach (var property0 in property.Value.EnumerateObject()) + { + dictionary.Add(property0.Name, EntityParameterSpecification.DeserializeEntityParameterSpecification(property0.Value)); + } + parameters = dictionary; + continue; + } + if (property.NameEquals("annotations"u8)) + { + if (property.Value.ValueKind == JsonValueKind.Null) + { + continue; + } + List array = new List(); + foreach (var item in property.Value.EnumerateArray()) + { + if (item.ValueKind == JsonValueKind.Null) + { + array.Add(null); + } + else + { + array.Add(BinaryData.FromString(item.GetRawText())); + } + } + annotations = array; + continue; + } + if (property.NameEquals("folder"u8)) + { + if (property.Value.ValueKind == JsonValueKind.Null) + { + continue; + } + folder = DatasetFolder.DeserializeDatasetFolder(property.Value); + continue; + } + if (property.NameEquals("typeProperties"u8)) + { + if (property.Value.ValueKind == JsonValueKind.Null) + { + property.ThrowNonNullablePropertyIsNull(); + continue; + } + foreach (var property0 in property.Value.EnumerateObject()) + { + if (property0.NameEquals("location"u8)) + { + if (property0.Value.ValueKind == JsonValueKind.Null) + { + continue; + } + location = DatasetLocation.DeserializeDatasetLocation(property0.Value); + continue; + } + if (property0.NameEquals("encodingName"u8)) + { + if (property0.Value.ValueKind == JsonValueKind.Null) + { + continue; + } + encodingName = JsonSerializer.Deserialize>(property0.Value.GetRawText()); + continue; + } + if (property0.NameEquals("compression"u8)) + { + if (property0.Value.ValueKind == JsonValueKind.Null) + { + continue; + } + compression = DatasetCompression.DeserializeDatasetCompression(property0.Value); + continue; + } + } + continue; + } + additionalPropertiesDictionary.Add(property.Name, BinaryData.FromString(property.Value.GetRawText())); + } + additionalProperties = additionalPropertiesDictionary; + return new JsonDataset(type, description.Value, structure.Value, schema.Value, linkedServiceName, Optional.ToDictionary(parameters), Optional.ToList(annotations), folder.Value, additionalProperties, location.Value, encodingName.Value, compression.Value); + } + } +} diff --git a/sdk/datafactory/Azure.ResourceManager.DataFactory/tests/Scenario/DataFactoryDatasetResourceTests.cs b/sdk/datafactory/Azure.ResourceManager.DataFactory/tests/Scenario/DataFactoryDatasetResourceTests.cs index 0f36f4c686d6..094d19294cd9 100644 --- a/sdk/datafactory/Azure.ResourceManager.DataFactory/tests/Scenario/DataFactoryDatasetResourceTests.cs +++ b/sdk/datafactory/Azure.ResourceManager.DataFactory/tests/Scenario/DataFactoryDatasetResourceTests.cs @@ -2213,5 +2213,41 @@ await DatasetCreate("sharepoint", CreateSharePointOnlineListLinkedService, (stri }); }); } + + [Test] + [RecordedTest] + public async Task Dataset_Avro_Create() + { + await DatasetCreate("avro", CreateAzureBlobStorageLinkedService, (string linkedServiceName) => + { + return new DataFactoryDatasetData(new AvroDataset(new DataFactoryLinkedServiceReference(DataFactoryLinkedServiceReferenceType.LinkedServiceReference, linkedServiceName)) + { + Schema = DataFactoryElement.FromLiteral(BinaryData.FromString("{\"type\": \"record\",\"name\": \"HybridDelivery.ClientLibraryJob\",\"fields\": [{\"name\": \"TIMESTAMP\",\"type\": [\"string\",\"null\"]}]}")), + DataLocation = new AzureBlobStorageLocation() + { + FileName = "TestQuerySchema.avro", + FolderPath = "querytest" + } + }); + }); + } + + [Test] + [RecordedTest] + public async Task Dataset_Json_Create() + { + await DatasetCreate("json", CreateAzureBlobStorageLinkedService, (string linkedServiceName) => + { + return new DataFactoryDatasetData(new JsonDataset(new DataFactoryLinkedServiceReference(DataFactoryLinkedServiceReferenceType.LinkedServiceReference, linkedServiceName)) + { + Schema = DataFactoryElement.FromLiteral(BinaryData.FromString("{\"type\": \"object\",\"properties\": {\"studentName\": {\"type\": \"string\"},\"age\": {\"type\": \"integer\"},\"gender\": {\"type\": \"string\"},\"studentID\": {\"type\": \"string\"},\"major\": {\"type\": \"string\"},\"grades\": {\"type\": \"object\",\"properties\": {\"math\": {\"type\": \"integer\"},\"english\": {\"type\": \"integer\"},\"programming\": {\"type\": \"integer\"}}},\"contact\": {\"type\":\"object\",\"properties\": {\"phone\": {\"type\": \"string\"},\"email\": {\"type\": \"string\"}}}}}")), + DataLocation = new AzureBlobStorageLocation() + { + FileName = "TestQuerySchema.json", + FolderPath = "querytest" + } + }); + }); + } } }