diff --git a/sdk/datafactory/Microsoft.Azure.Management.DataFactory/src/CHANGELOG.md b/sdk/datafactory/Microsoft.Azure.Management.DataFactory/src/CHANGELOG.md index a9b08fee6d64..f952e6243794 100644 --- a/sdk/datafactory/Microsoft.Azure.Management.DataFactory/src/CHANGELOG.md +++ b/sdk/datafactory/Microsoft.Azure.Management.DataFactory/src/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog for the Azure Data Factory V2 .NET SDK +## Version 4.10.0 +### Feature Additions +- Added new type conversion support in Copy activity + ## Version 4.9.0 ### Feature Additions - Added support for copying data to and from Snowflake using copy activity diff --git a/sdk/datafactory/Microsoft.Azure.Management.DataFactory/src/Extensions/TabularTranslator.cs b/sdk/datafactory/Microsoft.Azure.Management.DataFactory/src/Extensions/TabularTranslator.cs index 35a67c407028..c15d50bf35d1 100644 --- a/sdk/datafactory/Microsoft.Azure.Management.DataFactory/src/Extensions/TabularTranslator.cs +++ b/sdk/datafactory/Microsoft.Azure.Management.DataFactory/src/Extensions/TabularTranslator.cs @@ -10,10 +10,8 @@ namespace Microsoft.Azure.Management.DataFactory.Models { - using Newtonsoft.Json; - using System.Collections; using System.Collections.Generic; - using System.Linq; + using Newtonsoft.Json; /// /// A copy activity tabular translator. @@ -54,7 +52,11 @@ public TabularTranslator() /// {\"source\":{\"path\":\"$.CustomerAddress\",\"type\":\"String\"},\"sink\": /// {\"name\":\"ClientAddress\",\"type\":\"String\"}}]. Type: object /// (or Expression with resultType object). - public TabularTranslator(IDictionary additionalProperties = default(IDictionary), object columnMappings = default(object), object schemaMapping = default(object), object collectionReference = default(object), object mappings = default(object)) + /// Whether to enable the advanced type conversion + /// feature in the Copy activity. Type: boolean (or Expression with resultType + /// boolean). + /// Type conversion settings + public TabularTranslator(IDictionary additionalProperties = default(IDictionary), object columnMappings = default(object), object schemaMapping = default(object), object collectionReference = default(object), object mappings = default(object), object typeConversion = default(object), TypeConversionSettings typeConversionSettings = default(TypeConversionSettings)) : base(additionalProperties) { ColumnMappings = columnMappings; @@ -107,5 +109,18 @@ public TabularTranslator() /// [JsonProperty(PropertyName = "mappings")] public object Mappings { get; set; } + + /// + /// Gets or sets typeConversion indicating whether to enable the advanced type conversion + /// feature in the Copy activity. Type: boolean (or Expression with resultType boolean). + /// + [JsonProperty(PropertyName = "typeConversion")] + public object TypeConversion { get; set; } + + /// + /// Gets or sets typeConversionSettings. + /// + [JsonProperty(PropertyName = "typeConversionSettings")] + public TypeConversionSettings TypeConversionSettings { get; set; } } } diff --git a/sdk/datafactory/Microsoft.Azure.Management.DataFactory/src/Extensions/TypeConversionSettings.cs b/sdk/datafactory/Microsoft.Azure.Management.DataFactory/src/Extensions/TypeConversionSettings.cs new file mode 100644 index 000000000000..02c57496faa8 --- /dev/null +++ b/sdk/datafactory/Microsoft.Azure.Management.DataFactory/src/Extensions/TypeConversionSettings.cs @@ -0,0 +1,97 @@ +// +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for +// license information. +// +// Code generated by Microsoft (R) AutoRest Code Generator. +// Changes may cause incorrect behavior and will be lost if the code is +// regenerated. +// + + +namespace Microsoft.Azure.Management.DataFactory.Models +{ + using System.Collections.Generic; + using Newtonsoft.Json; + + public partial class TypeConversionSettings + { + /// + /// Initializes a new instance of the TypeConversionSettings class. + /// + public TypeConversionSettings() + { + CustomInit(); + } + + /// + /// Initializes a new instance of the TypeConversionSettings class. + /// + /// Unmatched properties from the message are deserialized this collection + /// Whether to allow data truncation when converting the data. Type: boolean (or Expression with resultType boolean). + /// Whether to treat boolean values as numbers. Type: boolean (or Expression with resultType boolean). + /// The format for DateTime values. Type: string (or Expression with resultType string). + /// The format for DateTimeOffset values. Type: string (or Expression with resultType string). + /// The format for TimeSpan values. Type: string (or Expression with resultType string). + /// The culture used to convert data from/to string. Type: string (or Expression with resultType string). + public TypeConversionSettings(IDictionary additionalProperties = default(IDictionary), object allowDataTruncation = default(object), object treatBooleanAsNumber = default(object), object dateTimeFormat = default(object), object dateTimeOffsetFormat = default(object), object timeSpanFormat = default(object), object culture = default(object)) + { + AdditionalProperties = additionalProperties; + AllowDataTruncation = AllowDataTruncation; + TreatBooleanAsNumber = treatBooleanAsNumber; + DateTimeFormat = dateTimeFormat; + DateTimeOffsetFormat = dateTimeOffsetFormat; + TimeSpanFormat = timeSpanFormat; + Culture = culture; + CustomInit(); + } + + /// + /// An initialization method that performs custom operations like setting defaults + /// + partial void CustomInit(); + + /// + /// Gets or sets unmatched properties from the message are deserialized + /// this collection + /// + [JsonExtensionData] + public IDictionary AdditionalProperties { get; set; } + + /// + /// Gets or sets allowDataTruncation indicating whether to allow data truncation when converting the data. Type: boolean (or Expression with resultType boolean). + /// + [JsonProperty(PropertyName = "allowDataTruncation")] + public object AllowDataTruncation { get; set; } + + /// + /// Gets or sets treatBooleanAsNumber indicating whether to treat boolean values as numbers. Type: boolean (or Expression with resultType boolean). + /// + [JsonProperty(PropertyName = "treatBooleanAsNumber")] + public object TreatBooleanAsNumber { get; set; } + + /// + /// Gets or sets the format for DateTime values. Type: string (or Expression with resultType string). + /// + [JsonProperty(PropertyName = "dateTimeFormat")] + public object DateTimeFormat { get; set; } + + /// + /// Gets or sets the format for DateTimeOffset values. Type: string (or Expression with resultType string). + /// + [JsonProperty(PropertyName = "dateTimeOffsetFormat")] + public object DateTimeOffsetFormat { get; set; } + + /// + /// Gets or sets the format for TimeSpan values. Type: string (or Expression with resultType string). + /// + [JsonProperty(PropertyName = "timeSpanFormat")] + public object TimeSpanFormat { get; set; } + + /// + /// Gets or sets the culture used to convert data from/to string. Type: string (or Expression with resultType string). + /// + [JsonProperty(PropertyName = "culture")] + public object Culture { get; set; } + } +} diff --git a/sdk/datafactory/Microsoft.Azure.Management.DataFactory/src/Generated/Models/ExcelDataset.cs b/sdk/datafactory/Microsoft.Azure.Management.DataFactory/src/Generated/Models/ExcelDataset.cs new file mode 100644 index 000000000000..7d28d92594b3 --- /dev/null +++ b/sdk/datafactory/Microsoft.Azure.Management.DataFactory/src/Generated/Models/ExcelDataset.cs @@ -0,0 +1,145 @@ +// +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for +// license information. +// +// Code generated by Microsoft (R) AutoRest Code Generator. +// Changes may cause incorrect behavior and will be lost if the code is +// regenerated. +// + +namespace Microsoft.Azure.Management.DataFactory.Models +{ + using Microsoft.Rest; + using Microsoft.Rest.Serialization; + using Newtonsoft.Json; + using System.Collections; + using System.Collections.Generic; + using System.Linq; + + /// + /// Excel dataset. + /// + [Newtonsoft.Json.JsonObject("Excel")] + [Rest.Serialization.JsonTransformation] + public partial class ExcelDataset : Dataset + { + /// + /// Initializes a new instance of the ExcelDataset class. + /// + public ExcelDataset() + { + LinkedServiceName = new LinkedServiceReference(); + CustomInit(); + } + + /// + /// Initializes a new instance of the ExcelDataset class. + /// + /// Linked service reference. + /// The location of the excel storage. + /// The sheet of excel file. Type: string (or + /// Expression with resultType string). + /// Unmatched properties from the + /// message are deserialized this collection + /// Dataset description. + /// Columns that define the structure of the + /// dataset. Type: array (or Expression with resultType array), + /// itemType: DatasetDataElement. + /// Columns that define the physical type schema + /// of the dataset. Type: array (or Expression with resultType array), + /// itemType: DatasetSchemaDataElement. + /// Parameters for dataset. + /// List of tags that can be used for + /// describing the Dataset. + /// The folder that this Dataset is in. If not + /// specified, Dataset will appear at the root level. + /// The partial data of one sheet. Type: string (or + /// Expression with resultType string). + /// When used as input, treat the first + /// row of data as headers. When used as output,write the headers into + /// the output as the first row of data. The default value is false. + /// Type: boolean (or Expression with resultType boolean). + /// The data compression method used for the + /// json dataset. + /// The null value string. Type: string (or + /// Expression with resultType string). + public ExcelDataset(LinkedServiceReference linkedServiceName, DatasetLocation location, object sheetName, IDictionary additionalProperties = default(IDictionary), string description = default(string), object structure = default(object), object schema = default(object), IDictionary parameters = default(IDictionary), IList annotations = default(IList), DatasetFolder folder = default(DatasetFolder), object range = default(object), object firstRowAsHeader = default(object), DatasetCompression compression = default(DatasetCompression), object nullValue = default(object)) + : base(linkedServiceName, additionalProperties, description, structure, schema, parameters, annotations, folder) + { + Location = location; + SheetName = sheetName; + Range = range; + FirstRowAsHeader = firstRowAsHeader; + Compression = compression; + NullValue = nullValue; + CustomInit(); + } + + /// + /// An initialization method that performs custom operations like setting defaults + /// + partial void CustomInit(); + + /// + /// Gets or sets the location of the excel storage. + /// + [JsonProperty(PropertyName = "typeProperties.location")] + public DatasetLocation Location { get; set; } + + /// + /// Gets or sets the sheet of excel file. Type: string (or Expression + /// with resultType string). + /// + [JsonProperty(PropertyName = "typeProperties.sheetName")] + public object SheetName { get; set; } + + /// + /// Gets or sets the partial data of one sheet. Type: string (or + /// Expression with resultType string). + /// + [JsonProperty(PropertyName = "typeProperties.range")] + public object Range { get; set; } + + /// + /// Gets or sets when used as input, treat the first row of data as + /// headers. When used as output,write the headers into the output as + /// the first row of data. The default value is false. Type: boolean + /// (or Expression with resultType boolean). + /// + [JsonProperty(PropertyName = "typeProperties.firstRowAsHeader")] + public object FirstRowAsHeader { get; set; } + + /// + /// Gets or sets the data compression method used for the json dataset. + /// + [JsonProperty(PropertyName = "typeProperties.compression")] + public DatasetCompression Compression { get; set; } + + /// + /// Gets or sets the null value string. Type: string (or Expression + /// with resultType string). + /// + [JsonProperty(PropertyName = "typeProperties.nullValue")] + public object NullValue { get; set; } + + /// + /// Validate the object. + /// + /// + /// Thrown if validation fails + /// + public override void Validate() + { + base.Validate(); + if (Location == null) + { + throw new ValidationException(ValidationRules.CannotBeNull, "Location"); + } + if (SheetName == null) + { + throw new ValidationException(ValidationRules.CannotBeNull, "SheetName"); + } + } + } +} diff --git a/sdk/datafactory/Microsoft.Azure.Management.DataFactory/src/Generated/Models/ExcelSource.cs b/sdk/datafactory/Microsoft.Azure.Management.DataFactory/src/Generated/Models/ExcelSource.cs new file mode 100644 index 000000000000..c7dd81315177 --- /dev/null +++ b/sdk/datafactory/Microsoft.Azure.Management.DataFactory/src/Generated/Models/ExcelSource.cs @@ -0,0 +1,77 @@ +// +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for +// license information. +// +// Code generated by Microsoft (R) AutoRest Code Generator. +// Changes may cause incorrect behavior and will be lost if the code is +// regenerated. +// + +namespace Microsoft.Azure.Management.DataFactory.Models +{ + using Newtonsoft.Json; + using System.Collections; + using System.Collections.Generic; + using System.Linq; + + /// + /// A copy activity excel source. + /// + [Newtonsoft.Json.JsonObject("ExcelSource")] + public partial class ExcelSource : CopySource + { + /// + /// Initializes a new instance of the ExcelSource class. + /// + public ExcelSource() + { + CustomInit(); + } + + /// + /// Initializes a new instance of the ExcelSource class. + /// + /// Unmatched properties from the + /// message are deserialized this collection + /// Source retry count. Type: integer + /// (or Expression with resultType integer). + /// Source retry wait. Type: string (or + /// Expression with resultType string), pattern: + /// ((\d+)\.)?(\d\d):(60|([0-5][0-9])):(60|([0-5][0-9])). + /// The maximum concurrent + /// connection count for the source data store. Type: integer (or + /// Expression with resultType integer). + /// Excel store settings. + /// Specifies the additional columns to + /// be added to source data. Type: array of objects (or Expression with + /// resultType array of objects). + public ExcelSource(IDictionary additionalProperties = default(IDictionary), object sourceRetryCount = default(object), object sourceRetryWait = default(object), object maxConcurrentConnections = default(object), StoreReadSettings storeSettings = default(StoreReadSettings), IList additionalColumns = default(IList)) + : base(additionalProperties, sourceRetryCount, sourceRetryWait, maxConcurrentConnections) + { + StoreSettings = storeSettings; + AdditionalColumns = additionalColumns; + CustomInit(); + } + + /// + /// An initialization method that performs custom operations like setting defaults + /// + partial void CustomInit(); + + /// + /// Gets or sets excel store settings. + /// + [JsonProperty(PropertyName = "storeSettings")] + public StoreReadSettings StoreSettings { get; set; } + + /// + /// Gets or sets specifies the additional columns to be added to source + /// data. Type: array of objects (or Expression with resultType array + /// of objects). + /// + [JsonProperty(PropertyName = "additionalColumns")] + public IList AdditionalColumns { get; set; } + + } +} diff --git a/sdk/datafactory/Microsoft.Azure.Management.DataFactory/src/Microsoft.Azure.Management.DataFactory.csproj b/sdk/datafactory/Microsoft.Azure.Management.DataFactory/src/Microsoft.Azure.Management.DataFactory.csproj index bafad5e7e6cb..2b12725873c4 100644 --- a/sdk/datafactory/Microsoft.Azure.Management.DataFactory/src/Microsoft.Azure.Management.DataFactory.csproj +++ b/sdk/datafactory/Microsoft.Azure.Management.DataFactory/src/Microsoft.Azure.Management.DataFactory.csproj @@ -5,18 +5,12 @@ Microsoft.Azure.Management.DataFactory Azure Data Factory V2 is the data integration platform that goes beyond Azure Data Factory V1's orchestration and batch-processing of time-series data, with a general purpose app model supporting modern data warehousing patterns and scenarios, lift-and-shift SSIS, and data-driven SaaS applications. Compose and manage reliable and secure data integration workflows at scale. Use native ADF data connectors and Integration Runtimes to move and transform cloud and on-premises data that can be unstructured, semi-structured, and structured with Hadoop, Azure Data Lake, Spark, SQL Server, Cosmos DB and many other data platforms. - 4.9.0 + 4.10.0 Microsoft.Azure.Management.DataFactory Microsoft Azure resource management;Data Factory;ADF; diff --git a/sdk/datafactory/Microsoft.Azure.Management.DataFactory/src/Properties/AssemblyInfo.cs b/sdk/datafactory/Microsoft.Azure.Management.DataFactory/src/Properties/AssemblyInfo.cs index 8c6c702fc1c3..4c746ec7f624 100644 --- a/sdk/datafactory/Microsoft.Azure.Management.DataFactory/src/Properties/AssemblyInfo.cs +++ b/sdk/datafactory/Microsoft.Azure.Management.DataFactory/src/Properties/AssemblyInfo.cs @@ -7,7 +7,7 @@ [assembly: AssemblyTitle("Microsoft Azure Data Factory Management Library")] [assembly: AssemblyDescription("Provides management functionality for Microsoft Azure Data Factory Resources.")] [assembly: AssemblyVersion("4.3.0.0")] -[assembly: AssemblyFileVersion("4.9.0.0")] +[assembly: AssemblyFileVersion("4.10.0.0")] [assembly: AssemblyConfiguration("")] [assembly: AssemblyCompany("Microsoft")] [assembly: AssemblyProduct("Microsoft Azure .NET SDK")] diff --git a/sdk/datafactory/Microsoft.Azure.Management.DataFactory/tests/JsonSamples/PipelineJsonSamples.cs b/sdk/datafactory/Microsoft.Azure.Management.DataFactory/tests/JsonSamples/PipelineJsonSamples.cs index 66a2de1c19fd..d609c363b9e1 100644 --- a/sdk/datafactory/Microsoft.Azure.Management.DataFactory/tests/JsonSamples/PipelineJsonSamples.cs +++ b/sdk/datafactory/Microsoft.Azure.Management.DataFactory/tests/JsonSamples/PipelineJsonSamples.cs @@ -6365,6 +6365,87 @@ public class PipelineJsonSamples : JsonSampleCollection ] } } +"; + + [JsonSample(version: "Copy")] + public const string CopyWithTypeConversion = @" +{ + ""name"": ""MyPipelineName"", + ""properties"": { + ""description"" : ""Copy from CSV to SQL DW with Copy Command"", + ""activities"": [ + { + ""type"": ""Copy"", + ""name"": ""TestActivity"", + ""description"": ""Test activity description"", + ""typeProperties"": { + ""source"": { + ""type"": ""DelimitedTextSource"", + ""storeSettings"": { + ""type"": ""AzureBlobStorageReadSettings"", + ""recursive"": true + }, + ""formatSettings"": { + ""type"": ""DelimitedTextReadSettings"", + ""rowDelimiter"": ""\n"", + ""quoteChar"": ""\"""", + ""escapeChar"": ""\"""" + } + }, + ""sink"": { + ""type"": ""SqlSink"" + }, + ""translator"": { + ""type"": ""TabularTranslator"", + ""mappings"": [ + { + ""source"": { + ""ordinal"": 3 + }, + ""sink"": { + ""name"": ""CustomerName"" + } + }, + { + ""source"": { + ""ordinal"": 2 + }, + ""sink"": { + ""name"": ""CustomerAddress"" + } + }, + { + ""source"": { + ""ordinal"": 1 + }, + ""sink"": { + ""name"": ""CustomerDate"" + } + } + ], + ""typeConversion"": true, + ""typeConversionSettings"": { + ""allowDataTruncation"": false, + ""dateTimeFormat"": ""MM/dd/yyyy HH:mm"" + } + } + }, + ""inputs"": [ + { + ""referenceName"": ""exampleDataset"", + ""type"": ""DatasetReference"" + } + ], + ""outputs"": [ + { + ""referenceName"": ""exampleDataset"", + ""type"": ""DatasetReference"" + } + ] + } + ] + } +} "; } }