[Azure Search] Add several indexer features to SDK

src/SDKs/Search/DataPlane/Microsoft.Azure.Search.Service/Customizations/Indexers/Models/FieldMappingFunction.Customization.cs

@@ -13,30 +13,72 @@ public partial class FieldMappingFunction
         /// the input is UTF-8 encoded.
         /// </summary>
         /// <remarks>
+        /// <para>Sample use case: Only URL-safe characters can appear in an Azure Search document key (because customers
+        /// must be able to address the document using the Lookup API, for example). If your data contains URL-unsafe
+        /// characters and you want to use it to populate a key field in your search index, use this function. 
+        /// </para>
+        /// <para>
+        /// For details on the encoding used, see <see href="https://docs.microsoft.com/azure/search/search-indexer-field-mappings#details-of-base64-encoding-and-decoding"/>.
+        /// Calling this method is the same as calling <c cref="Base64Encode(bool)"/> with <c>useHttpServerUtilityUrlTokenEncode</c> set to <c>true</c>.
+        /// </para>
+        /// </remarks>
+        /// <returns>A new field mapping function.</returns>
+        public static FieldMappingFunction Base64Encode() => new FieldMappingFunction("base64Encode");
+
+        /// <summary>
+        /// Creates a field mapping function that performs URL-safe Base64 encoding of the input string. Assumes that
+        /// the input is UTF-8 encoded.
+        /// </summary>
+        /// <param name="useHttpServerUtilityUrlTokenEncode">Determines how Base64 encoding is performed. See <see href="https://docs.microsoft.com/azure/search/search-indexer-field-mappings#details-of-base64-encoding-and-decoding"/> for details.</param>
+        /// <remarks>
         /// Sample use case: Only URL-safe characters can appear in an Azure Search document key (because customers
         /// must be able to address the document using the Lookup API, for example). If your data contains URL-unsafe
         /// characters and you want to use it to populate a key field in your search index, use this function. 
         /// </remarks>
         /// <returns>A new field mapping function.</returns>
-        public static FieldMappingFunction Base64Encode()
-        {
-            return new FieldMappingFunction("base64Encode");
-        }
+        public static FieldMappingFunction Base64Encode(bool useHttpServerUtilityUrlTokenEncode) => 
+            new FieldMappingFunction(
+                "base64Encode", 
+                new Dictionary<string, object>
+                {
+                    [nameof(useHttpServerUtilityUrlTokenEncode)] = useHttpServerUtilityUrlTokenEncode
+                });
 
         /// <summary>
         /// Creates a field mapping function that performs Base64 decoding of the input string. The input is assumed
         /// to a URL-safe Base64-encoded string. 
         /// </summary>
         /// <remarks>
+        /// <para>Sample use case: Blob custom metadata values must be ASCII-encoded. You can use Base64 encoding to
+        /// represent arbitrary Unicode strings in blob custom metadata. However, to make search meaningful, you can
+        /// use this function to turn the encoded data back into "regular" strings when populating your search index. 
+        /// </para>
+        /// <para>
+        /// For details on the decoding used, see <see href="https://docs.microsoft.com/azure/search/search-indexer-field-mappings#details-of-base64-encoding-and-decoding"/>.
+        /// Calling this method is the same as calling <c cref="Base64Decode(bool)"/> with <c>useHttpServerUtilityUrlTokenDecode</c> set to <c>true</c>.
+        /// </para>
+        /// </remarks>
+        /// <returns>A new field mapping function.</returns>
+        public static FieldMappingFunction Base64Decode() => new FieldMappingFunction("base64Decode");
+
+        /// <summary>
+        /// Creates a field mapping function that performs Base64 decoding of the input string. The input is assumed
+        /// to a URL-safe Base64-encoded string. 
+        /// </summary>
+        /// <param name="useHttpServerUtilityUrlTokenDecode">Determines how Base64 decoding is performed. See <see href="https://docs.microsoft.com/azure/search/search-indexer-field-mappings#details-of-base64-encoding-and-decoding"/> for details.</param>
+        /// <remarks>
         /// Sample use case: Blob custom metadata values must be ASCII-encoded. You can use Base64 encoding to
         /// represent arbitrary Unicode strings in blob custom metadata. However, to make search meaningful, you can
         /// use this function to turn the encoded data back into "regular" strings when populating your search index. 
         /// </remarks>
         /// <returns>A new field mapping function.</returns>
-        public static FieldMappingFunction Base64Decode()
-        {
-            return new FieldMappingFunction("base64Decode");
-        }
+        public static FieldMappingFunction Base64Decode(bool useHttpServerUtilityUrlTokenDecode) =>
+            new FieldMappingFunction(
+                "base64Decode",
+                new Dictionary<string, object>
+                {
+                    [nameof(useHttpServerUtilityUrlTokenDecode)] = useHttpServerUtilityUrlTokenDecode
+                });
 
         /// <summary>
         /// Creates a field mapping function that splits a string field using the specified delimiter, and picks the
@@ -57,17 +99,14 @@ public static FieldMappingFunction Base64Decode()
         /// </para>
         /// </remarks>
         /// <returns>A new field mapping function.</returns>
-        public static FieldMappingFunction ExtractTokenAtPosition(string delimiter, int position)
-        {
-            var parameters = 
-                new Dictionary<string, object>()
+        public static FieldMappingFunction ExtractTokenAtPosition(string delimiter, int position) =>
+            new FieldMappingFunction(
+                "extractTokenAtPosition", 
+                new Dictionary<string, object>
                 {
-                    { nameof(delimiter), delimiter },
-                    { nameof(position), position }
-                };
-
-            return new FieldMappingFunction("extractTokenAtPosition", parameters);
-        }
+                    [nameof(delimiter)] = delimiter,
+                    [nameof(position)] = position
+                });
 
         /// <summary>
         /// Creates a field mapping function that transforms a string formatted as a JSON array of strings into a string array that can be used to
@@ -84,9 +123,6 @@ public static FieldMappingFunction ExtractTokenAtPosition(string delimiter, int
         /// </para>
         /// </remarks>
         /// <returns>A new field mapping function.</returns>
-        public static FieldMappingFunction JsonArrayToStringCollection()
-        {
-            return new FieldMappingFunction("jsonArrayToStringCollection");
-        }
+        public static FieldMappingFunction JsonArrayToStringCollection() => new FieldMappingFunction("jsonArrayToStringCollection");
     }
 }

src/SDKs/Search/DataPlane/Microsoft.Azure.Search.Service/Customizations/Indexers/Models/IndexingParametersExtensions.cs

@@ -7,6 +7,7 @@ namespace Microsoft.Azure.Search.Models
     using System;
     using System.Collections.Generic;
     using System.Linq;
+    using System.Text;
     using Common;
 
     /// <summary>
@@ -15,20 +16,6 @@ namespace Microsoft.Azure.Search.Models
     public static class IndexingParametersExtensions
     {
         private const string ParsingModeKey = "parsingMode";
-
-        /// <summary> 
-        /// Specifies that the indexer will index only the storage metadata and completely skip the document extraction process. This is useful when 
-        /// you don't need the document content, nor do you need any of the content type-specific metadata properties. 
-        /// See <see href="https://docs.microsoft.com/azure/search/search-howto-indexing-azure-blob-storage" /> for details. 
-        /// </summary> 
-        /// <param name="parameters">IndexingParameters to configure.</param> 
-        /// <remarks> 
-        /// This option only applies to indexers that index Azure Blob Storage. 
-        /// </remarks> 
-        /// <returns>The IndexingParameters instance.</returns> 
-        [Obsolete("This method is obsolete. Please use SetBlobExtractionMode(BlobExtractionMode.StorageMetadata).")]
-        public static IndexingParameters IndexStorageMetadataOnly(this IndexingParameters parameters) =>
-            parameters.SetBlobExtractionMode(BlobExtractionMode.StorageMetadata); 
 
         /// <summary>
         /// Specifies that the indexer will index only the blobs with the file name extensions you specify. Each string is a file extensions with a
@@ -80,20 +67,6 @@ public static IndexingParameters ExcludeFileNameExtensions(this IndexingParamete
             return parameters;
         }
 
-        /// <summary>
-        /// Specifies that the indexer will extract metadata, but skip content extraction for all blobs. If you want to skip content extraction for 
-        /// only some blobs, add AzureSearch_SkipContent metadata to those blobs individually instead of using this option. 
-        /// See <see href="https://docs.microsoft.com/azure/search/search-howto-indexing-azure-blob-storage" /> for details. 
-        /// </summary> 
-        /// <param name="parameters">IndexingParameters to configure.</param> 
-        /// <remarks> 
-        /// This option only applies to indexers that index Azure Blob Storage. 
-        /// </remarks> 
-        /// <returns>The IndexingParameters instance.</returns> 
-        [Obsolete("This method is obsolete. Please use SetBlobExtractionMode(BlobExtractionMode.AllMetadata).")]
-        public static IndexingParameters SkipContent(this IndexingParameters parameters) =>
-            parameters.SetBlobExtractionMode(BlobExtractionMode.AllMetadata); 
-
         /// <summary>
         /// Specifies which parts of a blob will be indexed by the blob storage indexer. 
         /// </summary>
@@ -176,16 +149,42 @@ public static IndexingParameters ParseDelimitedTextFiles(this IndexingParameters
             return parameters;
         }
 
+        /// <summary>
+        /// Tells the indexer to assume that blobs should be parsed as text files in UTF-8 encoding.
+        /// See <see href="https://docs.microsoft.com/azure/search/search-howto-indexing-azure-blob-storage#indexing-plain-text"/>
+        /// </summary>
+        /// <param name="parameters">IndexingParameters to configure.</param>
+        /// <returns>The IndexingParameters instance.</returns>
+        public static IndexingParameters ParseText(this IndexingParameters parameters) => 
+            ParseText(parameters, Encoding.UTF8);
+
+        /// <summary>
+        /// Tells the indexer to assume that blobs should be parsed as text files in the desired encoding.
+        /// See <see href="https://docs.microsoft.com/azure/search/search-howto-indexing-azure-blob-storage#indexing-plain-text"/>
+        /// </summary>
+        /// <param name="parameters">IndexingParameters to configure.</param>
+        /// <param name="encoding">Encoding used to read the text stored in blobs.</param>
+        /// <returns>The IndexingParameters instance.</returns>
+        public static IndexingParameters ParseText(this IndexingParameters parameters, Encoding encoding)
+        {
+            Throw.IfArgumentNull(encoding, nameof(encoding));
+
+            Configure(parameters, ParsingModeKey, "text");
+            Configure(parameters, "encoding", encoding.WebName);
+            return parameters;
+        }
+
         /// <summary>
         /// Specifies that <c cref="BlobExtractionMode.StorageMetadata">BlobExtractionMode.StorageMetadata</c> blob extraction mode will be
-        /// automatically used for blobs of unsupported content types. The default is false.
+        /// automatically used for blobs of unsupported content types. This behavior is enabled by default.
         /// </summary>
         /// <remarks>
         /// This option only applies to indexers that index Azure Blob Storage.
         /// </remarks>
         /// <param name="parameters">IndexingParameters to configure.</param>
         /// <returns></returns>
         /// <returns>The IndexingParameters instance.</returns>
+        [Obsolete("This behavior is enabled by default in API version 2016-09-01-Preview and calling this method is no longer necessary")]
         public static IndexingParameters DoNotFailOnUnsupportedContentType(this IndexingParameters parameters) =>
             Configure(parameters, "failOnUnsupportedContentType", false);
 

src/SDKs/Search/DataPlane/Microsoft.Azure.Search.Service/Generated/Models/IndexerExecutionResult.cs

@@ -41,6 +41,7 @@ public IndexerExecutionResult()
         /// <param name="endTime">The end time of this indexer execution, if
         /// the execution has already completed.</param>
         /// <param name="errors">The item-level indexing errors.</param>
+        /// <param name="warnings">The item-level indexing warnings.</param>
         /// <param name="itemCount">The number of items that were processed
         /// during this indexer execution. This includes both successfully
         /// processed items and items where indexing was attempted but
@@ -51,13 +52,14 @@ public IndexerExecutionResult()
         /// an indexer execution started.</param>
         /// <param name="finalTrackingState">Change tracking state with which
         /// an indexer execution finished.</param>
-        public IndexerExecutionResult(IndexerExecutionStatus status = default(IndexerExecutionStatus), string errorMessage = default(string), System.DateTimeOffset? startTime = default(System.DateTimeOffset?), System.DateTimeOffset? endTime = default(System.DateTimeOffset?), IList<ItemError> errors = default(IList<ItemError>), int itemCount = default(int), int failedItemCount = default(int), string initialTrackingState = default(string), string finalTrackingState = default(string))
+        public IndexerExecutionResult(IndexerExecutionStatus status = default(IndexerExecutionStatus), string errorMessage = default(string), System.DateTimeOffset? startTime = default(System.DateTimeOffset?), System.DateTimeOffset? endTime = default(System.DateTimeOffset?), IList<ItemError> errors = default(IList<ItemError>), IList<ItemWarning> warnings = default(IList<ItemWarning>), int itemCount = default(int), int failedItemCount = default(int), string initialTrackingState = default(string), string finalTrackingState = default(string))
         {
             Status = status;
             ErrorMessage = errorMessage;
             StartTime = startTime;
             EndTime = endTime;
             Errors = errors;
+            Warnings = warnings;
             ItemCount = itemCount;
             FailedItemCount = failedItemCount;
             InitialTrackingState = initialTrackingState;
@@ -102,6 +104,12 @@ public IndexerExecutionResult()
         [JsonProperty(PropertyName = "errors")]
         public IList<ItemError> Errors { get; private set; }
 
+        /// <summary>
+        /// Gets the item-level indexing warnings.
+        /// </summary>
+        [JsonProperty(PropertyName = "warnings")]
+        public IList<ItemWarning> Warnings { get; private set; }
+
         /// <summary>
         /// Gets the number of items that were processed during this indexer
         /// execution. This includes both successfully processed items and

src/SDKs/Search/DataPlane/Microsoft.Azure.Search.Service/Generated/Models/ItemError.cs

@@ -32,7 +32,7 @@ public ItemError()
         /// <param name="key">The key of the item for which indexing
         /// failed.</param>
         /// <param name="errorMessage">The message describing the error that
-        /// occurred while attempting to index the item.</param>
+        /// occurred while processing the item.</param>
         public ItemError(string key = default(string), string errorMessage = default(string))
         {
             Key = key;
@@ -53,7 +53,7 @@ public ItemError()
 
         /// <summary>
         /// Gets the message describing the error that occurred while
-        /// attempting to index the item.
+        /// processing the item.
         /// </summary>
         [JsonProperty(PropertyName = "errorMessage")]
         public string ErrorMessage { get; private set; }

src/SDKs/Search/DataPlane/Microsoft.Azure.Search.Service/Generated/Models/ItemWarning.cs

@@ -0,0 +1,62 @@
+// <auto-generated>
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License. See License.txt in the project root for
+// license information.
+//
+// Code generated by Microsoft (R) AutoRest Code Generator.
+// Changes may cause incorrect behavior and will be lost if the code is
+// regenerated.
+// </auto-generated>
+
+namespace Microsoft.Azure.Search.Models
+{
+    using Newtonsoft.Json;
+    using System.Linq;
+
+    /// <summary>
+    /// Represents an item-level warning.
+    /// </summary>
+    public partial class ItemWarning
+    {
+        /// <summary>
+        /// Initializes a new instance of the ItemWarning class.
+        /// </summary>
+        public ItemWarning()
+        {
+            CustomInit();
+        }
+
+        /// <summary>
+        /// Initializes a new instance of the ItemWarning class.
+        /// </summary>
+        /// <param name="key">The key of the item which generated a
+        /// warning.</param>
+        /// <param name="message">The message describing the warning that
+        /// occurred while processing the item.</param>
+        public ItemWarning(string key = default(string), string message = default(string))
+        {
+            Key = key;
+            Message = message;
+            CustomInit();
+        }
+
+        /// <summary>
+        /// An initialization method that performs custom operations like setting defaults
+        /// </summary>
+        partial void CustomInit();
+
+        /// <summary>
+        /// Gets the key of the item which generated a warning.
+        /// </summary>
+        [JsonProperty(PropertyName = "key")]
+        public string Key { get; private set; }
+
+        /// <summary>
+        /// Gets the message describing the warning that occurred while
+        /// processing the item.
+        /// </summary>
+        [JsonProperty(PropertyName = "message")]
+        public string Message { get; private set; }
+
+    }
+}