From 74a3235453cbdbf6203761a575cabda30f7d914d Mon Sep 17 00:00:00 2001 From: Jennifer Marsman Date: Wed, 4 Mar 2020 21:13:57 -0500 Subject: [PATCH 01/38] Removed color and imageType from the visualFeature enumeration in the V2 SDK. These weren't referenced in the examples. --- .../Azure.Search/preview/2019-05-06-preview/searchservice.json | 2 -- 1 file changed, 2 deletions(-) diff --git a/specification/search/data-plane/Azure.Search/preview/2019-05-06-preview/searchservice.json b/specification/search/data-plane/Azure.Search/preview/2019-05-06-preview/searchservice.json index c8cd33339a40..74a006942dc2 100644 --- a/specification/search/data-plane/Azure.Search/preview/2019-05-06-preview/searchservice.json +++ b/specification/search/data-plane/Azure.Search/preview/2019-05-06-preview/searchservice.json @@ -5514,10 +5514,8 @@ "adult", "brands", "categories", - "color", "description", "faces", - "imageType", "objects", "tags" ], From bf3f6b8e7ee64a14e2f8c9bbc3067d79fa533d9f Mon Sep 17 00:00:00 2001 From: Jennifer Marsman Date: Sat, 20 Jun 2020 01:59:05 -0400 Subject: [PATCH 02/38] Added indexing parameters to the swagger for Cognitive Search --- .../2019-05-06-preview/searchservice.json | 210 +++++++++++++++++- 1 file changed, 209 insertions(+), 1 deletion(-) diff --git a/specification/search/data-plane/Azure.Search/preview/2019-05-06-preview/searchservice.json b/specification/search/data-plane/Azure.Search/preview/2019-05-06-preview/searchservice.json index c2a69aac29b9..edb13c87505a 100644 --- a/specification/search/data-plane/Azure.Search/preview/2019-05-06-preview/searchservice.json +++ b/specification/search/data-plane/Azure.Search/preview/2019-05-06-preview/searchservice.json @@ -5771,6 +5771,71 @@ "default": 0, "description": "The maximum number of items in a single batch that can fail indexing for the batch to still be considered successful. -1 means no limit. Default is 0." }, + "parsingMode": { + "type": "string", + "$ref": "#/definitions/ParsingMode" + }, + "excludedFileNameExtensions": { + "type": "string", + "description": "Comma-delimited list of filename extensions to ignore when processing from Azure blob storage. For example, you could exclude \".png, .mp4\" to skip over those files during indexing." + }, + "indexedFileNameExtensions": { + "type": "string", + "description": "Comma-delimited list of filename extensions to select when processing from Azure blob storage. For example, you could focus indexing on specific application files \".docx, .pptx, .msg\" to specifically include those file types." + }, + "failOnUnsupportedContentType": { + "type": "boolean", + "default": true, + "description": "For Azure blobs, set to false if you want to continue indexing when an unsupported content type is encountered, and you don't know all the content types (file extensions) in advance." + }, + "failOnUnprocessableDocument": { + "type": "boolean", + "default": true, + "description": "For Azure blobs, set to false if you want to continue indexing if a document fails indexing." + }, + "indexStorageMetadataOnlyForOversizedDocuments": { + "type": "boolean", + "default": false, + "description": "For Azure blobs, set this property to true to still index storage metadata for blob content that is too large to process. Oversized blobs are treated as errors by default. For limits on blob size, see https://docs.microsoft.com/azure/search/search-limits-quotas-capacity." + }, + "delimitedTextHeaders": { + "type": "string", + "description": "For CSV blobs, specifies a comma-delimited list of column headers, useful for mapping source fields to destination fields in an index." + }, + "delimitedTextDelimiter": { + "type": "string", + "description": "For CSV blobs, specifies the end-of-line single-character delimiter for CSV files where each line starts a new document (for example, \"|\")." + }, + "firstLineContainsHeaders": { + "type": "boolean", + "default": true, + "description": "For CSV blobs, indicates that the first (non-blank) line of each blob contains headers." + }, + "documentRoot": { + "type": "string", + "description": "For JSON arrays, given a structured or semi-structured document, you can specify a path to the array using this property." + }, + "dataToExtract": { + "type": "string", + "$ref": "#/definitions/DataToExtract" + }, + "imageAction": { + "type": "string", + "$ref": "#/definitions/ImageAction" + }, + "allowSkillsetToReadFileData": { + "type": "boolean", + "default": false, + "description": "If true, will create a path //document//file_data that is an object representing the original file data downloaded from your blob data source. This allows you to pass the original file data to a custom skill for processing within the enrichment pipeline, or to the Document Extraction skill. The object generated will be defined as follows: { /"$type/": /"file/", /"data/": /"BASE64 encoded string of the file/" }" + }, + "pdfTextRotationAlgorithm": { + "type": "string", + "$ref": "#/definitions/PdfTextRotationAlgorithm" + }, + "queryTimeout": { + "type": "string", + "description": "Increases the timeout beyond the 5-minute default for Azure SQL database data sources, specified in the format \"hh:mm:ss\"." + }, "configuration": { "type": "object", "additionalProperties": { @@ -5779,7 +5844,150 @@ "description": "A dictionary of indexer-specific configuration properties. Each name is the name of a specific property. Each value must be of a primitive type." } }, - "description": "Represents parameters for indexer execution." + "description": "Represents parameters for indexer execution.", + "externalDocs": { + "url": "https://docs.microsoft.com/en-us/rest/api/searchservice/create-indexer#parameters" + } + }, + "ParsingMode": { + "type": "string", + "enum": [ + "default", + "text", + "delimitedText", + "json", + "jsonArray", + "jsonLines" + ], + "x-ms-enum": { + "name": "BlobIndexerParsingMode", + "modelAsString": true, + "values": [ + { + "value": "default", + "name": "Default", + "description": "Set to default for normal file processing." + }, + { + "value": "text", + "name": "Text", + "description": "Set to text to improve indexing performance on plain text files in blob storage." + }, + { + "value": "delimitedText", + "name": "DelimitedText", + "description": "Set to delimitedText when blobs are plain CSV files." + }, + { + "value": "json", + "name": "Json", + "description": "Set to json to extract structured content from JSON files." + }, + { + "value": "jsonArray", + "name": "JsonArray", + "description": "Set to jsonArray to extract individual elements of a JSON array as separate documents in Azure Cognitive Search." + }, + { + "value": "jsonLines", + "name": "JsonLines", + "description": "Set to jsonLines to extract individual JSON entities, separated by a new line, as separate documents in Azure Cognitive Search." + } + ] + }, + "x-nullable": true, + "default": "default", + "description": "Represents the parsing mode for indexing from an Azure blob data source." + }, + "DataToExtract": { + "type": "string", + "enum": [ + "storageMetadata", + "allMetadata", + "contentAndMetadata" + ], + "x-ms-enum": { + "name": "BlobIndexerDataToExtract", + "modelAsString": true, + "values": [ + { + "value": "storageMetadata", + "name": "StorageMetadata", + "description": "Indexes just the standard blob properties and user-specified metadata." + }, + { + "value": "allMetadata", + "name": "AllMetadata", + "description": "Extracts metadata provided by the Azure blob storage subsystem and the content-type specific metadata (for example, metadata unique to just .png files are indexed)." + }, + { + "value": "contentAndMetadata", + "name": "ContentAndMetadata", + "description": "Extracts all metadata and textual content from each blob." + } + ] + }, + "x-nullable": true, + "default": "contentAndMetadata", + "description": "Specifies the data to extract from Azure blob storage and tells the indexer which data to extract from image content when \"imageAction\" is set to a value other than \"none\". This applies to embedded image content in a .PDF or other application, or image files such as .jpg and .png, in Azure blobs." + }, + "ImageAction": { + "type": "string", + "enum": [ + "none", + "generateNormalizedImages", + "generateNormalizedImagePerPage" + ], + "x-ms-enum": { + "name": "BlobIndexerImageAction", + "modelAsString": true, + "values": [ + { + "value": "none", + "name": "None", + "description": "Ignores embedded images or image files in the data set. This is the default." + }, + { + "value": "generateNormalizedImages", + "name": "GenerateNormalizedImages", + "description": "Extracts text from images (for example, the word \"STOP\" from a traffic stop sign), and embeds it into the content field. This action requires that \"dataToExtract\" is set to \"contentAndMetadata\". A normalized image refers to additional processing resulting in uniform image output, sized and rotated to promote consistent rendering when you include images in visual search results. This information is generated for each image when you use this option." + }, + { + "value": "generateNormalizedImagePerPage", + "name": "GenerateNormalizedImagePerPage", + "description": "Extracts text from images (for example, the word \"STOP\" from a traffic stop sign), and embeds it into the content field, but treats PDF files differently in that each page will be rendered as an image and normalized accordingly, instead of extracting embedded images. Non-PDF file types will be treated the same as if \"generateNormalizedImages\" was set." + } + ] + }, + "x-nullable": true, + "default": "none", + "description": "Determines how to process embedded images and image files in Azure blob storage. Setting the \"imageAction\" configuration to any value other than \"none\" requires that a skillset also be attached to that indexer." + }, + "PdfTextRotationAlgorithm": { + "type": "string", + "enum": [ + "none", + "detectAngles" + ], + "x-ms-enum": { + "name": "BlobIndexerPDFTextRotationAlgorithm", + "modelAsString": true, + "values": [ + { + "value": "none", + "name": "None", + "description": "Leverages normal text extraction. This is the default." + }, + { + "value": "detectAngles", + "name": "DetectAngles", + "description": "May produce better and more readable text extraction from PDF files that have rotated text within them. Note that there may be a small performance speed impact when this parameter is used. This parameter only applies to PDF files, and only to PDFs with embedded text. If the rotated text appears within an embedded image in the PDF, this parameter does not apply." + } + ] + }, + "x-nullable": true, + "default": "none", + "description": "Determines algorithm for text extraction from PDF files in Azure blob storage." }, "FieldMappingFunction": { "properties": { From dfe05dffdc66c97a9d0229a9fe34456fb73f7444 Mon Sep 17 00:00:00 2001 From: Jennifer Marsman Date: Wed, 24 Jun 2020 21:47:30 -0400 Subject: [PATCH 03/38] Moved properties under configuration --- .../2019-05-06-preview/searchservice.json | 133 +++++++++--------- 1 file changed, 67 insertions(+), 66 deletions(-) diff --git a/specification/search/data-plane/Azure.Search/preview/2019-05-06-preview/searchservice.json b/specification/search/data-plane/Azure.Search/preview/2019-05-06-preview/searchservice.json index edb13c87505a..567a0548e26c 100644 --- a/specification/search/data-plane/Azure.Search/preview/2019-05-06-preview/searchservice.json +++ b/specification/search/data-plane/Azure.Search/preview/2019-05-06-preview/searchservice.json @@ -5771,73 +5771,74 @@ "default": 0, "description": "The maximum number of items in a single batch that can fail indexing for the batch to still be considered successful. -1 means no limit. Default is 0." }, - "parsingMode": { - "type": "string", - "$ref": "#/definitions/ParsingMode" - }, - "excludedFileNameExtensions": { - "type": "string", - "description": "Comma-delimited list of filename extensions to ignore when processing from Azure blob storage. For example, you could exclude \".png, .mp4\" to skip over those files during indexing." - }, - "indexedFileNameExtensions": { - "type": "string", - "description": "Comma-delimited list of filename extensions to select when processing from Azure blob storage. For example, you could focus indexing on specific application files \".docx, .pptx, .msg\" to specifically include those file types." - }, - "failOnUnsupportedContentType": { - "type": "boolean", - "default": true, - "description": "For Azure blobs, set to false if you want to continue indexing when an unsupported content type is encountered, and you don't know all the content types (file extensions) in advance." - }, - "failOnUnprocessableDocument": { - "type": "boolean", - "default": true, - "description": "For Azure blobs, set to false if you want to continue indexing if a document fails indexing." - }, - "indexStorageMetadataOnlyForOversizedDocuments": { - "type": "boolean", - "default": false, - "description": "For Azure blobs, set this property to true to still index storage metadata for blob content that is too large to process. Oversized blobs are treated as errors by default. For limits on blob size, see https://docs.microsoft.com/azure/search/search-limits-quotas-capacity." - }, - "delimitedTextHeaders": { - "type": "string", - "description": "For CSV blobs, specifies a comma-delimited list of column headers, useful for mapping source fields to destination fields in an index." - }, - "delimitedTextDelimiter": { - "type": "string", - "description": "For CSV blobs, specifies the end-of-line single-character delimiter for CSV files where each line starts a new document (for example, \"|\")." - }, - "firstLineContainsHeaders": { - "type": "boolean", - "default": true, - "description": "For CSV blobs, indicates that the first (non-blank) line of each blob contains headers." - }, - "documentRoot": { - "type": "string", - "description": "For JSON arrays, given a structured or semi-structured document, you can specify a path to the array using this property." - }, - "dataToExtract": { - "type": "string", - "$ref": "#/definitions/DataToExtract" - }, - "imageAction": { - "type": "string", - "$ref": "#/definitions/ImageAction" - }, - "allowSkillsetToReadFileData": { - "type": "boolean", - "default": false, - "description": "If true, will create a path //document//file_data that is an object representing the original file data downloaded from your blob data source. This allows you to pass the original file data to a custom skill for processing within the enrichment pipeline, or to the Document Extraction skill. The object generated will be defined as follows: { /"$type/": /"file/", /"data/": /"BASE64 encoded string of the file/" }" - }, - "pdfTextRotationAlgorithm": { - "type": "string", - "$ref": "#/definitions/PdfTextRotationAlgorithm" - }, - "queryTimeout": { - "type": "string", - "description": "Increases the timeout beyond the 5-minute default for Azure SQL database data sources, specified in the format \"hh:mm:ss\"." - }, "configuration": { - "type": "object", + "properties": { + "parsingMode": { + "type": "string", + "$ref": "#/definitions/ParsingMode" + }, + "excludedFileNameExtensions": { + "type": "string", + "description": "Comma-delimited list of filename extensions to ignore when processing from Azure blob storage. For example, you could exclude \".png, .mp4\" to skip over those files during indexing." + }, + "indexedFileNameExtensions": { + "type": "string", + "description": "Comma-delimited list of filename extensions to select when processing from Azure blob storage. For example, you could focus indexing on specific application files \".docx, .pptx, .msg\" to specifically include those file types." + }, + "failOnUnsupportedContentType": { + "type": "boolean", + "default": true, + "description": "For Azure blobs, set to false if you want to continue indexing when an unsupported content type is encountered, and you don't know all the content types (file extensions) in advance." + }, + "failOnUnprocessableDocument": { + "type": "boolean", + "default": true, + "description": "For Azure blobs, set to false if you want to continue indexing if a document fails indexing." + }, + "indexStorageMetadataOnlyForOversizedDocuments": { + "type": "boolean", + "default": false, + "description": "For Azure blobs, set this property to true to still index storage metadata for blob content that is too large to process. Oversized blobs are treated as errors by default. For limits on blob size, see https://docs.microsoft.com/azure/search/search-limits-quotas-capacity." + }, + "delimitedTextHeaders": { + "type": "string", + "description": "For CSV blobs, specifies a comma-delimited list of column headers, useful for mapping source fields to destination fields in an index." + }, + "delimitedTextDelimiter": { + "type": "string", + "description": "For CSV blobs, specifies the end-of-line single-character delimiter for CSV files where each line starts a new document (for example, \"|\")." + }, + "firstLineContainsHeaders": { + "type": "boolean", + "default": true, + "description": "For CSV blobs, indicates that the first (non-blank) line of each blob contains headers." + }, + "documentRoot": { + "type": "string", + "description": "For JSON arrays, given a structured or semi-structured document, you can specify a path to the array using this property." + }, + "dataToExtract": { + "type": "string", + "$ref": "#/definitions/DataToExtract" + }, + "imageAction": { + "type": "string", + "$ref": "#/definitions/ImageAction" + }, + "allowSkillsetToReadFileData": { + "type": "boolean", + "default": false, + "description": "If true, will create a path //document//file_data that is an object representing the original file data downloaded from your blob data source. This allows you to pass the original file data to a custom skill for processing within the enrichment pipeline, or to the Document Extraction skill. The object generated will be defined as follows: { /"$type/": /"file/", /"data/": /"BASE64 encoded string of the file/" }" + }, + "pdfTextRotationAlgorithm": { + "type": "string", + "$ref": "#/definitions/PdfTextRotationAlgorithm" + }, + "queryTimeout": { + "type": "string", + "description": "Increases the timeout beyond the 5-minute default for Azure SQL database data sources, specified in the format \"hh:mm:ss\"." + } + }, "additionalProperties": { "type": "object" }, From 67f10019b2014e499e84133b3f2c341a9a4d68e4 Mon Sep 17 00:00:00 2001 From: Jennifer Marsman Date: Wed, 24 Jun 2020 23:14:15 -0400 Subject: [PATCH 04/38] Removed bad formatting so JSON is valid --- .../Azure.Search/preview/2019-05-06-preview/searchservice.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specification/search/data-plane/Azure.Search/preview/2019-05-06-preview/searchservice.json b/specification/search/data-plane/Azure.Search/preview/2019-05-06-preview/searchservice.json index 567a0548e26c..c41e3b5c83c0 100644 --- a/specification/search/data-plane/Azure.Search/preview/2019-05-06-preview/searchservice.json +++ b/specification/search/data-plane/Azure.Search/preview/2019-05-06-preview/searchservice.json @@ -5828,7 +5828,7 @@ "allowSkillsetToReadFileData": { "type": "boolean", "default": false, - "description": "If true, will create a path //document//file_data that is an object representing the original file data downloaded from your blob data source. This allows you to pass the original file data to a custom skill for processing within the enrichment pipeline, or to the Document Extraction skill. The object generated will be defined as follows: { /"$type/": /"file/", /"data/": /"BASE64 encoded string of the file/" }" + "description": "If true, will create a path //document//file_data that is an object representing the original file data downloaded from your blob data source. This allows you to pass the original file data to a custom skill for processing within the enrichment pipeline, or to the Document Extraction skill." }, "pdfTextRotationAlgorithm": { "type": "string", From 01ec186422eabc0ac9b402843a4c5ea5b0a5b8f1 Mon Sep 17 00:00:00 2001 From: Jennifer Marsman Date: Thu, 25 Jun 2020 00:03:53 -0400 Subject: [PATCH 05/38] Added 3 words to custom-words list --- custom-words.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/custom-words.txt b/custom-words.txt index 67cd81c107c0..5e4ed93e0d5f 100644 --- a/custom-words.txt +++ b/custom-words.txt @@ -1074,6 +1074,7 @@ Paypal PBIE PCIE PCNET +PDFs peerings Pendingissuance Pendingrevocation @@ -1117,6 +1118,7 @@ powerbidedicated powerbiembedded poweredoff poweroff +pptx prebackup prebackupvalidation prebuilts @@ -1608,6 +1610,7 @@ unparsed unprepare unprepares unpreparing +Unprocessable Unprovision unprovisioned Unprovisioning From fa9fe402a330582482cef9cb52f8fd29db186269 Mon Sep 17 00:00:00 2001 From: Jennifer Marsman Date: Thu, 25 Jun 2020 09:52:05 -0400 Subject: [PATCH 06/38] Marking IndexingParameter extra properties as not required --- .../2019-05-06-preview/searchservice.json | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/specification/search/data-plane/Azure.Search/preview/2019-05-06-preview/searchservice.json b/specification/search/data-plane/Azure.Search/preview/2019-05-06-preview/searchservice.json index c41e3b5c83c0..05889e0ab0c6 100644 --- a/specification/search/data-plane/Azure.Search/preview/2019-05-06-preview/searchservice.json +++ b/specification/search/data-plane/Azure.Search/preview/2019-05-06-preview/searchservice.json @@ -5772,70 +5772,86 @@ "description": "The maximum number of items in a single batch that can fail indexing for the batch to still be considered successful. -1 means no limit. Default is 0." }, "configuration": { + "type": "object", "properties": { "parsingMode": { "type": "string", + "required": false, "$ref": "#/definitions/ParsingMode" }, "excludedFileNameExtensions": { "type": "string", + "required": false, "description": "Comma-delimited list of filename extensions to ignore when processing from Azure blob storage. For example, you could exclude \".png, .mp4\" to skip over those files during indexing." }, "indexedFileNameExtensions": { "type": "string", + "required": false, "description": "Comma-delimited list of filename extensions to select when processing from Azure blob storage. For example, you could focus indexing on specific application files \".docx, .pptx, .msg\" to specifically include those file types." }, "failOnUnsupportedContentType": { "type": "boolean", + "required": false, "default": true, "description": "For Azure blobs, set to false if you want to continue indexing when an unsupported content type is encountered, and you don't know all the content types (file extensions) in advance." }, "failOnUnprocessableDocument": { "type": "boolean", + "required": false, "default": true, "description": "For Azure blobs, set to false if you want to continue indexing if a document fails indexing." }, "indexStorageMetadataOnlyForOversizedDocuments": { "type": "boolean", + "required": false, "default": false, "description": "For Azure blobs, set this property to true to still index storage metadata for blob content that is too large to process. Oversized blobs are treated as errors by default. For limits on blob size, see https://docs.microsoft.com/azure/search/search-limits-quotas-capacity." }, "delimitedTextHeaders": { "type": "string", + "required": false, "description": "For CSV blobs, specifies a comma-delimited list of column headers, useful for mapping source fields to destination fields in an index." }, "delimitedTextDelimiter": { "type": "string", + "required": false, "description": "For CSV blobs, specifies the end-of-line single-character delimiter for CSV files where each line starts a new document (for example, \"|\")." }, "firstLineContainsHeaders": { "type": "boolean", + "required": false, "default": true, "description": "For CSV blobs, indicates that the first (non-blank) line of each blob contains headers." }, "documentRoot": { "type": "string", + "required": false, "description": "For JSON arrays, given a structured or semi-structured document, you can specify a path to the array using this property." }, "dataToExtract": { "type": "string", + "required": false, "$ref": "#/definitions/DataToExtract" }, "imageAction": { "type": "string", + "required": false, "$ref": "#/definitions/ImageAction" }, "allowSkillsetToReadFileData": { "type": "boolean", + "required": false, "default": false, "description": "If true, will create a path //document//file_data that is an object representing the original file data downloaded from your blob data source. This allows you to pass the original file data to a custom skill for processing within the enrichment pipeline, or to the Document Extraction skill." }, "pdfTextRotationAlgorithm": { "type": "string", + "required": false, "$ref": "#/definitions/PdfTextRotationAlgorithm" }, "queryTimeout": { "type": "string", + "required": false, "description": "Increases the timeout beyond the 5-minute default for Azure SQL database data sources, specified in the format \"hh:mm:ss\"." } }, From 18b17f8ce55b9ca0c51bb93c7ed7a26c567fa8ed Mon Sep 17 00:00:00 2001 From: Jennifer Marsman Date: Thu, 25 Jun 2020 18:41:56 -0400 Subject: [PATCH 07/38] Removed duplicate type definitions with $ref, changed default values to false for failOn* properties, and removed en-us locale from URL --- .../preview/2019-05-06-preview/searchservice.json | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/specification/search/data-plane/Azure.Search/preview/2019-05-06-preview/searchservice.json b/specification/search/data-plane/Azure.Search/preview/2019-05-06-preview/searchservice.json index 05889e0ab0c6..de05a298e613 100644 --- a/specification/search/data-plane/Azure.Search/preview/2019-05-06-preview/searchservice.json +++ b/specification/search/data-plane/Azure.Search/preview/2019-05-06-preview/searchservice.json @@ -5775,7 +5775,6 @@ "type": "object", "properties": { "parsingMode": { - "type": "string", "required": false, "$ref": "#/definitions/ParsingMode" }, @@ -5792,13 +5791,13 @@ "failOnUnsupportedContentType": { "type": "boolean", "required": false, - "default": true, + "default": false, "description": "For Azure blobs, set to false if you want to continue indexing when an unsupported content type is encountered, and you don't know all the content types (file extensions) in advance." }, "failOnUnprocessableDocument": { "type": "boolean", "required": false, - "default": true, + "default": false, "description": "For Azure blobs, set to false if you want to continue indexing if a document fails indexing." }, "indexStorageMetadataOnlyForOversizedDocuments": { @@ -5829,12 +5828,10 @@ "description": "For JSON arrays, given a structured or semi-structured document, you can specify a path to the array using this property." }, "dataToExtract": { - "type": "string", "required": false, "$ref": "#/definitions/DataToExtract" }, "imageAction": { - "type": "string", "required": false, "$ref": "#/definitions/ImageAction" }, @@ -5845,7 +5842,6 @@ "description": "If true, will create a path //document//file_data that is an object representing the original file data downloaded from your blob data source. This allows you to pass the original file data to a custom skill for processing within the enrichment pipeline, or to the Document Extraction skill." }, "pdfTextRotationAlgorithm": { - "type": "string", "required": false, "$ref": "#/definitions/PdfTextRotationAlgorithm" }, @@ -5863,7 +5859,7 @@ }, "description": "Represents parameters for indexer execution.", "externalDocs": { - "url": "https://docs.microsoft.com/en-us/rest/api/searchservice/create-indexer#parameters" + "url": "https://docs.microsoft.com/rest/api/searchservice/create-indexer#parameters" } }, "ParsingMode": { From d5efa2283b81d54016978887fbc6d6568ff1186d Mon Sep 17 00:00:00 2001 From: Arvind Krishnaa Jagannathan Date: Thu, 23 Jul 2020 16:29:54 -0700 Subject: [PATCH 08/38] Add swagger spec for indexer execution environment --- .../2019-05-06-preview/searchservice.json | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/specification/search/data-plane/Azure.Search/preview/2019-05-06-preview/searchservice.json b/specification/search/data-plane/Azure.Search/preview/2019-05-06-preview/searchservice.json index de05a298e613..624a656b6249 100644 --- a/specification/search/data-plane/Azure.Search/preview/2019-05-06-preview/searchservice.json +++ b/specification/search/data-plane/Azure.Search/preview/2019-05-06-preview/searchservice.json @@ -5845,6 +5845,10 @@ "required": false, "$ref": "#/definitions/PdfTextRotationAlgorithm" }, + "executionEnvironment": { + "required": false, + "$ref": "#/definitions/ExecutionEnvironment" + }, "queryTimeout": { "type": "string", "required": false, @@ -6002,6 +6006,32 @@ "default": "none", "description": "Determines algorithm for text extraction from PDF files in Azure blob storage." }, + "ExecutionEnvironment": { + "type": "string", + "enum": [ + "standard", + "private" + ], + "x-ms-enum": { + "name": "IndexerExecutionEnvironment", + "modelAsString": true, + "values": [ + { + "value": "standard", + "name": "standard", + "description": "Indicates that Azure Cognitive Search can determine where the indexer should execute. This is the default environment when nothing is specified and is the recommended value." + }, + { + "value": "private", + "name": "private", + "description": "Indicates that the indexer should run with the environment provisioned specifically for the search service. This should only be specified as the execution environment if the indexer needs to access resources securely over shared private link resources." + } + ] + }, + "x-nullable": true, + "default": "none", + "description": "Specifies the environment in which the indexer should execute." + }, "FieldMappingFunction": { "properties": { "name": { From 18bee0a0a723833600981f37e69b4a7f01622555 Mon Sep 17 00:00:00 2001 From: Arvind Krishnaa Jagannathan Date: Fri, 24 Jul 2020 11:12:09 -0700 Subject: [PATCH 09/38] Change the default value to 'standard' --- .../Azure.Search/preview/2019-05-06-preview/searchservice.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specification/search/data-plane/Azure.Search/preview/2019-05-06-preview/searchservice.json b/specification/search/data-plane/Azure.Search/preview/2019-05-06-preview/searchservice.json index 624a656b6249..9ef95d9358f2 100644 --- a/specification/search/data-plane/Azure.Search/preview/2019-05-06-preview/searchservice.json +++ b/specification/search/data-plane/Azure.Search/preview/2019-05-06-preview/searchservice.json @@ -6029,7 +6029,7 @@ ] }, "x-nullable": true, - "default": "none", + "default": "standard", "description": "Specifies the environment in which the indexer should execute." }, "FieldMappingFunction": { From 1b500827f386bd56774a387433cc9a8d93f83ab1 Mon Sep 17 00:00:00 2001 From: Jennifer Marsman Date: Fri, 24 Jul 2020 14:18:19 -0400 Subject: [PATCH 10/38] Updating 2020 API swaggers with IndexingParameters in Cognitive Search --- .../2020-06-30-Preview/searchservice.json | 223 +++++++++++++++++- .../preview/2020-06-30/searchservice.json | 223 +++++++++++++++++- 2 files changed, 444 insertions(+), 2 deletions(-) diff --git a/specification/search/data-plane/Azure.Search/preview/2020-06-30-Preview/searchservice.json b/specification/search/data-plane/Azure.Search/preview/2020-06-30-Preview/searchservice.json index 215221732891..faf728dd6c6f 100644 --- a/specification/search/data-plane/Azure.Search/preview/2020-06-30-Preview/searchservice.json +++ b/specification/search/data-plane/Azure.Search/preview/2020-06-30-Preview/searchservice.json @@ -5773,13 +5773,234 @@ }, "configuration": { "type": "object", + "properties": { + "parsingMode": { + "required": false, + "$ref": "#/definitions/ParsingMode" + }, + "excludedFileNameExtensions": { + "type": "string", + "required": false, + "description": "Comma-delimited list of filename extensions to ignore when processing from Azure blob storage. For example, you could exclude \".png, .mp4\" to skip over those files during indexing." + }, + "indexedFileNameExtensions": { + "type": "string", + "required": false, + "description": "Comma-delimited list of filename extensions to select when processing from Azure blob storage. For example, you could focus indexing on specific application files \".docx, .pptx, .msg\" to specifically include those file types." + }, + "failOnUnsupportedContentType": { + "type": "boolean", + "required": false, + "default": false, + "description": "For Azure blobs, set to false if you want to continue indexing when an unsupported content type is encountered, and you don't know all the content types (file extensions) in advance." + }, + "failOnUnprocessableDocument": { + "type": "boolean", + "required": false, + "default": false, + "description": "For Azure blobs, set to false if you want to continue indexing if a document fails indexing." + }, + "indexStorageMetadataOnlyForOversizedDocuments": { + "type": "boolean", + "required": false, + "default": false, + "description": "For Azure blobs, set this property to true to still index storage metadata for blob content that is too large to process. Oversized blobs are treated as errors by default. For limits on blob size, see https://docs.microsoft.com/azure/search/search-limits-quotas-capacity." + }, + "delimitedTextHeaders": { + "type": "string", + "required": false, + "description": "For CSV blobs, specifies a comma-delimited list of column headers, useful for mapping source fields to destination fields in an index." + }, + "delimitedTextDelimiter": { + "type": "string", + "required": false, + "description": "For CSV blobs, specifies the end-of-line single-character delimiter for CSV files where each line starts a new document (for example, \"|\")." + }, + "firstLineContainsHeaders": { + "type": "boolean", + "required": false, + "default": true, + "description": "For CSV blobs, indicates that the first (non-blank) line of each blob contains headers." + }, + "documentRoot": { + "type": "string", + "required": false, + "description": "For JSON arrays, given a structured or semi-structured document, you can specify a path to the array using this property." + }, + "dataToExtract": { + "required": false, + "$ref": "#/definitions/DataToExtract" + }, + "imageAction": { + "required": false, + "$ref": "#/definitions/ImageAction" + }, + "allowSkillsetToReadFileData": { + "type": "boolean", + "required": false, + "default": false, + "description": "If true, will create a path //document//file_data that is an object representing the original file data downloaded from your blob data source. This allows you to pass the original file data to a custom skill for processing within the enrichment pipeline, or to the Document Extraction skill." + }, + "pdfTextRotationAlgorithm": { + "required": false, + "$ref": "#/definitions/PdfTextRotationAlgorithm" + }, + "queryTimeout": { + "type": "string", + "required": false, + "description": "Increases the timeout beyond the 5-minute default for Azure SQL database data sources, specified in the format \"hh:mm:ss\"." + } + }, "additionalProperties": { "type": "object" }, "description": "A dictionary of indexer-specific configuration properties. Each name is the name of a specific property. Each value must be of a primitive type." } }, - "description": "Represents parameters for indexer execution." + "description": "Represents parameters for indexer execution.", + "externalDocs": { + "url": "https://docs.microsoft.com/rest/api/searchservice/create-indexer#parameters" + } + }, + "ParsingMode": { + "type": "string", + "enum": [ + "default", + "text", + "delimitedText", + "json", + "jsonArray", + "jsonLines" + ], + "x-ms-enum": { + "name": "BlobIndexerParsingMode", + "modelAsString": true, + "values": [ + { + "value": "default", + "name": "Default", + "description": "Set to default for normal file processing." + }, + { + "value": "text", + "name": "Text", + "description": "Set to text to improve indexing performance on plain text files in blob storage." + }, + { + "value": "delimitedText", + "name": "DelimitedText", + "description": "Set to delimitedText when blobs are plain CSV files." + }, + { + "value": "json", + "name": "Json", + "description": "Set to json to extract structured content from JSON files." + }, + { + "value": "jsonArray", + "name": "JsonArray", + "description": "Set to jsonArray to extract individual elements of a JSON array as separate documents in Azure Cognitive Search." + }, + { + "value": "jsonLines", + "name": "JsonLines", + "description": "Set to jsonLines to extract individual JSON entities, separated by a new line, as separate documents in Azure Cognitive Search." + } + ] + }, + "x-nullable": true, + "default": "default", + "description": "Represents the parsing mode for indexing from an Azure blob data source." + }, + "DataToExtract": { + "type": "string", + "enum": [ + "storageMetadata", + "allMetadata", + "contentAndMetadata" + ], + "x-ms-enum": { + "name": "BlobIndexerDataToExtract", + "modelAsString": true, + "values": [ + { + "value": "storageMetadata", + "name": "StorageMetadata", + "description": "Indexes just the standard blob properties and user-specified metadata." + }, + { + "value": "allMetadata", + "name": "AllMetadata", + "description": "Extracts metadata provided by the Azure blob storage subsystem and the content-type specific metadata (for example, metadata unique to just .png files are indexed)." + }, + { + "value": "contentAndMetadata", + "name": "ContentAndMetadata", + "description": "Extracts all metadata and textual content from each blob." + } + ] + }, + "x-nullable": true, + "default": "contentAndMetadata", + "description": "Specifies the data to extract from Azure blob storage and tells the indexer which data to extract from image content when \"imageAction\" is set to a value other than \"none\". This applies to embedded image content in a .PDF or other application, or image files such as .jpg and .png, in Azure blobs." + }, + "ImageAction": { + "type": "string", + "enum": [ + "none", + "generateNormalizedImages", + "generateNormalizedImagePerPage" + ], + "x-ms-enum": { + "name": "BlobIndexerImageAction", + "modelAsString": true, + "values": [ + { + "value": "none", + "name": "None", + "description": "Ignores embedded images or image files in the data set. This is the default." + }, + { + "value": "generateNormalizedImages", + "name": "GenerateNormalizedImages", + "description": "Extracts text from images (for example, the word \"STOP\" from a traffic stop sign), and embeds it into the content field. This action requires that \"dataToExtract\" is set to \"contentAndMetadata\". A normalized image refers to additional processing resulting in uniform image output, sized and rotated to promote consistent rendering when you include images in visual search results. This information is generated for each image when you use this option." + }, + { + "value": "generateNormalizedImagePerPage", + "name": "GenerateNormalizedImagePerPage", + "description": "Extracts text from images (for example, the word \"STOP\" from a traffic stop sign), and embeds it into the content field, but treats PDF files differently in that each page will be rendered as an image and normalized accordingly, instead of extracting embedded images. Non-PDF file types will be treated the same as if \"generateNormalizedImages\" was set." + } + ] + }, + "x-nullable": true, + "default": "none", + "description": "Determines how to process embedded images and image files in Azure blob storage. Setting the \"imageAction\" configuration to any value other than \"none\" requires that a skillset also be attached to that indexer." + }, + "PdfTextRotationAlgorithm": { + "type": "string", + "enum": [ + "none", + "detectAngles" + ], + "x-ms-enum": { + "name": "BlobIndexerPDFTextRotationAlgorithm", + "modelAsString": true, + "values": [ + { + "value": "none", + "name": "None", + "description": "Leverages normal text extraction. This is the default." + }, + { + "value": "detectAngles", + "name": "DetectAngles", + "description": "May produce better and more readable text extraction from PDF files that have rotated text within them. Note that there may be a small performance speed impact when this parameter is used. This parameter only applies to PDF files, and only to PDFs with embedded text. If the rotated text appears within an embedded image in the PDF, this parameter does not apply." + } + ] + }, + "x-nullable": true, + "default": "none", + "description": "Determines algorithm for text extraction from PDF files in Azure blob storage." }, "FieldMappingFunction": { "properties": { diff --git a/specification/search/data-plane/Azure.Search/preview/2020-06-30/searchservice.json b/specification/search/data-plane/Azure.Search/preview/2020-06-30/searchservice.json index 6643d136084e..5fa032806153 100644 --- a/specification/search/data-plane/Azure.Search/preview/2020-06-30/searchservice.json +++ b/specification/search/data-plane/Azure.Search/preview/2020-06-30/searchservice.json @@ -5773,13 +5773,234 @@ }, "configuration": { "type": "object", + "properties": { + "parsingMode": { + "required": false, + "$ref": "#/definitions/ParsingMode" + }, + "excludedFileNameExtensions": { + "type": "string", + "required": false, + "description": "Comma-delimited list of filename extensions to ignore when processing from Azure blob storage. For example, you could exclude \".png, .mp4\" to skip over those files during indexing." + }, + "indexedFileNameExtensions": { + "type": "string", + "required": false, + "description": "Comma-delimited list of filename extensions to select when processing from Azure blob storage. For example, you could focus indexing on specific application files \".docx, .pptx, .msg\" to specifically include those file types." + }, + "failOnUnsupportedContentType": { + "type": "boolean", + "required": false, + "default": false, + "description": "For Azure blobs, set to false if you want to continue indexing when an unsupported content type is encountered, and you don't know all the content types (file extensions) in advance." + }, + "failOnUnprocessableDocument": { + "type": "boolean", + "required": false, + "default": false, + "description": "For Azure blobs, set to false if you want to continue indexing if a document fails indexing." + }, + "indexStorageMetadataOnlyForOversizedDocuments": { + "type": "boolean", + "required": false, + "default": false, + "description": "For Azure blobs, set this property to true to still index storage metadata for blob content that is too large to process. Oversized blobs are treated as errors by default. For limits on blob size, see https://docs.microsoft.com/azure/search/search-limits-quotas-capacity." + }, + "delimitedTextHeaders": { + "type": "string", + "required": false, + "description": "For CSV blobs, specifies a comma-delimited list of column headers, useful for mapping source fields to destination fields in an index." + }, + "delimitedTextDelimiter": { + "type": "string", + "required": false, + "description": "For CSV blobs, specifies the end-of-line single-character delimiter for CSV files where each line starts a new document (for example, \"|\")." + }, + "firstLineContainsHeaders": { + "type": "boolean", + "required": false, + "default": true, + "description": "For CSV blobs, indicates that the first (non-blank) line of each blob contains headers." + }, + "documentRoot": { + "type": "string", + "required": false, + "description": "For JSON arrays, given a structured or semi-structured document, you can specify a path to the array using this property." + }, + "dataToExtract": { + "required": false, + "$ref": "#/definitions/DataToExtract" + }, + "imageAction": { + "required": false, + "$ref": "#/definitions/ImageAction" + }, + "allowSkillsetToReadFileData": { + "type": "boolean", + "required": false, + "default": false, + "description": "If true, will create a path //document//file_data that is an object representing the original file data downloaded from your blob data source. This allows you to pass the original file data to a custom skill for processing within the enrichment pipeline, or to the Document Extraction skill." + }, + "pdfTextRotationAlgorithm": { + "required": false, + "$ref": "#/definitions/PdfTextRotationAlgorithm" + }, + "queryTimeout": { + "type": "string", + "required": false, + "description": "Increases the timeout beyond the 5-minute default for Azure SQL database data sources, specified in the format \"hh:mm:ss\"." + } + }, "additionalProperties": { "type": "object" }, "description": "A dictionary of indexer-specific configuration properties. Each name is the name of a specific property. Each value must be of a primitive type." } }, - "description": "Represents parameters for indexer execution." + "description": "Represents parameters for indexer execution.", + "externalDocs": { + "url": "https://docs.microsoft.com/rest/api/searchservice/create-indexer#parameters" + } + }, + "ParsingMode": { + "type": "string", + "enum": [ + "default", + "text", + "delimitedText", + "json", + "jsonArray", + "jsonLines" + ], + "x-ms-enum": { + "name": "BlobIndexerParsingMode", + "modelAsString": true, + "values": [ + { + "value": "default", + "name": "Default", + "description": "Set to default for normal file processing." + }, + { + "value": "text", + "name": "Text", + "description": "Set to text to improve indexing performance on plain text files in blob storage." + }, + { + "value": "delimitedText", + "name": "DelimitedText", + "description": "Set to delimitedText when blobs are plain CSV files." + }, + { + "value": "json", + "name": "Json", + "description": "Set to json to extract structured content from JSON files." + }, + { + "value": "jsonArray", + "name": "JsonArray", + "description": "Set to jsonArray to extract individual elements of a JSON array as separate documents in Azure Cognitive Search." + }, + { + "value": "jsonLines", + "name": "JsonLines", + "description": "Set to jsonLines to extract individual JSON entities, separated by a new line, as separate documents in Azure Cognitive Search." + } + ] + }, + "x-nullable": true, + "default": "default", + "description": "Represents the parsing mode for indexing from an Azure blob data source." + }, + "DataToExtract": { + "type": "string", + "enum": [ + "storageMetadata", + "allMetadata", + "contentAndMetadata" + ], + "x-ms-enum": { + "name": "BlobIndexerDataToExtract", + "modelAsString": true, + "values": [ + { + "value": "storageMetadata", + "name": "StorageMetadata", + "description": "Indexes just the standard blob properties and user-specified metadata." + }, + { + "value": "allMetadata", + "name": "AllMetadata", + "description": "Extracts metadata provided by the Azure blob storage subsystem and the content-type specific metadata (for example, metadata unique to just .png files are indexed)." + }, + { + "value": "contentAndMetadata", + "name": "ContentAndMetadata", + "description": "Extracts all metadata and textual content from each blob." + } + ] + }, + "x-nullable": true, + "default": "contentAndMetadata", + "description": "Specifies the data to extract from Azure blob storage and tells the indexer which data to extract from image content when \"imageAction\" is set to a value other than \"none\". This applies to embedded image content in a .PDF or other application, or image files such as .jpg and .png, in Azure blobs." + }, + "ImageAction": { + "type": "string", + "enum": [ + "none", + "generateNormalizedImages", + "generateNormalizedImagePerPage" + ], + "x-ms-enum": { + "name": "BlobIndexerImageAction", + "modelAsString": true, + "values": [ + { + "value": "none", + "name": "None", + "description": "Ignores embedded images or image files in the data set. This is the default." + }, + { + "value": "generateNormalizedImages", + "name": "GenerateNormalizedImages", + "description": "Extracts text from images (for example, the word \"STOP\" from a traffic stop sign), and embeds it into the content field. This action requires that \"dataToExtract\" is set to \"contentAndMetadata\". A normalized image refers to additional processing resulting in uniform image output, sized and rotated to promote consistent rendering when you include images in visual search results. This information is generated for each image when you use this option." + }, + { + "value": "generateNormalizedImagePerPage", + "name": "GenerateNormalizedImagePerPage", + "description": "Extracts text from images (for example, the word \"STOP\" from a traffic stop sign), and embeds it into the content field, but treats PDF files differently in that each page will be rendered as an image and normalized accordingly, instead of extracting embedded images. Non-PDF file types will be treated the same as if \"generateNormalizedImages\" was set." + } + ] + }, + "x-nullable": true, + "default": "none", + "description": "Determines how to process embedded images and image files in Azure blob storage. Setting the \"imageAction\" configuration to any value other than \"none\" requires that a skillset also be attached to that indexer." + }, + "PdfTextRotationAlgorithm": { + "type": "string", + "enum": [ + "none", + "detectAngles" + ], + "x-ms-enum": { + "name": "BlobIndexerPDFTextRotationAlgorithm", + "modelAsString": true, + "values": [ + { + "value": "none", + "name": "None", + "description": "Leverages normal text extraction. This is the default." + }, + { + "value": "detectAngles", + "name": "DetectAngles", + "description": "May produce better and more readable text extraction from PDF files that have rotated text within them. Note that there may be a small performance speed impact when this parameter is used. This parameter only applies to PDF files, and only to PDFs with embedded text. If the rotated text appears within an embedded image in the PDF, this parameter does not apply." + } + ] + }, + "x-nullable": true, + "default": "none", + "description": "Determines algorithm for text extraction from PDF files in Azure blob storage." }, "FieldMappingFunction": { "properties": { From f232143bb45c7bc8f0e2b7a3bf7408c5ebb24de9 Mon Sep 17 00:00:00 2001 From: Arvind Krishnaa Jagannathan Date: Fri, 24 Jul 2020 11:27:52 -0700 Subject: [PATCH 11/38] Add swagger spec for indexer execution environment in 2 new API versions --- .../2020-06-30-Preview/searchservice.json | 30 +++++++++++++++++++ .../preview/2020-06-30/searchservice.json | 30 +++++++++++++++++++ 2 files changed, 60 insertions(+) diff --git a/specification/search/data-plane/Azure.Search/preview/2020-06-30-Preview/searchservice.json b/specification/search/data-plane/Azure.Search/preview/2020-06-30-Preview/searchservice.json index faf728dd6c6f..24110d3dabe3 100644 --- a/specification/search/data-plane/Azure.Search/preview/2020-06-30-Preview/searchservice.json +++ b/specification/search/data-plane/Azure.Search/preview/2020-06-30-Preview/searchservice.json @@ -5845,6 +5845,10 @@ "required": false, "$ref": "#/definitions/PdfTextRotationAlgorithm" }, + "executionEnvironment": { + "required": false, + "$ref": "#/definitions/ExecutionEnvironment" + }, "queryTimeout": { "type": "string", "required": false, @@ -6002,6 +6006,32 @@ "default": "none", "description": "Determines algorithm for text extraction from PDF files in Azure blob storage." }, + "ExecutionEnvironment": { + "type": "string", + "enum": [ + "standard", + "private" + ], + "x-ms-enum": { + "name": "IndexerExecutionEnvironment", + "modelAsString": true, + "values": [ + { + "value": "standard", + "name": "standard", + "description": "Indicates that Azure Cognitive Search can determine where the indexer should execute. This is the default environment when nothing is specified and is the recommended value." + }, + { + "value": "private", + "name": "private", + "description": "Indicates that the indexer should run with the environment provisioned specifically for the search service. This should only be specified as the execution environment if the indexer needs to access resources securely over shared private link resources." + } + ] + }, + "x-nullable": true, + "default": "standard", + "description": "Specifies the environment in which the indexer should execute." + }, "FieldMappingFunction": { "properties": { "name": { diff --git a/specification/search/data-plane/Azure.Search/preview/2020-06-30/searchservice.json b/specification/search/data-plane/Azure.Search/preview/2020-06-30/searchservice.json index 5fa032806153..8a78d01f919c 100644 --- a/specification/search/data-plane/Azure.Search/preview/2020-06-30/searchservice.json +++ b/specification/search/data-plane/Azure.Search/preview/2020-06-30/searchservice.json @@ -5845,6 +5845,10 @@ "required": false, "$ref": "#/definitions/PdfTextRotationAlgorithm" }, + "executionEnvironment": { + "required": false, + "$ref": "#/definitions/ExecutionEnvironment" + }, "queryTimeout": { "type": "string", "required": false, @@ -6002,6 +6006,32 @@ "default": "none", "description": "Determines algorithm for text extraction from PDF files in Azure blob storage." }, + "ExecutionEnvironment": { + "type": "string", + "enum": [ + "standard", + "private" + ], + "x-ms-enum": { + "name": "IndexerExecutionEnvironment", + "modelAsString": true, + "values": [ + { + "value": "standard", + "name": "standard", + "description": "Indicates that Azure Cognitive Search can determine where the indexer should execute. This is the default environment when nothing is specified and is the recommended value." + }, + { + "value": "private", + "name": "private", + "description": "Indicates that the indexer should run with the environment provisioned specifically for the search service. This should only be specified as the execution environment if the indexer needs to access resources securely over shared private link resources." + } + ] + }, + "x-nullable": true, + "default": "standard", + "description": "Specifies the environment in which the indexer should execute." + }, "FieldMappingFunction": { "properties": { "name": { From a589333b19f4a75fde19b1d8556057c221b32fc8 Mon Sep 17 00:00:00 2001 From: Jennifer Marsman Date: Mon, 27 Jul 2020 10:19:39 -0400 Subject: [PATCH 12/38] Added default value for queryTimeout parameter --- .../Azure.Search/preview/2020-06-30-Preview/searchservice.json | 1 + .../Azure.Search/preview/2020-06-30/searchservice.json | 1 + 2 files changed, 2 insertions(+) diff --git a/specification/search/data-plane/Azure.Search/preview/2020-06-30-Preview/searchservice.json b/specification/search/data-plane/Azure.Search/preview/2020-06-30-Preview/searchservice.json index 24110d3dabe3..ed069dd5350b 100644 --- a/specification/search/data-plane/Azure.Search/preview/2020-06-30-Preview/searchservice.json +++ b/specification/search/data-plane/Azure.Search/preview/2020-06-30-Preview/searchservice.json @@ -5852,6 +5852,7 @@ "queryTimeout": { "type": "string", "required": false, + "default": "00:05:00", "description": "Increases the timeout beyond the 5-minute default for Azure SQL database data sources, specified in the format \"hh:mm:ss\"." } }, diff --git a/specification/search/data-plane/Azure.Search/preview/2020-06-30/searchservice.json b/specification/search/data-plane/Azure.Search/preview/2020-06-30/searchservice.json index 8a78d01f919c..507379f5ae74 100644 --- a/specification/search/data-plane/Azure.Search/preview/2020-06-30/searchservice.json +++ b/specification/search/data-plane/Azure.Search/preview/2020-06-30/searchservice.json @@ -5852,6 +5852,7 @@ "queryTimeout": { "type": "string", "required": false, + "default": "00:05:00", "description": "Increases the timeout beyond the 5-minute default for Azure SQL database data sources, specified in the format \"hh:mm:ss\"." } }, From 5669fecd90cc41ddd3fd4ed601327e4d943e5e7f Mon Sep 17 00:00:00 2001 From: Jennifer Marsman Date: Mon, 27 Jul 2020 10:59:29 -0400 Subject: [PATCH 13/38] Removing indexing parameter changes from 2019-05-06-preview version of swagger --- .../2019-05-06-preview/searchservice.json | 253 +----------------- 1 file changed, 1 insertion(+), 252 deletions(-) diff --git a/specification/search/data-plane/Azure.Search/preview/2019-05-06-preview/searchservice.json b/specification/search/data-plane/Azure.Search/preview/2019-05-06-preview/searchservice.json index 9ef95d9358f2..c2a69aac29b9 100644 --- a/specification/search/data-plane/Azure.Search/preview/2019-05-06-preview/searchservice.json +++ b/specification/search/data-plane/Azure.Search/preview/2019-05-06-preview/searchservice.json @@ -5773,264 +5773,13 @@ }, "configuration": { "type": "object", - "properties": { - "parsingMode": { - "required": false, - "$ref": "#/definitions/ParsingMode" - }, - "excludedFileNameExtensions": { - "type": "string", - "required": false, - "description": "Comma-delimited list of filename extensions to ignore when processing from Azure blob storage. For example, you could exclude \".png, .mp4\" to skip over those files during indexing." - }, - "indexedFileNameExtensions": { - "type": "string", - "required": false, - "description": "Comma-delimited list of filename extensions to select when processing from Azure blob storage. For example, you could focus indexing on specific application files \".docx, .pptx, .msg\" to specifically include those file types." - }, - "failOnUnsupportedContentType": { - "type": "boolean", - "required": false, - "default": false, - "description": "For Azure blobs, set to false if you want to continue indexing when an unsupported content type is encountered, and you don't know all the content types (file extensions) in advance." - }, - "failOnUnprocessableDocument": { - "type": "boolean", - "required": false, - "default": false, - "description": "For Azure blobs, set to false if you want to continue indexing if a document fails indexing." - }, - "indexStorageMetadataOnlyForOversizedDocuments": { - "type": "boolean", - "required": false, - "default": false, - "description": "For Azure blobs, set this property to true to still index storage metadata for blob content that is too large to process. Oversized blobs are treated as errors by default. For limits on blob size, see https://docs.microsoft.com/azure/search/search-limits-quotas-capacity." - }, - "delimitedTextHeaders": { - "type": "string", - "required": false, - "description": "For CSV blobs, specifies a comma-delimited list of column headers, useful for mapping source fields to destination fields in an index." - }, - "delimitedTextDelimiter": { - "type": "string", - "required": false, - "description": "For CSV blobs, specifies the end-of-line single-character delimiter for CSV files where each line starts a new document (for example, \"|\")." - }, - "firstLineContainsHeaders": { - "type": "boolean", - "required": false, - "default": true, - "description": "For CSV blobs, indicates that the first (non-blank) line of each blob contains headers." - }, - "documentRoot": { - "type": "string", - "required": false, - "description": "For JSON arrays, given a structured or semi-structured document, you can specify a path to the array using this property." - }, - "dataToExtract": { - "required": false, - "$ref": "#/definitions/DataToExtract" - }, - "imageAction": { - "required": false, - "$ref": "#/definitions/ImageAction" - }, - "allowSkillsetToReadFileData": { - "type": "boolean", - "required": false, - "default": false, - "description": "If true, will create a path //document//file_data that is an object representing the original file data downloaded from your blob data source. This allows you to pass the original file data to a custom skill for processing within the enrichment pipeline, or to the Document Extraction skill." - }, - "pdfTextRotationAlgorithm": { - "required": false, - "$ref": "#/definitions/PdfTextRotationAlgorithm" - }, - "executionEnvironment": { - "required": false, - "$ref": "#/definitions/ExecutionEnvironment" - }, - "queryTimeout": { - "type": "string", - "required": false, - "description": "Increases the timeout beyond the 5-minute default for Azure SQL database data sources, specified in the format \"hh:mm:ss\"." - } - }, "additionalProperties": { "type": "object" }, "description": "A dictionary of indexer-specific configuration properties. Each name is the name of a specific property. Each value must be of a primitive type." } }, - "description": "Represents parameters for indexer execution.", - "externalDocs": { - "url": "https://docs.microsoft.com/rest/api/searchservice/create-indexer#parameters" - } - }, - "ParsingMode": { - "type": "string", - "enum": [ - "default", - "text", - "delimitedText", - "json", - "jsonArray", - "jsonLines" - ], - "x-ms-enum": { - "name": "BlobIndexerParsingMode", - "modelAsString": true, - "values": [ - { - "value": "default", - "name": "Default", - "description": "Set to default for normal file processing." - }, - { - "value": "text", - "name": "Text", - "description": "Set to text to improve indexing performance on plain text files in blob storage." - }, - { - "value": "delimitedText", - "name": "DelimitedText", - "description": "Set to delimitedText when blobs are plain CSV files." - }, - { - "value": "json", - "name": "Json", - "description": "Set to json to extract structured content from JSON files." - }, - { - "value": "jsonArray", - "name": "JsonArray", - "description": "Set to jsonArray to extract individual elements of a JSON array as separate documents in Azure Cognitive Search." - }, - { - "value": "jsonLines", - "name": "JsonLines", - "description": "Set to jsonLines to extract individual JSON entities, separated by a new line, as separate documents in Azure Cognitive Search." - } - ] - }, - "x-nullable": true, - "default": "default", - "description": "Represents the parsing mode for indexing from an Azure blob data source." - }, - "DataToExtract": { - "type": "string", - "enum": [ - "storageMetadata", - "allMetadata", - "contentAndMetadata" - ], - "x-ms-enum": { - "name": "BlobIndexerDataToExtract", - "modelAsString": true, - "values": [ - { - "value": "storageMetadata", - "name": "StorageMetadata", - "description": "Indexes just the standard blob properties and user-specified metadata." - }, - { - "value": "allMetadata", - "name": "AllMetadata", - "description": "Extracts metadata provided by the Azure blob storage subsystem and the content-type specific metadata (for example, metadata unique to just .png files are indexed)." - }, - { - "value": "contentAndMetadata", - "name": "ContentAndMetadata", - "description": "Extracts all metadata and textual content from each blob." - } - ] - }, - "x-nullable": true, - "default": "contentAndMetadata", - "description": "Specifies the data to extract from Azure blob storage and tells the indexer which data to extract from image content when \"imageAction\" is set to a value other than \"none\". This applies to embedded image content in a .PDF or other application, or image files such as .jpg and .png, in Azure blobs." - }, - "ImageAction": { - "type": "string", - "enum": [ - "none", - "generateNormalizedImages", - "generateNormalizedImagePerPage" - ], - "x-ms-enum": { - "name": "BlobIndexerImageAction", - "modelAsString": true, - "values": [ - { - "value": "none", - "name": "None", - "description": "Ignores embedded images or image files in the data set. This is the default." - }, - { - "value": "generateNormalizedImages", - "name": "GenerateNormalizedImages", - "description": "Extracts text from images (for example, the word \"STOP\" from a traffic stop sign), and embeds it into the content field. This action requires that \"dataToExtract\" is set to \"contentAndMetadata\". A normalized image refers to additional processing resulting in uniform image output, sized and rotated to promote consistent rendering when you include images in visual search results. This information is generated for each image when you use this option." - }, - { - "value": "generateNormalizedImagePerPage", - "name": "GenerateNormalizedImagePerPage", - "description": "Extracts text from images (for example, the word \"STOP\" from a traffic stop sign), and embeds it into the content field, but treats PDF files differently in that each page will be rendered as an image and normalized accordingly, instead of extracting embedded images. Non-PDF file types will be treated the same as if \"generateNormalizedImages\" was set." - } - ] - }, - "x-nullable": true, - "default": "none", - "description": "Determines how to process embedded images and image files in Azure blob storage. Setting the \"imageAction\" configuration to any value other than \"none\" requires that a skillset also be attached to that indexer." - }, - "PdfTextRotationAlgorithm": { - "type": "string", - "enum": [ - "none", - "detectAngles" - ], - "x-ms-enum": { - "name": "BlobIndexerPDFTextRotationAlgorithm", - "modelAsString": true, - "values": [ - { - "value": "none", - "name": "None", - "description": "Leverages normal text extraction. This is the default." - }, - { - "value": "detectAngles", - "name": "DetectAngles", - "description": "May produce better and more readable text extraction from PDF files that have rotated text within them. Note that there may be a small performance speed impact when this parameter is used. This parameter only applies to PDF files, and only to PDFs with embedded text. If the rotated text appears within an embedded image in the PDF, this parameter does not apply." - } - ] - }, - "x-nullable": true, - "default": "none", - "description": "Determines algorithm for text extraction from PDF files in Azure blob storage." - }, - "ExecutionEnvironment": { - "type": "string", - "enum": [ - "standard", - "private" - ], - "x-ms-enum": { - "name": "IndexerExecutionEnvironment", - "modelAsString": true, - "values": [ - { - "value": "standard", - "name": "standard", - "description": "Indicates that Azure Cognitive Search can determine where the indexer should execute. This is the default environment when nothing is specified and is the recommended value." - }, - { - "value": "private", - "name": "private", - "description": "Indicates that the indexer should run with the environment provisioned specifically for the search service. This should only be specified as the execution environment if the indexer needs to access resources securely over shared private link resources." - } - ] - }, - "x-nullable": true, - "default": "standard", - "description": "Specifies the environment in which the indexer should execute." + "description": "Represents parameters for indexer execution." }, "FieldMappingFunction": { "properties": { From bb01e6970084e07ec0e4f63df489d9e86ae62882 Mon Sep 17 00:00:00 2001 From: Jennifer Marsman Date: Tue, 28 Jul 2020 09:32:53 -0400 Subject: [PATCH 14/38] Removed x-nullable from indexing parameter enums --- .../preview/2020-06-30-Preview/searchservice.json | 5 ----- .../Azure.Search/preview/2020-06-30/searchservice.json | 5 ----- 2 files changed, 10 deletions(-) diff --git a/specification/search/data-plane/Azure.Search/preview/2020-06-30-Preview/searchservice.json b/specification/search/data-plane/Azure.Search/preview/2020-06-30-Preview/searchservice.json index ed069dd5350b..88b7ce6934f6 100644 --- a/specification/search/data-plane/Azure.Search/preview/2020-06-30-Preview/searchservice.json +++ b/specification/search/data-plane/Azure.Search/preview/2020-06-30-Preview/searchservice.json @@ -5913,7 +5913,6 @@ } ] }, - "x-nullable": true, "default": "default", "description": "Represents the parsing mode for indexing from an Azure blob data source." }, @@ -5945,7 +5944,6 @@ } ] }, - "x-nullable": true, "default": "contentAndMetadata", "description": "Specifies the data to extract from Azure blob storage and tells the indexer which data to extract from image content when \"imageAction\" is set to a value other than \"none\". This applies to embedded image content in a .PDF or other application, or image files such as .jpg and .png, in Azure blobs." }, @@ -5977,7 +5975,6 @@ } ] }, - "x-nullable": true, "default": "none", "description": "Determines how to process embedded images and image files in Azure blob storage. Setting the \"imageAction\" configuration to any value other than \"none\" requires that a skillset also be attached to that indexer." }, @@ -6003,7 +6000,6 @@ } ] }, - "x-nullable": true, "default": "none", "description": "Determines algorithm for text extraction from PDF files in Azure blob storage." }, @@ -6029,7 +6025,6 @@ } ] }, - "x-nullable": true, "default": "standard", "description": "Specifies the environment in which the indexer should execute." }, diff --git a/specification/search/data-plane/Azure.Search/preview/2020-06-30/searchservice.json b/specification/search/data-plane/Azure.Search/preview/2020-06-30/searchservice.json index 507379f5ae74..649dbcd3a992 100644 --- a/specification/search/data-plane/Azure.Search/preview/2020-06-30/searchservice.json +++ b/specification/search/data-plane/Azure.Search/preview/2020-06-30/searchservice.json @@ -5913,7 +5913,6 @@ } ] }, - "x-nullable": true, "default": "default", "description": "Represents the parsing mode for indexing from an Azure blob data source." }, @@ -5945,7 +5944,6 @@ } ] }, - "x-nullable": true, "default": "contentAndMetadata", "description": "Specifies the data to extract from Azure blob storage and tells the indexer which data to extract from image content when \"imageAction\" is set to a value other than \"none\". This applies to embedded image content in a .PDF or other application, or image files such as .jpg and .png, in Azure blobs." }, @@ -5977,7 +5975,6 @@ } ] }, - "x-nullable": true, "default": "none", "description": "Determines how to process embedded images and image files in Azure blob storage. Setting the \"imageAction\" configuration to any value other than \"none\" requires that a skillset also be attached to that indexer." }, @@ -6003,7 +6000,6 @@ } ] }, - "x-nullable": true, "default": "none", "description": "Determines algorithm for text extraction from PDF files in Azure blob storage." }, @@ -6029,7 +6025,6 @@ } ] }, - "x-nullable": true, "default": "standard", "description": "Specifies the environment in which the indexer should execute." }, From 096a86b5885eaa008e701023c9899a51572cf2fa Mon Sep 17 00:00:00 2001 From: Jennifer Marsman Date: Thu, 30 Jul 2020 10:36:51 -0400 Subject: [PATCH 15/38] Removed the required: false since everything is optional/not required by default --- .../2020-06-30-Preview/searchservice.json | 16 ---------------- .../preview/2020-06-30/searchservice.json | 16 ---------------- 2 files changed, 32 deletions(-) diff --git a/specification/search/data-plane/Azure.Search/preview/2020-06-30-Preview/searchservice.json b/specification/search/data-plane/Azure.Search/preview/2020-06-30-Preview/searchservice.json index 88b7ce6934f6..d54b85d0e70d 100644 --- a/specification/search/data-plane/Azure.Search/preview/2020-06-30-Preview/searchservice.json +++ b/specification/search/data-plane/Azure.Search/preview/2020-06-30-Preview/searchservice.json @@ -5775,83 +5775,67 @@ "type": "object", "properties": { "parsingMode": { - "required": false, "$ref": "#/definitions/ParsingMode" }, "excludedFileNameExtensions": { "type": "string", - "required": false, "description": "Comma-delimited list of filename extensions to ignore when processing from Azure blob storage. For example, you could exclude \".png, .mp4\" to skip over those files during indexing." }, "indexedFileNameExtensions": { "type": "string", - "required": false, "description": "Comma-delimited list of filename extensions to select when processing from Azure blob storage. For example, you could focus indexing on specific application files \".docx, .pptx, .msg\" to specifically include those file types." }, "failOnUnsupportedContentType": { "type": "boolean", - "required": false, "default": false, "description": "For Azure blobs, set to false if you want to continue indexing when an unsupported content type is encountered, and you don't know all the content types (file extensions) in advance." }, "failOnUnprocessableDocument": { "type": "boolean", - "required": false, "default": false, "description": "For Azure blobs, set to false if you want to continue indexing if a document fails indexing." }, "indexStorageMetadataOnlyForOversizedDocuments": { "type": "boolean", - "required": false, "default": false, "description": "For Azure blobs, set this property to true to still index storage metadata for blob content that is too large to process. Oversized blobs are treated as errors by default. For limits on blob size, see https://docs.microsoft.com/azure/search/search-limits-quotas-capacity." }, "delimitedTextHeaders": { "type": "string", - "required": false, "description": "For CSV blobs, specifies a comma-delimited list of column headers, useful for mapping source fields to destination fields in an index." }, "delimitedTextDelimiter": { "type": "string", - "required": false, "description": "For CSV blobs, specifies the end-of-line single-character delimiter for CSV files where each line starts a new document (for example, \"|\")." }, "firstLineContainsHeaders": { "type": "boolean", - "required": false, "default": true, "description": "For CSV blobs, indicates that the first (non-blank) line of each blob contains headers." }, "documentRoot": { "type": "string", - "required": false, "description": "For JSON arrays, given a structured or semi-structured document, you can specify a path to the array using this property." }, "dataToExtract": { - "required": false, "$ref": "#/definitions/DataToExtract" }, "imageAction": { - "required": false, "$ref": "#/definitions/ImageAction" }, "allowSkillsetToReadFileData": { "type": "boolean", - "required": false, "default": false, "description": "If true, will create a path //document//file_data that is an object representing the original file data downloaded from your blob data source. This allows you to pass the original file data to a custom skill for processing within the enrichment pipeline, or to the Document Extraction skill." }, "pdfTextRotationAlgorithm": { - "required": false, "$ref": "#/definitions/PdfTextRotationAlgorithm" }, "executionEnvironment": { - "required": false, "$ref": "#/definitions/ExecutionEnvironment" }, "queryTimeout": { "type": "string", - "required": false, "default": "00:05:00", "description": "Increases the timeout beyond the 5-minute default for Azure SQL database data sources, specified in the format \"hh:mm:ss\"." } diff --git a/specification/search/data-plane/Azure.Search/preview/2020-06-30/searchservice.json b/specification/search/data-plane/Azure.Search/preview/2020-06-30/searchservice.json index 649dbcd3a992..5b15e428aafd 100644 --- a/specification/search/data-plane/Azure.Search/preview/2020-06-30/searchservice.json +++ b/specification/search/data-plane/Azure.Search/preview/2020-06-30/searchservice.json @@ -5775,83 +5775,67 @@ "type": "object", "properties": { "parsingMode": { - "required": false, "$ref": "#/definitions/ParsingMode" }, "excludedFileNameExtensions": { "type": "string", - "required": false, "description": "Comma-delimited list of filename extensions to ignore when processing from Azure blob storage. For example, you could exclude \".png, .mp4\" to skip over those files during indexing." }, "indexedFileNameExtensions": { "type": "string", - "required": false, "description": "Comma-delimited list of filename extensions to select when processing from Azure blob storage. For example, you could focus indexing on specific application files \".docx, .pptx, .msg\" to specifically include those file types." }, "failOnUnsupportedContentType": { "type": "boolean", - "required": false, "default": false, "description": "For Azure blobs, set to false if you want to continue indexing when an unsupported content type is encountered, and you don't know all the content types (file extensions) in advance." }, "failOnUnprocessableDocument": { "type": "boolean", - "required": false, "default": false, "description": "For Azure blobs, set to false if you want to continue indexing if a document fails indexing." }, "indexStorageMetadataOnlyForOversizedDocuments": { "type": "boolean", - "required": false, "default": false, "description": "For Azure blobs, set this property to true to still index storage metadata for blob content that is too large to process. Oversized blobs are treated as errors by default. For limits on blob size, see https://docs.microsoft.com/azure/search/search-limits-quotas-capacity." }, "delimitedTextHeaders": { "type": "string", - "required": false, "description": "For CSV blobs, specifies a comma-delimited list of column headers, useful for mapping source fields to destination fields in an index." }, "delimitedTextDelimiter": { "type": "string", - "required": false, "description": "For CSV blobs, specifies the end-of-line single-character delimiter for CSV files where each line starts a new document (for example, \"|\")." }, "firstLineContainsHeaders": { "type": "boolean", - "required": false, "default": true, "description": "For CSV blobs, indicates that the first (non-blank) line of each blob contains headers." }, "documentRoot": { "type": "string", - "required": false, "description": "For JSON arrays, given a structured or semi-structured document, you can specify a path to the array using this property." }, "dataToExtract": { - "required": false, "$ref": "#/definitions/DataToExtract" }, "imageAction": { - "required": false, "$ref": "#/definitions/ImageAction" }, "allowSkillsetToReadFileData": { "type": "boolean", - "required": false, "default": false, "description": "If true, will create a path //document//file_data that is an object representing the original file data downloaded from your blob data source. This allows you to pass the original file data to a custom skill for processing within the enrichment pipeline, or to the Document Extraction skill." }, "pdfTextRotationAlgorithm": { - "required": false, "$ref": "#/definitions/PdfTextRotationAlgorithm" }, "executionEnvironment": { - "required": false, "$ref": "#/definitions/ExecutionEnvironment" }, "queryTimeout": { "type": "string", - "required": false, "default": "00:05:00", "description": "Increases the timeout beyond the 5-minute default for Azure SQL database data sources, specified in the format \"hh:mm:ss\"." } From 2a090f1cc6f6770732cec54528770e7173fd203a Mon Sep 17 00:00:00 2001 From: Jennifer Marsman Date: Thu, 30 Jul 2020 17:31:24 -0400 Subject: [PATCH 16/38] Marked batchSize, maxFailedItems, and maxFailedItemsPerBatch as x-nullable --- .../Azure.Search/preview/2020-06-30-Preview/searchservice.json | 3 +++ .../Azure.Search/preview/2020-06-30/searchservice.json | 3 +++ 2 files changed, 6 insertions(+) diff --git a/specification/search/data-plane/Azure.Search/preview/2020-06-30-Preview/searchservice.json b/specification/search/data-plane/Azure.Search/preview/2020-06-30-Preview/searchservice.json index d54b85d0e70d..7062634db330 100644 --- a/specification/search/data-plane/Azure.Search/preview/2020-06-30-Preview/searchservice.json +++ b/specification/search/data-plane/Azure.Search/preview/2020-06-30-Preview/searchservice.json @@ -5757,17 +5757,20 @@ "batchSize": { "type": "integer", "format": "int32", + "x-nullable": true, "description": "The number of items that are read from the data source and indexed as a single batch in order to improve performance. The default depends on the data source type." }, "maxFailedItems": { "type": "integer", "format": "int32", + "x-nullable": true, "default": 0, "description": "The maximum number of items that can fail indexing for indexer execution to still be considered successful. -1 means no limit. Default is 0." }, "maxFailedItemsPerBatch": { "type": "integer", "format": "int32", + "x-nullable": true, "default": 0, "description": "The maximum number of items in a single batch that can fail indexing for the batch to still be considered successful. -1 means no limit. Default is 0." }, diff --git a/specification/search/data-plane/Azure.Search/preview/2020-06-30/searchservice.json b/specification/search/data-plane/Azure.Search/preview/2020-06-30/searchservice.json index 5b15e428aafd..43f483b82439 100644 --- a/specification/search/data-plane/Azure.Search/preview/2020-06-30/searchservice.json +++ b/specification/search/data-plane/Azure.Search/preview/2020-06-30/searchservice.json @@ -5757,17 +5757,20 @@ "batchSize": { "type": "integer", "format": "int32", + "x-nullable": true, "description": "The number of items that are read from the data source and indexed as a single batch in order to improve performance. The default depends on the data source type." }, "maxFailedItems": { "type": "integer", "format": "int32", + "x-nullable": true, "default": 0, "description": "The maximum number of items that can fail indexing for indexer execution to still be considered successful. -1 means no limit. Default is 0." }, "maxFailedItemsPerBatch": { "type": "integer", "format": "int32", + "x-nullable": true, "default": 0, "description": "The maximum number of items in a single batch that can fail indexing for the batch to still be considered successful. -1 means no limit. Default is 0." }, From 51750c666ff7641c700add239bc6989dcc75c21f Mon Sep 17 00:00:00 2001 From: Jennifer Marsman Date: Mon, 3 Aug 2020 13:15:39 -0400 Subject: [PATCH 17/38] Added x-nullable to configuration parameters --- .../2020-06-30-Preview/searchservice.json | 16 ++++++++++++++++ .../preview/2020-06-30/searchservice.json | 16 ++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/specification/search/data-plane/Azure.Search/preview/2020-06-30-Preview/searchservice.json b/specification/search/data-plane/Azure.Search/preview/2020-06-30-Preview/searchservice.json index 7062634db330..73af85d57f8d 100644 --- a/specification/search/data-plane/Azure.Search/preview/2020-06-30-Preview/searchservice.json +++ b/specification/search/data-plane/Azure.Search/preview/2020-06-30-Preview/searchservice.json @@ -5778,68 +5778,84 @@ "type": "object", "properties": { "parsingMode": { + "x-nullable": true, "$ref": "#/definitions/ParsingMode" }, "excludedFileNameExtensions": { "type": "string", + "x-nullable": true, "description": "Comma-delimited list of filename extensions to ignore when processing from Azure blob storage. For example, you could exclude \".png, .mp4\" to skip over those files during indexing." }, "indexedFileNameExtensions": { "type": "string", + "x-nullable": true, "description": "Comma-delimited list of filename extensions to select when processing from Azure blob storage. For example, you could focus indexing on specific application files \".docx, .pptx, .msg\" to specifically include those file types." }, "failOnUnsupportedContentType": { "type": "boolean", "default": false, + "x-nullable": true, "description": "For Azure blobs, set to false if you want to continue indexing when an unsupported content type is encountered, and you don't know all the content types (file extensions) in advance." }, "failOnUnprocessableDocument": { "type": "boolean", "default": false, + "x-nullable": true, "description": "For Azure blobs, set to false if you want to continue indexing if a document fails indexing." }, "indexStorageMetadataOnlyForOversizedDocuments": { "type": "boolean", "default": false, + "x-nullable": true, "description": "For Azure blobs, set this property to true to still index storage metadata for blob content that is too large to process. Oversized blobs are treated as errors by default. For limits on blob size, see https://docs.microsoft.com/azure/search/search-limits-quotas-capacity." }, "delimitedTextHeaders": { "type": "string", + "x-nullable": true, "description": "For CSV blobs, specifies a comma-delimited list of column headers, useful for mapping source fields to destination fields in an index." }, "delimitedTextDelimiter": { "type": "string", + "x-nullable": true, "description": "For CSV blobs, specifies the end-of-line single-character delimiter for CSV files where each line starts a new document (for example, \"|\")." }, "firstLineContainsHeaders": { "type": "boolean", "default": true, + "x-nullable": true, "description": "For CSV blobs, indicates that the first (non-blank) line of each blob contains headers." }, "documentRoot": { "type": "string", + "x-nullable": true, "description": "For JSON arrays, given a structured or semi-structured document, you can specify a path to the array using this property." }, "dataToExtract": { + "x-nullable": true, "$ref": "#/definitions/DataToExtract" }, "imageAction": { + "x-nullable": true, "$ref": "#/definitions/ImageAction" }, "allowSkillsetToReadFileData": { "type": "boolean", "default": false, + "x-nullable": true, "description": "If true, will create a path //document//file_data that is an object representing the original file data downloaded from your blob data source. This allows you to pass the original file data to a custom skill for processing within the enrichment pipeline, or to the Document Extraction skill." }, "pdfTextRotationAlgorithm": { + "x-nullable": true, "$ref": "#/definitions/PdfTextRotationAlgorithm" }, "executionEnvironment": { + "x-nullable": true, "$ref": "#/definitions/ExecutionEnvironment" }, "queryTimeout": { "type": "string", "default": "00:05:00", + "x-nullable": true, "description": "Increases the timeout beyond the 5-minute default for Azure SQL database data sources, specified in the format \"hh:mm:ss\"." } }, diff --git a/specification/search/data-plane/Azure.Search/preview/2020-06-30/searchservice.json b/specification/search/data-plane/Azure.Search/preview/2020-06-30/searchservice.json index 43f483b82439..704e480dcc90 100644 --- a/specification/search/data-plane/Azure.Search/preview/2020-06-30/searchservice.json +++ b/specification/search/data-plane/Azure.Search/preview/2020-06-30/searchservice.json @@ -5778,68 +5778,84 @@ "type": "object", "properties": { "parsingMode": { + "x-nullable": true, "$ref": "#/definitions/ParsingMode" }, "excludedFileNameExtensions": { "type": "string", + "x-nullable": true, "description": "Comma-delimited list of filename extensions to ignore when processing from Azure blob storage. For example, you could exclude \".png, .mp4\" to skip over those files during indexing." }, "indexedFileNameExtensions": { "type": "string", + "x-nullable": true, "description": "Comma-delimited list of filename extensions to select when processing from Azure blob storage. For example, you could focus indexing on specific application files \".docx, .pptx, .msg\" to specifically include those file types." }, "failOnUnsupportedContentType": { "type": "boolean", "default": false, + "x-nullable": true, "description": "For Azure blobs, set to false if you want to continue indexing when an unsupported content type is encountered, and you don't know all the content types (file extensions) in advance." }, "failOnUnprocessableDocument": { "type": "boolean", "default": false, + "x-nullable": true, "description": "For Azure blobs, set to false if you want to continue indexing if a document fails indexing." }, "indexStorageMetadataOnlyForOversizedDocuments": { "type": "boolean", "default": false, + "x-nullable": true, "description": "For Azure blobs, set this property to true to still index storage metadata for blob content that is too large to process. Oversized blobs are treated as errors by default. For limits on blob size, see https://docs.microsoft.com/azure/search/search-limits-quotas-capacity." }, "delimitedTextHeaders": { "type": "string", + "x-nullable": true, "description": "For CSV blobs, specifies a comma-delimited list of column headers, useful for mapping source fields to destination fields in an index." }, "delimitedTextDelimiter": { "type": "string", + "x-nullable": true, "description": "For CSV blobs, specifies the end-of-line single-character delimiter for CSV files where each line starts a new document (for example, \"|\")." }, "firstLineContainsHeaders": { "type": "boolean", "default": true, + "x-nullable": true, "description": "For CSV blobs, indicates that the first (non-blank) line of each blob contains headers." }, "documentRoot": { "type": "string", + "x-nullable": true, "description": "For JSON arrays, given a structured or semi-structured document, you can specify a path to the array using this property." }, "dataToExtract": { + "x-nullable": true, "$ref": "#/definitions/DataToExtract" }, "imageAction": { + "x-nullable": true, "$ref": "#/definitions/ImageAction" }, "allowSkillsetToReadFileData": { "type": "boolean", "default": false, + "x-nullable": true, "description": "If true, will create a path //document//file_data that is an object representing the original file data downloaded from your blob data source. This allows you to pass the original file data to a custom skill for processing within the enrichment pipeline, or to the Document Extraction skill." }, "pdfTextRotationAlgorithm": { + "x-nullable": true, "$ref": "#/definitions/PdfTextRotationAlgorithm" }, "executionEnvironment": { + "x-nullable": true, "$ref": "#/definitions/ExecutionEnvironment" }, "queryTimeout": { "type": "string", "default": "00:05:00", + "x-nullable": true, "description": "Increases the timeout beyond the 5-minute default for Azure SQL database data sources, specified in the format \"hh:mm:ss\"." } }, From 5c3f4260b33124059e25c2eb0da8082e92cee53d Mon Sep 17 00:00:00 2001 From: Jennifer Marsman Date: Sat, 20 Jun 2020 01:59:05 -0400 Subject: [PATCH 18/38] Added indexing parameters to the swagger for Cognitive Search --- .../2019-05-06-preview/searchservice.json | 210 +++++++++++++++++- 1 file changed, 209 insertions(+), 1 deletion(-) diff --git a/specification/search/data-plane/Azure.Search/preview/2019-05-06-preview/searchservice.json b/specification/search/data-plane/Azure.Search/preview/2019-05-06-preview/searchservice.json index ddead0180328..de0160203615 100644 --- a/specification/search/data-plane/Azure.Search/preview/2019-05-06-preview/searchservice.json +++ b/specification/search/data-plane/Azure.Search/preview/2019-05-06-preview/searchservice.json @@ -5780,6 +5780,71 @@ "default": 0, "description": "The maximum number of items in a single batch that can fail indexing for the batch to still be considered successful. -1 means no limit. Default is 0." }, + "parsingMode": { + "type": "string", + "$ref": "#/definitions/ParsingMode" + }, + "excludedFileNameExtensions": { + "type": "string", + "description": "Comma-delimited list of filename extensions to ignore when processing from Azure blob storage. For example, you could exclude \".png, .mp4\" to skip over those files during indexing." + }, + "indexedFileNameExtensions": { + "type": "string", + "description": "Comma-delimited list of filename extensions to select when processing from Azure blob storage. For example, you could focus indexing on specific application files \".docx, .pptx, .msg\" to specifically include those file types." + }, + "failOnUnsupportedContentType": { + "type": "boolean", + "default": true, + "description": "For Azure blobs, set to false if you want to continue indexing when an unsupported content type is encountered, and you don't know all the content types (file extensions) in advance." + }, + "failOnUnprocessableDocument": { + "type": "boolean", + "default": true, + "description": "For Azure blobs, set to false if you want to continue indexing if a document fails indexing." + }, + "indexStorageMetadataOnlyForOversizedDocuments": { + "type": "boolean", + "default": false, + "description": "For Azure blobs, set this property to true to still index storage metadata for blob content that is too large to process. Oversized blobs are treated as errors by default. For limits on blob size, see https://docs.microsoft.com/azure/search/search-limits-quotas-capacity." + }, + "delimitedTextHeaders": { + "type": "string", + "description": "For CSV blobs, specifies a comma-delimited list of column headers, useful for mapping source fields to destination fields in an index." + }, + "delimitedTextDelimiter": { + "type": "string", + "description": "For CSV blobs, specifies the end-of-line single-character delimiter for CSV files where each line starts a new document (for example, \"|\")." + }, + "firstLineContainsHeaders": { + "type": "boolean", + "default": true, + "description": "For CSV blobs, indicates that the first (non-blank) line of each blob contains headers." + }, + "documentRoot": { + "type": "string", + "description": "For JSON arrays, given a structured or semi-structured document, you can specify a path to the array using this property." + }, + "dataToExtract": { + "type": "string", + "$ref": "#/definitions/DataToExtract" + }, + "imageAction": { + "type": "string", + "$ref": "#/definitions/ImageAction" + }, + "allowSkillsetToReadFileData": { + "type": "boolean", + "default": false, + "description": "If true, will create a path //document//file_data that is an object representing the original file data downloaded from your blob data source. This allows you to pass the original file data to a custom skill for processing within the enrichment pipeline, or to the Document Extraction skill. The object generated will be defined as follows: { /"$type/": /"file/", /"data/": /"BASE64 encoded string of the file/" }" + }, + "pdfTextRotationAlgorithm": { + "type": "string", + "$ref": "#/definitions/PdfTextRotationAlgorithm" + }, + "queryTimeout": { + "type": "string", + "description": "Increases the timeout beyond the 5-minute default for Azure SQL database data sources, specified in the format \"hh:mm:ss\"." + }, "configuration": { "type": "object", "additionalProperties": { @@ -5788,7 +5853,150 @@ "description": "A dictionary of indexer-specific configuration properties. Each name is the name of a specific property. Each value must be of a primitive type." } }, - "description": "Represents parameters for indexer execution." + "description": "Represents parameters for indexer execution.", + "externalDocs": { + "url": "https://docs.microsoft.com/en-us/rest/api/searchservice/create-indexer#parameters" + } + }, + "ParsingMode": { + "type": "string", + "enum": [ + "default", + "text", + "delimitedText", + "json", + "jsonArray", + "jsonLines" + ], + "x-ms-enum": { + "name": "BlobIndexerParsingMode", + "modelAsString": true, + "values": [ + { + "value": "default", + "name": "Default", + "description": "Set to default for normal file processing." + }, + { + "value": "text", + "name": "Text", + "description": "Set to text to improve indexing performance on plain text files in blob storage." + }, + { + "value": "delimitedText", + "name": "DelimitedText", + "description": "Set to delimitedText when blobs are plain CSV files." + }, + { + "value": "json", + "name": "Json", + "description": "Set to json to extract structured content from JSON files." + }, + { + "value": "jsonArray", + "name": "JsonArray", + "description": "Set to jsonArray to extract individual elements of a JSON array as separate documents in Azure Cognitive Search." + }, + { + "value": "jsonLines", + "name": "JsonLines", + "description": "Set to jsonLines to extract individual JSON entities, separated by a new line, as separate documents in Azure Cognitive Search." + } + ] + }, + "x-nullable": true, + "default": "default", + "description": "Represents the parsing mode for indexing from an Azure blob data source." + }, + "DataToExtract": { + "type": "string", + "enum": [ + "storageMetadata", + "allMetadata", + "contentAndMetadata" + ], + "x-ms-enum": { + "name": "BlobIndexerDataToExtract", + "modelAsString": true, + "values": [ + { + "value": "storageMetadata", + "name": "StorageMetadata", + "description": "Indexes just the standard blob properties and user-specified metadata." + }, + { + "value": "allMetadata", + "name": "AllMetadata", + "description": "Extracts metadata provided by the Azure blob storage subsystem and the content-type specific metadata (for example, metadata unique to just .png files are indexed)." + }, + { + "value": "contentAndMetadata", + "name": "ContentAndMetadata", + "description": "Extracts all metadata and textual content from each blob." + } + ] + }, + "x-nullable": true, + "default": "contentAndMetadata", + "description": "Specifies the data to extract from Azure blob storage and tells the indexer which data to extract from image content when \"imageAction\" is set to a value other than \"none\". This applies to embedded image content in a .PDF or other application, or image files such as .jpg and .png, in Azure blobs." + }, + "ImageAction": { + "type": "string", + "enum": [ + "none", + "generateNormalizedImages", + "generateNormalizedImagePerPage" + ], + "x-ms-enum": { + "name": "BlobIndexerImageAction", + "modelAsString": true, + "values": [ + { + "value": "none", + "name": "None", + "description": "Ignores embedded images or image files in the data set. This is the default." + }, + { + "value": "generateNormalizedImages", + "name": "GenerateNormalizedImages", + "description": "Extracts text from images (for example, the word \"STOP\" from a traffic stop sign), and embeds it into the content field. This action requires that \"dataToExtract\" is set to \"contentAndMetadata\". A normalized image refers to additional processing resulting in uniform image output, sized and rotated to promote consistent rendering when you include images in visual search results. This information is generated for each image when you use this option." + }, + { + "value": "generateNormalizedImagePerPage", + "name": "GenerateNormalizedImagePerPage", + "description": "Extracts text from images (for example, the word \"STOP\" from a traffic stop sign), and embeds it into the content field, but treats PDF files differently in that each page will be rendered as an image and normalized accordingly, instead of extracting embedded images. Non-PDF file types will be treated the same as if \"generateNormalizedImages\" was set." + } + ] + }, + "x-nullable": true, + "default": "none", + "description": "Determines how to process embedded images and image files in Azure blob storage. Setting the \"imageAction\" configuration to any value other than \"none\" requires that a skillset also be attached to that indexer." + }, + "PdfTextRotationAlgorithm": { + "type": "string", + "enum": [ + "none", + "detectAngles" + ], + "x-ms-enum": { + "name": "BlobIndexerPDFTextRotationAlgorithm", + "modelAsString": true, + "values": [ + { + "value": "none", + "name": "None", + "description": "Leverages normal text extraction. This is the default." + }, + { + "value": "detectAngles", + "name": "DetectAngles", + "description": "May produce better and more readable text extraction from PDF files that have rotated text within them. Note that there may be a small performance speed impact when this parameter is used. This parameter only applies to PDF files, and only to PDFs with embedded text. If the rotated text appears within an embedded image in the PDF, this parameter does not apply." + } + ] + }, + "x-nullable": true, + "default": "none", + "description": "Determines algorithm for text extraction from PDF files in Azure blob storage." }, "FieldMappingFunction": { "properties": { From b439d16fabb644895116911030f70e5a998bff52 Mon Sep 17 00:00:00 2001 From: Jennifer Marsman Date: Wed, 24 Jun 2020 21:47:30 -0400 Subject: [PATCH 19/38] Moved properties under configuration --- .../2019-05-06-preview/searchservice.json | 133 +++++++++--------- 1 file changed, 67 insertions(+), 66 deletions(-) diff --git a/specification/search/data-plane/Azure.Search/preview/2019-05-06-preview/searchservice.json b/specification/search/data-plane/Azure.Search/preview/2019-05-06-preview/searchservice.json index de0160203615..c4066cd5186b 100644 --- a/specification/search/data-plane/Azure.Search/preview/2019-05-06-preview/searchservice.json +++ b/specification/search/data-plane/Azure.Search/preview/2019-05-06-preview/searchservice.json @@ -5780,73 +5780,74 @@ "default": 0, "description": "The maximum number of items in a single batch that can fail indexing for the batch to still be considered successful. -1 means no limit. Default is 0." }, - "parsingMode": { - "type": "string", - "$ref": "#/definitions/ParsingMode" - }, - "excludedFileNameExtensions": { - "type": "string", - "description": "Comma-delimited list of filename extensions to ignore when processing from Azure blob storage. For example, you could exclude \".png, .mp4\" to skip over those files during indexing." - }, - "indexedFileNameExtensions": { - "type": "string", - "description": "Comma-delimited list of filename extensions to select when processing from Azure blob storage. For example, you could focus indexing on specific application files \".docx, .pptx, .msg\" to specifically include those file types." - }, - "failOnUnsupportedContentType": { - "type": "boolean", - "default": true, - "description": "For Azure blobs, set to false if you want to continue indexing when an unsupported content type is encountered, and you don't know all the content types (file extensions) in advance." - }, - "failOnUnprocessableDocument": { - "type": "boolean", - "default": true, - "description": "For Azure blobs, set to false if you want to continue indexing if a document fails indexing." - }, - "indexStorageMetadataOnlyForOversizedDocuments": { - "type": "boolean", - "default": false, - "description": "For Azure blobs, set this property to true to still index storage metadata for blob content that is too large to process. Oversized blobs are treated as errors by default. For limits on blob size, see https://docs.microsoft.com/azure/search/search-limits-quotas-capacity." - }, - "delimitedTextHeaders": { - "type": "string", - "description": "For CSV blobs, specifies a comma-delimited list of column headers, useful for mapping source fields to destination fields in an index." - }, - "delimitedTextDelimiter": { - "type": "string", - "description": "For CSV blobs, specifies the end-of-line single-character delimiter for CSV files where each line starts a new document (for example, \"|\")." - }, - "firstLineContainsHeaders": { - "type": "boolean", - "default": true, - "description": "For CSV blobs, indicates that the first (non-blank) line of each blob contains headers." - }, - "documentRoot": { - "type": "string", - "description": "For JSON arrays, given a structured or semi-structured document, you can specify a path to the array using this property." - }, - "dataToExtract": { - "type": "string", - "$ref": "#/definitions/DataToExtract" - }, - "imageAction": { - "type": "string", - "$ref": "#/definitions/ImageAction" - }, - "allowSkillsetToReadFileData": { - "type": "boolean", - "default": false, - "description": "If true, will create a path //document//file_data that is an object representing the original file data downloaded from your blob data source. This allows you to pass the original file data to a custom skill for processing within the enrichment pipeline, or to the Document Extraction skill. The object generated will be defined as follows: { /"$type/": /"file/", /"data/": /"BASE64 encoded string of the file/" }" - }, - "pdfTextRotationAlgorithm": { - "type": "string", - "$ref": "#/definitions/PdfTextRotationAlgorithm" - }, - "queryTimeout": { - "type": "string", - "description": "Increases the timeout beyond the 5-minute default for Azure SQL database data sources, specified in the format \"hh:mm:ss\"." - }, "configuration": { - "type": "object", + "properties": { + "parsingMode": { + "type": "string", + "$ref": "#/definitions/ParsingMode" + }, + "excludedFileNameExtensions": { + "type": "string", + "description": "Comma-delimited list of filename extensions to ignore when processing from Azure blob storage. For example, you could exclude \".png, .mp4\" to skip over those files during indexing." + }, + "indexedFileNameExtensions": { + "type": "string", + "description": "Comma-delimited list of filename extensions to select when processing from Azure blob storage. For example, you could focus indexing on specific application files \".docx, .pptx, .msg\" to specifically include those file types." + }, + "failOnUnsupportedContentType": { + "type": "boolean", + "default": true, + "description": "For Azure blobs, set to false if you want to continue indexing when an unsupported content type is encountered, and you don't know all the content types (file extensions) in advance." + }, + "failOnUnprocessableDocument": { + "type": "boolean", + "default": true, + "description": "For Azure blobs, set to false if you want to continue indexing if a document fails indexing." + }, + "indexStorageMetadataOnlyForOversizedDocuments": { + "type": "boolean", + "default": false, + "description": "For Azure blobs, set this property to true to still index storage metadata for blob content that is too large to process. Oversized blobs are treated as errors by default. For limits on blob size, see https://docs.microsoft.com/azure/search/search-limits-quotas-capacity." + }, + "delimitedTextHeaders": { + "type": "string", + "description": "For CSV blobs, specifies a comma-delimited list of column headers, useful for mapping source fields to destination fields in an index." + }, + "delimitedTextDelimiter": { + "type": "string", + "description": "For CSV blobs, specifies the end-of-line single-character delimiter for CSV files where each line starts a new document (for example, \"|\")." + }, + "firstLineContainsHeaders": { + "type": "boolean", + "default": true, + "description": "For CSV blobs, indicates that the first (non-blank) line of each blob contains headers." + }, + "documentRoot": { + "type": "string", + "description": "For JSON arrays, given a structured or semi-structured document, you can specify a path to the array using this property." + }, + "dataToExtract": { + "type": "string", + "$ref": "#/definitions/DataToExtract" + }, + "imageAction": { + "type": "string", + "$ref": "#/definitions/ImageAction" + }, + "allowSkillsetToReadFileData": { + "type": "boolean", + "default": false, + "description": "If true, will create a path //document//file_data that is an object representing the original file data downloaded from your blob data source. This allows you to pass the original file data to a custom skill for processing within the enrichment pipeline, or to the Document Extraction skill. The object generated will be defined as follows: { /"$type/": /"file/", /"data/": /"BASE64 encoded string of the file/" }" + }, + "pdfTextRotationAlgorithm": { + "type": "string", + "$ref": "#/definitions/PdfTextRotationAlgorithm" + }, + "queryTimeout": { + "type": "string", + "description": "Increases the timeout beyond the 5-minute default for Azure SQL database data sources, specified in the format \"hh:mm:ss\"." + } + }, "additionalProperties": { "type": "object" }, From 2c5820026825d03a6209905dd83e5d87f604640a Mon Sep 17 00:00:00 2001 From: Jennifer Marsman Date: Wed, 24 Jun 2020 23:14:15 -0400 Subject: [PATCH 20/38] Removed bad formatting so JSON is valid --- .../Azure.Search/preview/2019-05-06-preview/searchservice.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specification/search/data-plane/Azure.Search/preview/2019-05-06-preview/searchservice.json b/specification/search/data-plane/Azure.Search/preview/2019-05-06-preview/searchservice.json index c4066cd5186b..d15b6bb179ee 100644 --- a/specification/search/data-plane/Azure.Search/preview/2019-05-06-preview/searchservice.json +++ b/specification/search/data-plane/Azure.Search/preview/2019-05-06-preview/searchservice.json @@ -5837,7 +5837,7 @@ "allowSkillsetToReadFileData": { "type": "boolean", "default": false, - "description": "If true, will create a path //document//file_data that is an object representing the original file data downloaded from your blob data source. This allows you to pass the original file data to a custom skill for processing within the enrichment pipeline, or to the Document Extraction skill. The object generated will be defined as follows: { /"$type/": /"file/", /"data/": /"BASE64 encoded string of the file/" }" + "description": "If true, will create a path //document//file_data that is an object representing the original file data downloaded from your blob data source. This allows you to pass the original file data to a custom skill for processing within the enrichment pipeline, or to the Document Extraction skill." }, "pdfTextRotationAlgorithm": { "type": "string", From 29fd239ddf05ba6c1c329522549300aa4992e80b Mon Sep 17 00:00:00 2001 From: Jennifer Marsman Date: Thu, 25 Jun 2020 00:03:53 -0400 Subject: [PATCH 21/38] Added 3 words to custom-words list --- custom-words.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/custom-words.txt b/custom-words.txt index 2301b3ecdcc3..5ff90252c4f8 100644 --- a/custom-words.txt +++ b/custom-words.txt @@ -1100,6 +1100,7 @@ Paypal PBIE PCIE PCNET +PDFs peerings Pendingissuance Pendingrevocation @@ -1144,6 +1145,7 @@ powerbidedicated powerbiembedded poweredoff poweroff +pptx prebackup prebackupvalidation prebuilts @@ -1652,6 +1654,7 @@ unparsed unprepare unprepares unpreparing +Unprocessable Unprovision unprovisioned Unprovisioning From 5e63f31a42e4d39a8b7a2c4b71d0a1493a75519e Mon Sep 17 00:00:00 2001 From: Jennifer Marsman Date: Thu, 25 Jun 2020 09:52:05 -0400 Subject: [PATCH 22/38] Marking IndexingParameter extra properties as not required --- .../2019-05-06-preview/searchservice.json | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/specification/search/data-plane/Azure.Search/preview/2019-05-06-preview/searchservice.json b/specification/search/data-plane/Azure.Search/preview/2019-05-06-preview/searchservice.json index d15b6bb179ee..150a33475b79 100644 --- a/specification/search/data-plane/Azure.Search/preview/2019-05-06-preview/searchservice.json +++ b/specification/search/data-plane/Azure.Search/preview/2019-05-06-preview/searchservice.json @@ -5781,70 +5781,86 @@ "description": "The maximum number of items in a single batch that can fail indexing for the batch to still be considered successful. -1 means no limit. Default is 0." }, "configuration": { + "type": "object", "properties": { "parsingMode": { "type": "string", + "required": false, "$ref": "#/definitions/ParsingMode" }, "excludedFileNameExtensions": { "type": "string", + "required": false, "description": "Comma-delimited list of filename extensions to ignore when processing from Azure blob storage. For example, you could exclude \".png, .mp4\" to skip over those files during indexing." }, "indexedFileNameExtensions": { "type": "string", + "required": false, "description": "Comma-delimited list of filename extensions to select when processing from Azure blob storage. For example, you could focus indexing on specific application files \".docx, .pptx, .msg\" to specifically include those file types." }, "failOnUnsupportedContentType": { "type": "boolean", + "required": false, "default": true, "description": "For Azure blobs, set to false if you want to continue indexing when an unsupported content type is encountered, and you don't know all the content types (file extensions) in advance." }, "failOnUnprocessableDocument": { "type": "boolean", + "required": false, "default": true, "description": "For Azure blobs, set to false if you want to continue indexing if a document fails indexing." }, "indexStorageMetadataOnlyForOversizedDocuments": { "type": "boolean", + "required": false, "default": false, "description": "For Azure blobs, set this property to true to still index storage metadata for blob content that is too large to process. Oversized blobs are treated as errors by default. For limits on blob size, see https://docs.microsoft.com/azure/search/search-limits-quotas-capacity." }, "delimitedTextHeaders": { "type": "string", + "required": false, "description": "For CSV blobs, specifies a comma-delimited list of column headers, useful for mapping source fields to destination fields in an index." }, "delimitedTextDelimiter": { "type": "string", + "required": false, "description": "For CSV blobs, specifies the end-of-line single-character delimiter for CSV files where each line starts a new document (for example, \"|\")." }, "firstLineContainsHeaders": { "type": "boolean", + "required": false, "default": true, "description": "For CSV blobs, indicates that the first (non-blank) line of each blob contains headers." }, "documentRoot": { "type": "string", + "required": false, "description": "For JSON arrays, given a structured or semi-structured document, you can specify a path to the array using this property." }, "dataToExtract": { "type": "string", + "required": false, "$ref": "#/definitions/DataToExtract" }, "imageAction": { "type": "string", + "required": false, "$ref": "#/definitions/ImageAction" }, "allowSkillsetToReadFileData": { "type": "boolean", + "required": false, "default": false, "description": "If true, will create a path //document//file_data that is an object representing the original file data downloaded from your blob data source. This allows you to pass the original file data to a custom skill for processing within the enrichment pipeline, or to the Document Extraction skill." }, "pdfTextRotationAlgorithm": { "type": "string", + "required": false, "$ref": "#/definitions/PdfTextRotationAlgorithm" }, "queryTimeout": { "type": "string", + "required": false, "description": "Increases the timeout beyond the 5-minute default for Azure SQL database data sources, specified in the format \"hh:mm:ss\"." } }, From ed1504c7da2a2dc184ffb7f3b6263f48150517e2 Mon Sep 17 00:00:00 2001 From: Jennifer Marsman Date: Thu, 25 Jun 2020 18:41:56 -0400 Subject: [PATCH 23/38] Removed duplicate type definitions with $ref, changed default values to false for failOn* properties, and removed en-us locale from URL --- .../preview/2019-05-06-preview/searchservice.json | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/specification/search/data-plane/Azure.Search/preview/2019-05-06-preview/searchservice.json b/specification/search/data-plane/Azure.Search/preview/2019-05-06-preview/searchservice.json index 150a33475b79..3bf99635a584 100644 --- a/specification/search/data-plane/Azure.Search/preview/2019-05-06-preview/searchservice.json +++ b/specification/search/data-plane/Azure.Search/preview/2019-05-06-preview/searchservice.json @@ -5784,7 +5784,6 @@ "type": "object", "properties": { "parsingMode": { - "type": "string", "required": false, "$ref": "#/definitions/ParsingMode" }, @@ -5801,13 +5800,13 @@ "failOnUnsupportedContentType": { "type": "boolean", "required": false, - "default": true, + "default": false, "description": "For Azure blobs, set to false if you want to continue indexing when an unsupported content type is encountered, and you don't know all the content types (file extensions) in advance." }, "failOnUnprocessableDocument": { "type": "boolean", "required": false, - "default": true, + "default": false, "description": "For Azure blobs, set to false if you want to continue indexing if a document fails indexing." }, "indexStorageMetadataOnlyForOversizedDocuments": { @@ -5838,12 +5837,10 @@ "description": "For JSON arrays, given a structured or semi-structured document, you can specify a path to the array using this property." }, "dataToExtract": { - "type": "string", "required": false, "$ref": "#/definitions/DataToExtract" }, "imageAction": { - "type": "string", "required": false, "$ref": "#/definitions/ImageAction" }, @@ -5854,7 +5851,6 @@ "description": "If true, will create a path //document//file_data that is an object representing the original file data downloaded from your blob data source. This allows you to pass the original file data to a custom skill for processing within the enrichment pipeline, or to the Document Extraction skill." }, "pdfTextRotationAlgorithm": { - "type": "string", "required": false, "$ref": "#/definitions/PdfTextRotationAlgorithm" }, @@ -5872,7 +5868,7 @@ }, "description": "Represents parameters for indexer execution.", "externalDocs": { - "url": "https://docs.microsoft.com/en-us/rest/api/searchservice/create-indexer#parameters" + "url": "https://docs.microsoft.com/rest/api/searchservice/create-indexer#parameters" } }, "ParsingMode": { From e9c606a10ee15e4246286a09bd3b26ac3abd994f Mon Sep 17 00:00:00 2001 From: Arvind Krishnaa Jagannathan Date: Thu, 23 Jul 2020 16:29:54 -0700 Subject: [PATCH 24/38] Add swagger spec for indexer execution environment --- .../2019-05-06-preview/searchservice.json | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/specification/search/data-plane/Azure.Search/preview/2019-05-06-preview/searchservice.json b/specification/search/data-plane/Azure.Search/preview/2019-05-06-preview/searchservice.json index 3bf99635a584..a92dfcd75155 100644 --- a/specification/search/data-plane/Azure.Search/preview/2019-05-06-preview/searchservice.json +++ b/specification/search/data-plane/Azure.Search/preview/2019-05-06-preview/searchservice.json @@ -5854,6 +5854,10 @@ "required": false, "$ref": "#/definitions/PdfTextRotationAlgorithm" }, + "executionEnvironment": { + "required": false, + "$ref": "#/definitions/ExecutionEnvironment" + }, "queryTimeout": { "type": "string", "required": false, @@ -6011,6 +6015,32 @@ "default": "none", "description": "Determines algorithm for text extraction from PDF files in Azure blob storage." }, + "ExecutionEnvironment": { + "type": "string", + "enum": [ + "standard", + "private" + ], + "x-ms-enum": { + "name": "IndexerExecutionEnvironment", + "modelAsString": true, + "values": [ + { + "value": "standard", + "name": "standard", + "description": "Indicates that Azure Cognitive Search can determine where the indexer should execute. This is the default environment when nothing is specified and is the recommended value." + }, + { + "value": "private", + "name": "private", + "description": "Indicates that the indexer should run with the environment provisioned specifically for the search service. This should only be specified as the execution environment if the indexer needs to access resources securely over shared private link resources." + } + ] + }, + "x-nullable": true, + "default": "none", + "description": "Specifies the environment in which the indexer should execute." + }, "FieldMappingFunction": { "properties": { "name": { From c3d358168568b44a9c1c35d033fcd982c5432a8f Mon Sep 17 00:00:00 2001 From: Arvind Krishnaa Jagannathan Date: Fri, 24 Jul 2020 11:12:09 -0700 Subject: [PATCH 25/38] Change the default value to 'standard' --- .../Azure.Search/preview/2019-05-06-preview/searchservice.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specification/search/data-plane/Azure.Search/preview/2019-05-06-preview/searchservice.json b/specification/search/data-plane/Azure.Search/preview/2019-05-06-preview/searchservice.json index a92dfcd75155..b0b977a128d2 100644 --- a/specification/search/data-plane/Azure.Search/preview/2019-05-06-preview/searchservice.json +++ b/specification/search/data-plane/Azure.Search/preview/2019-05-06-preview/searchservice.json @@ -6038,7 +6038,7 @@ ] }, "x-nullable": true, - "default": "none", + "default": "standard", "description": "Specifies the environment in which the indexer should execute." }, "FieldMappingFunction": { From 3cefddaa2793dc838632fb6aa72ef7eef564bc18 Mon Sep 17 00:00:00 2001 From: Jennifer Marsman Date: Fri, 24 Jul 2020 14:18:19 -0400 Subject: [PATCH 26/38] Updating 2020 API swaggers with IndexingParameters in Cognitive Search --- .../2020-06-30-Preview/searchservice.json | 223 +++++++++++++++++- .../preview/2020-06-30/searchservice.json | 223 +++++++++++++++++- 2 files changed, 444 insertions(+), 2 deletions(-) diff --git a/specification/search/data-plane/Azure.Search/preview/2020-06-30-Preview/searchservice.json b/specification/search/data-plane/Azure.Search/preview/2020-06-30-Preview/searchservice.json index 4bf96e5ca874..c9b4f31b2023 100644 --- a/specification/search/data-plane/Azure.Search/preview/2020-06-30-Preview/searchservice.json +++ b/specification/search/data-plane/Azure.Search/preview/2020-06-30-Preview/searchservice.json @@ -5789,13 +5789,234 @@ }, "configuration": { "type": "object", + "properties": { + "parsingMode": { + "required": false, + "$ref": "#/definitions/ParsingMode" + }, + "excludedFileNameExtensions": { + "type": "string", + "required": false, + "description": "Comma-delimited list of filename extensions to ignore when processing from Azure blob storage. For example, you could exclude \".png, .mp4\" to skip over those files during indexing." + }, + "indexedFileNameExtensions": { + "type": "string", + "required": false, + "description": "Comma-delimited list of filename extensions to select when processing from Azure blob storage. For example, you could focus indexing on specific application files \".docx, .pptx, .msg\" to specifically include those file types." + }, + "failOnUnsupportedContentType": { + "type": "boolean", + "required": false, + "default": false, + "description": "For Azure blobs, set to false if you want to continue indexing when an unsupported content type is encountered, and you don't know all the content types (file extensions) in advance." + }, + "failOnUnprocessableDocument": { + "type": "boolean", + "required": false, + "default": false, + "description": "For Azure blobs, set to false if you want to continue indexing if a document fails indexing." + }, + "indexStorageMetadataOnlyForOversizedDocuments": { + "type": "boolean", + "required": false, + "default": false, + "description": "For Azure blobs, set this property to true to still index storage metadata for blob content that is too large to process. Oversized blobs are treated as errors by default. For limits on blob size, see https://docs.microsoft.com/azure/search/search-limits-quotas-capacity." + }, + "delimitedTextHeaders": { + "type": "string", + "required": false, + "description": "For CSV blobs, specifies a comma-delimited list of column headers, useful for mapping source fields to destination fields in an index." + }, + "delimitedTextDelimiter": { + "type": "string", + "required": false, + "description": "For CSV blobs, specifies the end-of-line single-character delimiter for CSV files where each line starts a new document (for example, \"|\")." + }, + "firstLineContainsHeaders": { + "type": "boolean", + "required": false, + "default": true, + "description": "For CSV blobs, indicates that the first (non-blank) line of each blob contains headers." + }, + "documentRoot": { + "type": "string", + "required": false, + "description": "For JSON arrays, given a structured or semi-structured document, you can specify a path to the array using this property." + }, + "dataToExtract": { + "required": false, + "$ref": "#/definitions/DataToExtract" + }, + "imageAction": { + "required": false, + "$ref": "#/definitions/ImageAction" + }, + "allowSkillsetToReadFileData": { + "type": "boolean", + "required": false, + "default": false, + "description": "If true, will create a path //document//file_data that is an object representing the original file data downloaded from your blob data source. This allows you to pass the original file data to a custom skill for processing within the enrichment pipeline, or to the Document Extraction skill." + }, + "pdfTextRotationAlgorithm": { + "required": false, + "$ref": "#/definitions/PdfTextRotationAlgorithm" + }, + "queryTimeout": { + "type": "string", + "required": false, + "description": "Increases the timeout beyond the 5-minute default for Azure SQL database data sources, specified in the format \"hh:mm:ss\"." + } + }, "additionalProperties": { "type": "object" }, "description": "A dictionary of indexer-specific configuration properties. Each name is the name of a specific property. Each value must be of a primitive type." } }, - "description": "Represents parameters for indexer execution." + "description": "Represents parameters for indexer execution.", + "externalDocs": { + "url": "https://docs.microsoft.com/rest/api/searchservice/create-indexer#parameters" + } + }, + "ParsingMode": { + "type": "string", + "enum": [ + "default", + "text", + "delimitedText", + "json", + "jsonArray", + "jsonLines" + ], + "x-ms-enum": { + "name": "BlobIndexerParsingMode", + "modelAsString": true, + "values": [ + { + "value": "default", + "name": "Default", + "description": "Set to default for normal file processing." + }, + { + "value": "text", + "name": "Text", + "description": "Set to text to improve indexing performance on plain text files in blob storage." + }, + { + "value": "delimitedText", + "name": "DelimitedText", + "description": "Set to delimitedText when blobs are plain CSV files." + }, + { + "value": "json", + "name": "Json", + "description": "Set to json to extract structured content from JSON files." + }, + { + "value": "jsonArray", + "name": "JsonArray", + "description": "Set to jsonArray to extract individual elements of a JSON array as separate documents in Azure Cognitive Search." + }, + { + "value": "jsonLines", + "name": "JsonLines", + "description": "Set to jsonLines to extract individual JSON entities, separated by a new line, as separate documents in Azure Cognitive Search." + } + ] + }, + "x-nullable": true, + "default": "default", + "description": "Represents the parsing mode for indexing from an Azure blob data source." + }, + "DataToExtract": { + "type": "string", + "enum": [ + "storageMetadata", + "allMetadata", + "contentAndMetadata" + ], + "x-ms-enum": { + "name": "BlobIndexerDataToExtract", + "modelAsString": true, + "values": [ + { + "value": "storageMetadata", + "name": "StorageMetadata", + "description": "Indexes just the standard blob properties and user-specified metadata." + }, + { + "value": "allMetadata", + "name": "AllMetadata", + "description": "Extracts metadata provided by the Azure blob storage subsystem and the content-type specific metadata (for example, metadata unique to just .png files are indexed)." + }, + { + "value": "contentAndMetadata", + "name": "ContentAndMetadata", + "description": "Extracts all metadata and textual content from each blob." + } + ] + }, + "x-nullable": true, + "default": "contentAndMetadata", + "description": "Specifies the data to extract from Azure blob storage and tells the indexer which data to extract from image content when \"imageAction\" is set to a value other than \"none\". This applies to embedded image content in a .PDF or other application, or image files such as .jpg and .png, in Azure blobs." + }, + "ImageAction": { + "type": "string", + "enum": [ + "none", + "generateNormalizedImages", + "generateNormalizedImagePerPage" + ], + "x-ms-enum": { + "name": "BlobIndexerImageAction", + "modelAsString": true, + "values": [ + { + "value": "none", + "name": "None", + "description": "Ignores embedded images or image files in the data set. This is the default." + }, + { + "value": "generateNormalizedImages", + "name": "GenerateNormalizedImages", + "description": "Extracts text from images (for example, the word \"STOP\" from a traffic stop sign), and embeds it into the content field. This action requires that \"dataToExtract\" is set to \"contentAndMetadata\". A normalized image refers to additional processing resulting in uniform image output, sized and rotated to promote consistent rendering when you include images in visual search results. This information is generated for each image when you use this option." + }, + { + "value": "generateNormalizedImagePerPage", + "name": "GenerateNormalizedImagePerPage", + "description": "Extracts text from images (for example, the word \"STOP\" from a traffic stop sign), and embeds it into the content field, but treats PDF files differently in that each page will be rendered as an image and normalized accordingly, instead of extracting embedded images. Non-PDF file types will be treated the same as if \"generateNormalizedImages\" was set." + } + ] + }, + "x-nullable": true, + "default": "none", + "description": "Determines how to process embedded images and image files in Azure blob storage. Setting the \"imageAction\" configuration to any value other than \"none\" requires that a skillset also be attached to that indexer." + }, + "PdfTextRotationAlgorithm": { + "type": "string", + "enum": [ + "none", + "detectAngles" + ], + "x-ms-enum": { + "name": "BlobIndexerPDFTextRotationAlgorithm", + "modelAsString": true, + "values": [ + { + "value": "none", + "name": "None", + "description": "Leverages normal text extraction. This is the default." + }, + { + "value": "detectAngles", + "name": "DetectAngles", + "description": "May produce better and more readable text extraction from PDF files that have rotated text within them. Note that there may be a small performance speed impact when this parameter is used. This parameter only applies to PDF files, and only to PDFs with embedded text. If the rotated text appears within an embedded image in the PDF, this parameter does not apply." + } + ] + }, + "x-nullable": true, + "default": "none", + "description": "Determines algorithm for text extraction from PDF files in Azure blob storage." }, "FieldMappingFunction": { "properties": { diff --git a/specification/search/data-plane/Azure.Search/preview/2020-06-30/searchservice.json b/specification/search/data-plane/Azure.Search/preview/2020-06-30/searchservice.json index a3229ec0e5dd..6f0afc20595d 100644 --- a/specification/search/data-plane/Azure.Search/preview/2020-06-30/searchservice.json +++ b/specification/search/data-plane/Azure.Search/preview/2020-06-30/searchservice.json @@ -5789,13 +5789,234 @@ }, "configuration": { "type": "object", + "properties": { + "parsingMode": { + "required": false, + "$ref": "#/definitions/ParsingMode" + }, + "excludedFileNameExtensions": { + "type": "string", + "required": false, + "description": "Comma-delimited list of filename extensions to ignore when processing from Azure blob storage. For example, you could exclude \".png, .mp4\" to skip over those files during indexing." + }, + "indexedFileNameExtensions": { + "type": "string", + "required": false, + "description": "Comma-delimited list of filename extensions to select when processing from Azure blob storage. For example, you could focus indexing on specific application files \".docx, .pptx, .msg\" to specifically include those file types." + }, + "failOnUnsupportedContentType": { + "type": "boolean", + "required": false, + "default": false, + "description": "For Azure blobs, set to false if you want to continue indexing when an unsupported content type is encountered, and you don't know all the content types (file extensions) in advance." + }, + "failOnUnprocessableDocument": { + "type": "boolean", + "required": false, + "default": false, + "description": "For Azure blobs, set to false if you want to continue indexing if a document fails indexing." + }, + "indexStorageMetadataOnlyForOversizedDocuments": { + "type": "boolean", + "required": false, + "default": false, + "description": "For Azure blobs, set this property to true to still index storage metadata for blob content that is too large to process. Oversized blobs are treated as errors by default. For limits on blob size, see https://docs.microsoft.com/azure/search/search-limits-quotas-capacity." + }, + "delimitedTextHeaders": { + "type": "string", + "required": false, + "description": "For CSV blobs, specifies a comma-delimited list of column headers, useful for mapping source fields to destination fields in an index." + }, + "delimitedTextDelimiter": { + "type": "string", + "required": false, + "description": "For CSV blobs, specifies the end-of-line single-character delimiter for CSV files where each line starts a new document (for example, \"|\")." + }, + "firstLineContainsHeaders": { + "type": "boolean", + "required": false, + "default": true, + "description": "For CSV blobs, indicates that the first (non-blank) line of each blob contains headers." + }, + "documentRoot": { + "type": "string", + "required": false, + "description": "For JSON arrays, given a structured or semi-structured document, you can specify a path to the array using this property." + }, + "dataToExtract": { + "required": false, + "$ref": "#/definitions/DataToExtract" + }, + "imageAction": { + "required": false, + "$ref": "#/definitions/ImageAction" + }, + "allowSkillsetToReadFileData": { + "type": "boolean", + "required": false, + "default": false, + "description": "If true, will create a path //document//file_data that is an object representing the original file data downloaded from your blob data source. This allows you to pass the original file data to a custom skill for processing within the enrichment pipeline, or to the Document Extraction skill." + }, + "pdfTextRotationAlgorithm": { + "required": false, + "$ref": "#/definitions/PdfTextRotationAlgorithm" + }, + "queryTimeout": { + "type": "string", + "required": false, + "description": "Increases the timeout beyond the 5-minute default for Azure SQL database data sources, specified in the format \"hh:mm:ss\"." + } + }, "additionalProperties": { "type": "object" }, "description": "A dictionary of indexer-specific configuration properties. Each name is the name of a specific property. Each value must be of a primitive type." } }, - "description": "Represents parameters for indexer execution." + "description": "Represents parameters for indexer execution.", + "externalDocs": { + "url": "https://docs.microsoft.com/rest/api/searchservice/create-indexer#parameters" + } + }, + "ParsingMode": { + "type": "string", + "enum": [ + "default", + "text", + "delimitedText", + "json", + "jsonArray", + "jsonLines" + ], + "x-ms-enum": { + "name": "BlobIndexerParsingMode", + "modelAsString": true, + "values": [ + { + "value": "default", + "name": "Default", + "description": "Set to default for normal file processing." + }, + { + "value": "text", + "name": "Text", + "description": "Set to text to improve indexing performance on plain text files in blob storage." + }, + { + "value": "delimitedText", + "name": "DelimitedText", + "description": "Set to delimitedText when blobs are plain CSV files." + }, + { + "value": "json", + "name": "Json", + "description": "Set to json to extract structured content from JSON files." + }, + { + "value": "jsonArray", + "name": "JsonArray", + "description": "Set to jsonArray to extract individual elements of a JSON array as separate documents in Azure Cognitive Search." + }, + { + "value": "jsonLines", + "name": "JsonLines", + "description": "Set to jsonLines to extract individual JSON entities, separated by a new line, as separate documents in Azure Cognitive Search." + } + ] + }, + "x-nullable": true, + "default": "default", + "description": "Represents the parsing mode for indexing from an Azure blob data source." + }, + "DataToExtract": { + "type": "string", + "enum": [ + "storageMetadata", + "allMetadata", + "contentAndMetadata" + ], + "x-ms-enum": { + "name": "BlobIndexerDataToExtract", + "modelAsString": true, + "values": [ + { + "value": "storageMetadata", + "name": "StorageMetadata", + "description": "Indexes just the standard blob properties and user-specified metadata." + }, + { + "value": "allMetadata", + "name": "AllMetadata", + "description": "Extracts metadata provided by the Azure blob storage subsystem and the content-type specific metadata (for example, metadata unique to just .png files are indexed)." + }, + { + "value": "contentAndMetadata", + "name": "ContentAndMetadata", + "description": "Extracts all metadata and textual content from each blob." + } + ] + }, + "x-nullable": true, + "default": "contentAndMetadata", + "description": "Specifies the data to extract from Azure blob storage and tells the indexer which data to extract from image content when \"imageAction\" is set to a value other than \"none\". This applies to embedded image content in a .PDF or other application, or image files such as .jpg and .png, in Azure blobs." + }, + "ImageAction": { + "type": "string", + "enum": [ + "none", + "generateNormalizedImages", + "generateNormalizedImagePerPage" + ], + "x-ms-enum": { + "name": "BlobIndexerImageAction", + "modelAsString": true, + "values": [ + { + "value": "none", + "name": "None", + "description": "Ignores embedded images or image files in the data set. This is the default." + }, + { + "value": "generateNormalizedImages", + "name": "GenerateNormalizedImages", + "description": "Extracts text from images (for example, the word \"STOP\" from a traffic stop sign), and embeds it into the content field. This action requires that \"dataToExtract\" is set to \"contentAndMetadata\". A normalized image refers to additional processing resulting in uniform image output, sized and rotated to promote consistent rendering when you include images in visual search results. This information is generated for each image when you use this option." + }, + { + "value": "generateNormalizedImagePerPage", + "name": "GenerateNormalizedImagePerPage", + "description": "Extracts text from images (for example, the word \"STOP\" from a traffic stop sign), and embeds it into the content field, but treats PDF files differently in that each page will be rendered as an image and normalized accordingly, instead of extracting embedded images. Non-PDF file types will be treated the same as if \"generateNormalizedImages\" was set." + } + ] + }, + "x-nullable": true, + "default": "none", + "description": "Determines how to process embedded images and image files in Azure blob storage. Setting the \"imageAction\" configuration to any value other than \"none\" requires that a skillset also be attached to that indexer." + }, + "PdfTextRotationAlgorithm": { + "type": "string", + "enum": [ + "none", + "detectAngles" + ], + "x-ms-enum": { + "name": "BlobIndexerPDFTextRotationAlgorithm", + "modelAsString": true, + "values": [ + { + "value": "none", + "name": "None", + "description": "Leverages normal text extraction. This is the default." + }, + { + "value": "detectAngles", + "name": "DetectAngles", + "description": "May produce better and more readable text extraction from PDF files that have rotated text within them. Note that there may be a small performance speed impact when this parameter is used. This parameter only applies to PDF files, and only to PDFs with embedded text. If the rotated text appears within an embedded image in the PDF, this parameter does not apply." + } + ] + }, + "x-nullable": true, + "default": "none", + "description": "Determines algorithm for text extraction from PDF files in Azure blob storage." }, "FieldMappingFunction": { "properties": { From f349d809eae0c0595d00f065380f6f0f49760416 Mon Sep 17 00:00:00 2001 From: Arvind Krishnaa Jagannathan Date: Fri, 24 Jul 2020 11:27:52 -0700 Subject: [PATCH 27/38] Add swagger spec for indexer execution environment in 2 new API versions --- .../2020-06-30-Preview/searchservice.json | 30 +++++++++++++++++++ .../preview/2020-06-30/searchservice.json | 30 +++++++++++++++++++ 2 files changed, 60 insertions(+) diff --git a/specification/search/data-plane/Azure.Search/preview/2020-06-30-Preview/searchservice.json b/specification/search/data-plane/Azure.Search/preview/2020-06-30-Preview/searchservice.json index c9b4f31b2023..f54c7e8cf665 100644 --- a/specification/search/data-plane/Azure.Search/preview/2020-06-30-Preview/searchservice.json +++ b/specification/search/data-plane/Azure.Search/preview/2020-06-30-Preview/searchservice.json @@ -5861,6 +5861,10 @@ "required": false, "$ref": "#/definitions/PdfTextRotationAlgorithm" }, + "executionEnvironment": { + "required": false, + "$ref": "#/definitions/ExecutionEnvironment" + }, "queryTimeout": { "type": "string", "required": false, @@ -6018,6 +6022,32 @@ "default": "none", "description": "Determines algorithm for text extraction from PDF files in Azure blob storage." }, + "ExecutionEnvironment": { + "type": "string", + "enum": [ + "standard", + "private" + ], + "x-ms-enum": { + "name": "IndexerExecutionEnvironment", + "modelAsString": true, + "values": [ + { + "value": "standard", + "name": "standard", + "description": "Indicates that Azure Cognitive Search can determine where the indexer should execute. This is the default environment when nothing is specified and is the recommended value." + }, + { + "value": "private", + "name": "private", + "description": "Indicates that the indexer should run with the environment provisioned specifically for the search service. This should only be specified as the execution environment if the indexer needs to access resources securely over shared private link resources." + } + ] + }, + "x-nullable": true, + "default": "standard", + "description": "Specifies the environment in which the indexer should execute." + }, "FieldMappingFunction": { "properties": { "name": { diff --git a/specification/search/data-plane/Azure.Search/preview/2020-06-30/searchservice.json b/specification/search/data-plane/Azure.Search/preview/2020-06-30/searchservice.json index 6f0afc20595d..698055b4a829 100644 --- a/specification/search/data-plane/Azure.Search/preview/2020-06-30/searchservice.json +++ b/specification/search/data-plane/Azure.Search/preview/2020-06-30/searchservice.json @@ -5861,6 +5861,10 @@ "required": false, "$ref": "#/definitions/PdfTextRotationAlgorithm" }, + "executionEnvironment": { + "required": false, + "$ref": "#/definitions/ExecutionEnvironment" + }, "queryTimeout": { "type": "string", "required": false, @@ -6018,6 +6022,32 @@ "default": "none", "description": "Determines algorithm for text extraction from PDF files in Azure blob storage." }, + "ExecutionEnvironment": { + "type": "string", + "enum": [ + "standard", + "private" + ], + "x-ms-enum": { + "name": "IndexerExecutionEnvironment", + "modelAsString": true, + "values": [ + { + "value": "standard", + "name": "standard", + "description": "Indicates that Azure Cognitive Search can determine where the indexer should execute. This is the default environment when nothing is specified and is the recommended value." + }, + { + "value": "private", + "name": "private", + "description": "Indicates that the indexer should run with the environment provisioned specifically for the search service. This should only be specified as the execution environment if the indexer needs to access resources securely over shared private link resources." + } + ] + }, + "x-nullable": true, + "default": "standard", + "description": "Specifies the environment in which the indexer should execute." + }, "FieldMappingFunction": { "properties": { "name": { From e91f3653562b25516e5bb0f0f7dc08d05edde0a3 Mon Sep 17 00:00:00 2001 From: Jennifer Marsman Date: Mon, 27 Jul 2020 10:19:39 -0400 Subject: [PATCH 28/38] Added default value for queryTimeout parameter --- .../Azure.Search/preview/2020-06-30-Preview/searchservice.json | 1 + .../Azure.Search/preview/2020-06-30/searchservice.json | 1 + 2 files changed, 2 insertions(+) diff --git a/specification/search/data-plane/Azure.Search/preview/2020-06-30-Preview/searchservice.json b/specification/search/data-plane/Azure.Search/preview/2020-06-30-Preview/searchservice.json index f54c7e8cf665..b27b56be3a36 100644 --- a/specification/search/data-plane/Azure.Search/preview/2020-06-30-Preview/searchservice.json +++ b/specification/search/data-plane/Azure.Search/preview/2020-06-30-Preview/searchservice.json @@ -5868,6 +5868,7 @@ "queryTimeout": { "type": "string", "required": false, + "default": "00:05:00", "description": "Increases the timeout beyond the 5-minute default for Azure SQL database data sources, specified in the format \"hh:mm:ss\"." } }, diff --git a/specification/search/data-plane/Azure.Search/preview/2020-06-30/searchservice.json b/specification/search/data-plane/Azure.Search/preview/2020-06-30/searchservice.json index 698055b4a829..a7883cc6f66a 100644 --- a/specification/search/data-plane/Azure.Search/preview/2020-06-30/searchservice.json +++ b/specification/search/data-plane/Azure.Search/preview/2020-06-30/searchservice.json @@ -5868,6 +5868,7 @@ "queryTimeout": { "type": "string", "required": false, + "default": "00:05:00", "description": "Increases the timeout beyond the 5-minute default for Azure SQL database data sources, specified in the format \"hh:mm:ss\"." } }, From 7ad4e47540c914de3dd0570ddb2f6aee0a6a74bb Mon Sep 17 00:00:00 2001 From: Jennifer Marsman Date: Mon, 27 Jul 2020 10:59:29 -0400 Subject: [PATCH 29/38] Removing indexing parameter changes from 2019-05-06-preview version of swagger --- .../2019-05-06-preview/searchservice.json | 253 +----------------- 1 file changed, 1 insertion(+), 252 deletions(-) diff --git a/specification/search/data-plane/Azure.Search/preview/2019-05-06-preview/searchservice.json b/specification/search/data-plane/Azure.Search/preview/2019-05-06-preview/searchservice.json index b0b977a128d2..ddead0180328 100644 --- a/specification/search/data-plane/Azure.Search/preview/2019-05-06-preview/searchservice.json +++ b/specification/search/data-plane/Azure.Search/preview/2019-05-06-preview/searchservice.json @@ -5782,264 +5782,13 @@ }, "configuration": { "type": "object", - "properties": { - "parsingMode": { - "required": false, - "$ref": "#/definitions/ParsingMode" - }, - "excludedFileNameExtensions": { - "type": "string", - "required": false, - "description": "Comma-delimited list of filename extensions to ignore when processing from Azure blob storage. For example, you could exclude \".png, .mp4\" to skip over those files during indexing." - }, - "indexedFileNameExtensions": { - "type": "string", - "required": false, - "description": "Comma-delimited list of filename extensions to select when processing from Azure blob storage. For example, you could focus indexing on specific application files \".docx, .pptx, .msg\" to specifically include those file types." - }, - "failOnUnsupportedContentType": { - "type": "boolean", - "required": false, - "default": false, - "description": "For Azure blobs, set to false if you want to continue indexing when an unsupported content type is encountered, and you don't know all the content types (file extensions) in advance." - }, - "failOnUnprocessableDocument": { - "type": "boolean", - "required": false, - "default": false, - "description": "For Azure blobs, set to false if you want to continue indexing if a document fails indexing." - }, - "indexStorageMetadataOnlyForOversizedDocuments": { - "type": "boolean", - "required": false, - "default": false, - "description": "For Azure blobs, set this property to true to still index storage metadata for blob content that is too large to process. Oversized blobs are treated as errors by default. For limits on blob size, see https://docs.microsoft.com/azure/search/search-limits-quotas-capacity." - }, - "delimitedTextHeaders": { - "type": "string", - "required": false, - "description": "For CSV blobs, specifies a comma-delimited list of column headers, useful for mapping source fields to destination fields in an index." - }, - "delimitedTextDelimiter": { - "type": "string", - "required": false, - "description": "For CSV blobs, specifies the end-of-line single-character delimiter for CSV files where each line starts a new document (for example, \"|\")." - }, - "firstLineContainsHeaders": { - "type": "boolean", - "required": false, - "default": true, - "description": "For CSV blobs, indicates that the first (non-blank) line of each blob contains headers." - }, - "documentRoot": { - "type": "string", - "required": false, - "description": "For JSON arrays, given a structured or semi-structured document, you can specify a path to the array using this property." - }, - "dataToExtract": { - "required": false, - "$ref": "#/definitions/DataToExtract" - }, - "imageAction": { - "required": false, - "$ref": "#/definitions/ImageAction" - }, - "allowSkillsetToReadFileData": { - "type": "boolean", - "required": false, - "default": false, - "description": "If true, will create a path //document//file_data that is an object representing the original file data downloaded from your blob data source. This allows you to pass the original file data to a custom skill for processing within the enrichment pipeline, or to the Document Extraction skill." - }, - "pdfTextRotationAlgorithm": { - "required": false, - "$ref": "#/definitions/PdfTextRotationAlgorithm" - }, - "executionEnvironment": { - "required": false, - "$ref": "#/definitions/ExecutionEnvironment" - }, - "queryTimeout": { - "type": "string", - "required": false, - "description": "Increases the timeout beyond the 5-minute default for Azure SQL database data sources, specified in the format \"hh:mm:ss\"." - } - }, "additionalProperties": { "type": "object" }, "description": "A dictionary of indexer-specific configuration properties. Each name is the name of a specific property. Each value must be of a primitive type." } }, - "description": "Represents parameters for indexer execution.", - "externalDocs": { - "url": "https://docs.microsoft.com/rest/api/searchservice/create-indexer#parameters" - } - }, - "ParsingMode": { - "type": "string", - "enum": [ - "default", - "text", - "delimitedText", - "json", - "jsonArray", - "jsonLines" - ], - "x-ms-enum": { - "name": "BlobIndexerParsingMode", - "modelAsString": true, - "values": [ - { - "value": "default", - "name": "Default", - "description": "Set to default for normal file processing." - }, - { - "value": "text", - "name": "Text", - "description": "Set to text to improve indexing performance on plain text files in blob storage." - }, - { - "value": "delimitedText", - "name": "DelimitedText", - "description": "Set to delimitedText when blobs are plain CSV files." - }, - { - "value": "json", - "name": "Json", - "description": "Set to json to extract structured content from JSON files." - }, - { - "value": "jsonArray", - "name": "JsonArray", - "description": "Set to jsonArray to extract individual elements of a JSON array as separate documents in Azure Cognitive Search." - }, - { - "value": "jsonLines", - "name": "JsonLines", - "description": "Set to jsonLines to extract individual JSON entities, separated by a new line, as separate documents in Azure Cognitive Search." - } - ] - }, - "x-nullable": true, - "default": "default", - "description": "Represents the parsing mode for indexing from an Azure blob data source." - }, - "DataToExtract": { - "type": "string", - "enum": [ - "storageMetadata", - "allMetadata", - "contentAndMetadata" - ], - "x-ms-enum": { - "name": "BlobIndexerDataToExtract", - "modelAsString": true, - "values": [ - { - "value": "storageMetadata", - "name": "StorageMetadata", - "description": "Indexes just the standard blob properties and user-specified metadata." - }, - { - "value": "allMetadata", - "name": "AllMetadata", - "description": "Extracts metadata provided by the Azure blob storage subsystem and the content-type specific metadata (for example, metadata unique to just .png files are indexed)." - }, - { - "value": "contentAndMetadata", - "name": "ContentAndMetadata", - "description": "Extracts all metadata and textual content from each blob." - } - ] - }, - "x-nullable": true, - "default": "contentAndMetadata", - "description": "Specifies the data to extract from Azure blob storage and tells the indexer which data to extract from image content when \"imageAction\" is set to a value other than \"none\". This applies to embedded image content in a .PDF or other application, or image files such as .jpg and .png, in Azure blobs." - }, - "ImageAction": { - "type": "string", - "enum": [ - "none", - "generateNormalizedImages", - "generateNormalizedImagePerPage" - ], - "x-ms-enum": { - "name": "BlobIndexerImageAction", - "modelAsString": true, - "values": [ - { - "value": "none", - "name": "None", - "description": "Ignores embedded images or image files in the data set. This is the default." - }, - { - "value": "generateNormalizedImages", - "name": "GenerateNormalizedImages", - "description": "Extracts text from images (for example, the word \"STOP\" from a traffic stop sign), and embeds it into the content field. This action requires that \"dataToExtract\" is set to \"contentAndMetadata\". A normalized image refers to additional processing resulting in uniform image output, sized and rotated to promote consistent rendering when you include images in visual search results. This information is generated for each image when you use this option." - }, - { - "value": "generateNormalizedImagePerPage", - "name": "GenerateNormalizedImagePerPage", - "description": "Extracts text from images (for example, the word \"STOP\" from a traffic stop sign), and embeds it into the content field, but treats PDF files differently in that each page will be rendered as an image and normalized accordingly, instead of extracting embedded images. Non-PDF file types will be treated the same as if \"generateNormalizedImages\" was set." - } - ] - }, - "x-nullable": true, - "default": "none", - "description": "Determines how to process embedded images and image files in Azure blob storage. Setting the \"imageAction\" configuration to any value other than \"none\" requires that a skillset also be attached to that indexer." - }, - "PdfTextRotationAlgorithm": { - "type": "string", - "enum": [ - "none", - "detectAngles" - ], - "x-ms-enum": { - "name": "BlobIndexerPDFTextRotationAlgorithm", - "modelAsString": true, - "values": [ - { - "value": "none", - "name": "None", - "description": "Leverages normal text extraction. This is the default." - }, - { - "value": "detectAngles", - "name": "DetectAngles", - "description": "May produce better and more readable text extraction from PDF files that have rotated text within them. Note that there may be a small performance speed impact when this parameter is used. This parameter only applies to PDF files, and only to PDFs with embedded text. If the rotated text appears within an embedded image in the PDF, this parameter does not apply." - } - ] - }, - "x-nullable": true, - "default": "none", - "description": "Determines algorithm for text extraction from PDF files in Azure blob storage." - }, - "ExecutionEnvironment": { - "type": "string", - "enum": [ - "standard", - "private" - ], - "x-ms-enum": { - "name": "IndexerExecutionEnvironment", - "modelAsString": true, - "values": [ - { - "value": "standard", - "name": "standard", - "description": "Indicates that Azure Cognitive Search can determine where the indexer should execute. This is the default environment when nothing is specified and is the recommended value." - }, - { - "value": "private", - "name": "private", - "description": "Indicates that the indexer should run with the environment provisioned specifically for the search service. This should only be specified as the execution environment if the indexer needs to access resources securely over shared private link resources." - } - ] - }, - "x-nullable": true, - "default": "standard", - "description": "Specifies the environment in which the indexer should execute." + "description": "Represents parameters for indexer execution." }, "FieldMappingFunction": { "properties": { From f1bcfe069141618521ff34add75bb90f149bc18d Mon Sep 17 00:00:00 2001 From: Jennifer Marsman Date: Tue, 28 Jul 2020 09:32:53 -0400 Subject: [PATCH 30/38] Removed x-nullable from indexing parameter enums --- .../preview/2020-06-30-Preview/searchservice.json | 5 ----- .../Azure.Search/preview/2020-06-30/searchservice.json | 5 ----- 2 files changed, 10 deletions(-) diff --git a/specification/search/data-plane/Azure.Search/preview/2020-06-30-Preview/searchservice.json b/specification/search/data-plane/Azure.Search/preview/2020-06-30-Preview/searchservice.json index b27b56be3a36..af15f66a2084 100644 --- a/specification/search/data-plane/Azure.Search/preview/2020-06-30-Preview/searchservice.json +++ b/specification/search/data-plane/Azure.Search/preview/2020-06-30-Preview/searchservice.json @@ -5929,7 +5929,6 @@ } ] }, - "x-nullable": true, "default": "default", "description": "Represents the parsing mode for indexing from an Azure blob data source." }, @@ -5961,7 +5960,6 @@ } ] }, - "x-nullable": true, "default": "contentAndMetadata", "description": "Specifies the data to extract from Azure blob storage and tells the indexer which data to extract from image content when \"imageAction\" is set to a value other than \"none\". This applies to embedded image content in a .PDF or other application, or image files such as .jpg and .png, in Azure blobs." }, @@ -5993,7 +5991,6 @@ } ] }, - "x-nullable": true, "default": "none", "description": "Determines how to process embedded images and image files in Azure blob storage. Setting the \"imageAction\" configuration to any value other than \"none\" requires that a skillset also be attached to that indexer." }, @@ -6019,7 +6016,6 @@ } ] }, - "x-nullable": true, "default": "none", "description": "Determines algorithm for text extraction from PDF files in Azure blob storage." }, @@ -6045,7 +6041,6 @@ } ] }, - "x-nullable": true, "default": "standard", "description": "Specifies the environment in which the indexer should execute." }, diff --git a/specification/search/data-plane/Azure.Search/preview/2020-06-30/searchservice.json b/specification/search/data-plane/Azure.Search/preview/2020-06-30/searchservice.json index a7883cc6f66a..c096186fd005 100644 --- a/specification/search/data-plane/Azure.Search/preview/2020-06-30/searchservice.json +++ b/specification/search/data-plane/Azure.Search/preview/2020-06-30/searchservice.json @@ -5929,7 +5929,6 @@ } ] }, - "x-nullable": true, "default": "default", "description": "Represents the parsing mode for indexing from an Azure blob data source." }, @@ -5961,7 +5960,6 @@ } ] }, - "x-nullable": true, "default": "contentAndMetadata", "description": "Specifies the data to extract from Azure blob storage and tells the indexer which data to extract from image content when \"imageAction\" is set to a value other than \"none\". This applies to embedded image content in a .PDF or other application, or image files such as .jpg and .png, in Azure blobs." }, @@ -5993,7 +5991,6 @@ } ] }, - "x-nullable": true, "default": "none", "description": "Determines how to process embedded images and image files in Azure blob storage. Setting the \"imageAction\" configuration to any value other than \"none\" requires that a skillset also be attached to that indexer." }, @@ -6019,7 +6016,6 @@ } ] }, - "x-nullable": true, "default": "none", "description": "Determines algorithm for text extraction from PDF files in Azure blob storage." }, @@ -6045,7 +6041,6 @@ } ] }, - "x-nullable": true, "default": "standard", "description": "Specifies the environment in which the indexer should execute." }, From eb9b7a3f3edaa23ae72eacb454a470ae10ba2763 Mon Sep 17 00:00:00 2001 From: Jennifer Marsman Date: Thu, 30 Jul 2020 10:36:51 -0400 Subject: [PATCH 31/38] Removed the required: false since everything is optional/not required by default --- .../2020-06-30-Preview/searchservice.json | 16 ---------------- .../preview/2020-06-30/searchservice.json | 16 ---------------- 2 files changed, 32 deletions(-) diff --git a/specification/search/data-plane/Azure.Search/preview/2020-06-30-Preview/searchservice.json b/specification/search/data-plane/Azure.Search/preview/2020-06-30-Preview/searchservice.json index af15f66a2084..3db2bee6813e 100644 --- a/specification/search/data-plane/Azure.Search/preview/2020-06-30-Preview/searchservice.json +++ b/specification/search/data-plane/Azure.Search/preview/2020-06-30-Preview/searchservice.json @@ -5791,83 +5791,67 @@ "type": "object", "properties": { "parsingMode": { - "required": false, "$ref": "#/definitions/ParsingMode" }, "excludedFileNameExtensions": { "type": "string", - "required": false, "description": "Comma-delimited list of filename extensions to ignore when processing from Azure blob storage. For example, you could exclude \".png, .mp4\" to skip over those files during indexing." }, "indexedFileNameExtensions": { "type": "string", - "required": false, "description": "Comma-delimited list of filename extensions to select when processing from Azure blob storage. For example, you could focus indexing on specific application files \".docx, .pptx, .msg\" to specifically include those file types." }, "failOnUnsupportedContentType": { "type": "boolean", - "required": false, "default": false, "description": "For Azure blobs, set to false if you want to continue indexing when an unsupported content type is encountered, and you don't know all the content types (file extensions) in advance." }, "failOnUnprocessableDocument": { "type": "boolean", - "required": false, "default": false, "description": "For Azure blobs, set to false if you want to continue indexing if a document fails indexing." }, "indexStorageMetadataOnlyForOversizedDocuments": { "type": "boolean", - "required": false, "default": false, "description": "For Azure blobs, set this property to true to still index storage metadata for blob content that is too large to process. Oversized blobs are treated as errors by default. For limits on blob size, see https://docs.microsoft.com/azure/search/search-limits-quotas-capacity." }, "delimitedTextHeaders": { "type": "string", - "required": false, "description": "For CSV blobs, specifies a comma-delimited list of column headers, useful for mapping source fields to destination fields in an index." }, "delimitedTextDelimiter": { "type": "string", - "required": false, "description": "For CSV blobs, specifies the end-of-line single-character delimiter for CSV files where each line starts a new document (for example, \"|\")." }, "firstLineContainsHeaders": { "type": "boolean", - "required": false, "default": true, "description": "For CSV blobs, indicates that the first (non-blank) line of each blob contains headers." }, "documentRoot": { "type": "string", - "required": false, "description": "For JSON arrays, given a structured or semi-structured document, you can specify a path to the array using this property." }, "dataToExtract": { - "required": false, "$ref": "#/definitions/DataToExtract" }, "imageAction": { - "required": false, "$ref": "#/definitions/ImageAction" }, "allowSkillsetToReadFileData": { "type": "boolean", - "required": false, "default": false, "description": "If true, will create a path //document//file_data that is an object representing the original file data downloaded from your blob data source. This allows you to pass the original file data to a custom skill for processing within the enrichment pipeline, or to the Document Extraction skill." }, "pdfTextRotationAlgorithm": { - "required": false, "$ref": "#/definitions/PdfTextRotationAlgorithm" }, "executionEnvironment": { - "required": false, "$ref": "#/definitions/ExecutionEnvironment" }, "queryTimeout": { "type": "string", - "required": false, "default": "00:05:00", "description": "Increases the timeout beyond the 5-minute default for Azure SQL database data sources, specified in the format \"hh:mm:ss\"." } diff --git a/specification/search/data-plane/Azure.Search/preview/2020-06-30/searchservice.json b/specification/search/data-plane/Azure.Search/preview/2020-06-30/searchservice.json index c096186fd005..0ee96bce0b00 100644 --- a/specification/search/data-plane/Azure.Search/preview/2020-06-30/searchservice.json +++ b/specification/search/data-plane/Azure.Search/preview/2020-06-30/searchservice.json @@ -5791,83 +5791,67 @@ "type": "object", "properties": { "parsingMode": { - "required": false, "$ref": "#/definitions/ParsingMode" }, "excludedFileNameExtensions": { "type": "string", - "required": false, "description": "Comma-delimited list of filename extensions to ignore when processing from Azure blob storage. For example, you could exclude \".png, .mp4\" to skip over those files during indexing." }, "indexedFileNameExtensions": { "type": "string", - "required": false, "description": "Comma-delimited list of filename extensions to select when processing from Azure blob storage. For example, you could focus indexing on specific application files \".docx, .pptx, .msg\" to specifically include those file types." }, "failOnUnsupportedContentType": { "type": "boolean", - "required": false, "default": false, "description": "For Azure blobs, set to false if you want to continue indexing when an unsupported content type is encountered, and you don't know all the content types (file extensions) in advance." }, "failOnUnprocessableDocument": { "type": "boolean", - "required": false, "default": false, "description": "For Azure blobs, set to false if you want to continue indexing if a document fails indexing." }, "indexStorageMetadataOnlyForOversizedDocuments": { "type": "boolean", - "required": false, "default": false, "description": "For Azure blobs, set this property to true to still index storage metadata for blob content that is too large to process. Oversized blobs are treated as errors by default. For limits on blob size, see https://docs.microsoft.com/azure/search/search-limits-quotas-capacity." }, "delimitedTextHeaders": { "type": "string", - "required": false, "description": "For CSV blobs, specifies a comma-delimited list of column headers, useful for mapping source fields to destination fields in an index." }, "delimitedTextDelimiter": { "type": "string", - "required": false, "description": "For CSV blobs, specifies the end-of-line single-character delimiter for CSV files where each line starts a new document (for example, \"|\")." }, "firstLineContainsHeaders": { "type": "boolean", - "required": false, "default": true, "description": "For CSV blobs, indicates that the first (non-blank) line of each blob contains headers." }, "documentRoot": { "type": "string", - "required": false, "description": "For JSON arrays, given a structured or semi-structured document, you can specify a path to the array using this property." }, "dataToExtract": { - "required": false, "$ref": "#/definitions/DataToExtract" }, "imageAction": { - "required": false, "$ref": "#/definitions/ImageAction" }, "allowSkillsetToReadFileData": { "type": "boolean", - "required": false, "default": false, "description": "If true, will create a path //document//file_data that is an object representing the original file data downloaded from your blob data source. This allows you to pass the original file data to a custom skill for processing within the enrichment pipeline, or to the Document Extraction skill." }, "pdfTextRotationAlgorithm": { - "required": false, "$ref": "#/definitions/PdfTextRotationAlgorithm" }, "executionEnvironment": { - "required": false, "$ref": "#/definitions/ExecutionEnvironment" }, "queryTimeout": { "type": "string", - "required": false, "default": "00:05:00", "description": "Increases the timeout beyond the 5-minute default for Azure SQL database data sources, specified in the format \"hh:mm:ss\"." } From dd2ca6fcb6aa3b79f49a9214ed5e7875a85e74e4 Mon Sep 17 00:00:00 2001 From: Jennifer Marsman Date: Thu, 30 Jul 2020 17:31:24 -0400 Subject: [PATCH 32/38] Marked batchSize, maxFailedItems, and maxFailedItemsPerBatch as x-nullable --- .../Azure.Search/preview/2020-06-30-Preview/searchservice.json | 2 ++ .../Azure.Search/preview/2020-06-30/searchservice.json | 2 ++ 2 files changed, 4 insertions(+) diff --git a/specification/search/data-plane/Azure.Search/preview/2020-06-30-Preview/searchservice.json b/specification/search/data-plane/Azure.Search/preview/2020-06-30-Preview/searchservice.json index 3db2bee6813e..3a303e0e83a0 100644 --- a/specification/search/data-plane/Azure.Search/preview/2020-06-30-Preview/searchservice.json +++ b/specification/search/data-plane/Azure.Search/preview/2020-06-30-Preview/searchservice.json @@ -5776,6 +5776,7 @@ "maxFailedItems": { "type": "integer", "format": "int32", + "x-nullable": true, "default": 0, "x-nullable": true, "description": "The maximum number of items that can fail indexing for indexer execution to still be considered successful. -1 means no limit. Default is 0." @@ -5783,6 +5784,7 @@ "maxFailedItemsPerBatch": { "type": "integer", "format": "int32", + "x-nullable": true, "default": 0, "x-nullable": true, "description": "The maximum number of items in a single batch that can fail indexing for the batch to still be considered successful. -1 means no limit. Default is 0." diff --git a/specification/search/data-plane/Azure.Search/preview/2020-06-30/searchservice.json b/specification/search/data-plane/Azure.Search/preview/2020-06-30/searchservice.json index 0ee96bce0b00..6e3dd5948bc7 100644 --- a/specification/search/data-plane/Azure.Search/preview/2020-06-30/searchservice.json +++ b/specification/search/data-plane/Azure.Search/preview/2020-06-30/searchservice.json @@ -5776,6 +5776,7 @@ "maxFailedItems": { "type": "integer", "format": "int32", + "x-nullable": true, "default": 0, "x-nullable": true, "description": "The maximum number of items that can fail indexing for indexer execution to still be considered successful. -1 means no limit. Default is 0." @@ -5783,6 +5784,7 @@ "maxFailedItemsPerBatch": { "type": "integer", "format": "int32", + "x-nullable": true, "default": 0, "x-nullable": true, "description": "The maximum number of items in a single batch that can fail indexing for the batch to still be considered successful. -1 means no limit. Default is 0." From 732f032fdc77c2314ab29a270818d2af1e1ec42f Mon Sep 17 00:00:00 2001 From: Jennifer Marsman Date: Mon, 3 Aug 2020 13:15:39 -0400 Subject: [PATCH 33/38] Added x-nullable to configuration parameters --- .../2020-06-30-Preview/searchservice.json | 16 ++++++++++++++++ .../preview/2020-06-30/searchservice.json | 16 ++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/specification/search/data-plane/Azure.Search/preview/2020-06-30-Preview/searchservice.json b/specification/search/data-plane/Azure.Search/preview/2020-06-30-Preview/searchservice.json index 3a303e0e83a0..f05ad5fa5ee4 100644 --- a/specification/search/data-plane/Azure.Search/preview/2020-06-30-Preview/searchservice.json +++ b/specification/search/data-plane/Azure.Search/preview/2020-06-30-Preview/searchservice.json @@ -5793,68 +5793,84 @@ "type": "object", "properties": { "parsingMode": { + "x-nullable": true, "$ref": "#/definitions/ParsingMode" }, "excludedFileNameExtensions": { "type": "string", + "x-nullable": true, "description": "Comma-delimited list of filename extensions to ignore when processing from Azure blob storage. For example, you could exclude \".png, .mp4\" to skip over those files during indexing." }, "indexedFileNameExtensions": { "type": "string", + "x-nullable": true, "description": "Comma-delimited list of filename extensions to select when processing from Azure blob storage. For example, you could focus indexing on specific application files \".docx, .pptx, .msg\" to specifically include those file types." }, "failOnUnsupportedContentType": { "type": "boolean", "default": false, + "x-nullable": true, "description": "For Azure blobs, set to false if you want to continue indexing when an unsupported content type is encountered, and you don't know all the content types (file extensions) in advance." }, "failOnUnprocessableDocument": { "type": "boolean", "default": false, + "x-nullable": true, "description": "For Azure blobs, set to false if you want to continue indexing if a document fails indexing." }, "indexStorageMetadataOnlyForOversizedDocuments": { "type": "boolean", "default": false, + "x-nullable": true, "description": "For Azure blobs, set this property to true to still index storage metadata for blob content that is too large to process. Oversized blobs are treated as errors by default. For limits on blob size, see https://docs.microsoft.com/azure/search/search-limits-quotas-capacity." }, "delimitedTextHeaders": { "type": "string", + "x-nullable": true, "description": "For CSV blobs, specifies a comma-delimited list of column headers, useful for mapping source fields to destination fields in an index." }, "delimitedTextDelimiter": { "type": "string", + "x-nullable": true, "description": "For CSV blobs, specifies the end-of-line single-character delimiter for CSV files where each line starts a new document (for example, \"|\")." }, "firstLineContainsHeaders": { "type": "boolean", "default": true, + "x-nullable": true, "description": "For CSV blobs, indicates that the first (non-blank) line of each blob contains headers." }, "documentRoot": { "type": "string", + "x-nullable": true, "description": "For JSON arrays, given a structured or semi-structured document, you can specify a path to the array using this property." }, "dataToExtract": { + "x-nullable": true, "$ref": "#/definitions/DataToExtract" }, "imageAction": { + "x-nullable": true, "$ref": "#/definitions/ImageAction" }, "allowSkillsetToReadFileData": { "type": "boolean", "default": false, + "x-nullable": true, "description": "If true, will create a path //document//file_data that is an object representing the original file data downloaded from your blob data source. This allows you to pass the original file data to a custom skill for processing within the enrichment pipeline, or to the Document Extraction skill." }, "pdfTextRotationAlgorithm": { + "x-nullable": true, "$ref": "#/definitions/PdfTextRotationAlgorithm" }, "executionEnvironment": { + "x-nullable": true, "$ref": "#/definitions/ExecutionEnvironment" }, "queryTimeout": { "type": "string", "default": "00:05:00", + "x-nullable": true, "description": "Increases the timeout beyond the 5-minute default for Azure SQL database data sources, specified in the format \"hh:mm:ss\"." } }, diff --git a/specification/search/data-plane/Azure.Search/preview/2020-06-30/searchservice.json b/specification/search/data-plane/Azure.Search/preview/2020-06-30/searchservice.json index 6e3dd5948bc7..09d5cd120f35 100644 --- a/specification/search/data-plane/Azure.Search/preview/2020-06-30/searchservice.json +++ b/specification/search/data-plane/Azure.Search/preview/2020-06-30/searchservice.json @@ -5793,68 +5793,84 @@ "type": "object", "properties": { "parsingMode": { + "x-nullable": true, "$ref": "#/definitions/ParsingMode" }, "excludedFileNameExtensions": { "type": "string", + "x-nullable": true, "description": "Comma-delimited list of filename extensions to ignore when processing from Azure blob storage. For example, you could exclude \".png, .mp4\" to skip over those files during indexing." }, "indexedFileNameExtensions": { "type": "string", + "x-nullable": true, "description": "Comma-delimited list of filename extensions to select when processing from Azure blob storage. For example, you could focus indexing on specific application files \".docx, .pptx, .msg\" to specifically include those file types." }, "failOnUnsupportedContentType": { "type": "boolean", "default": false, + "x-nullable": true, "description": "For Azure blobs, set to false if you want to continue indexing when an unsupported content type is encountered, and you don't know all the content types (file extensions) in advance." }, "failOnUnprocessableDocument": { "type": "boolean", "default": false, + "x-nullable": true, "description": "For Azure blobs, set to false if you want to continue indexing if a document fails indexing." }, "indexStorageMetadataOnlyForOversizedDocuments": { "type": "boolean", "default": false, + "x-nullable": true, "description": "For Azure blobs, set this property to true to still index storage metadata for blob content that is too large to process. Oversized blobs are treated as errors by default. For limits on blob size, see https://docs.microsoft.com/azure/search/search-limits-quotas-capacity." }, "delimitedTextHeaders": { "type": "string", + "x-nullable": true, "description": "For CSV blobs, specifies a comma-delimited list of column headers, useful for mapping source fields to destination fields in an index." }, "delimitedTextDelimiter": { "type": "string", + "x-nullable": true, "description": "For CSV blobs, specifies the end-of-line single-character delimiter for CSV files where each line starts a new document (for example, \"|\")." }, "firstLineContainsHeaders": { "type": "boolean", "default": true, + "x-nullable": true, "description": "For CSV blobs, indicates that the first (non-blank) line of each blob contains headers." }, "documentRoot": { "type": "string", + "x-nullable": true, "description": "For JSON arrays, given a structured or semi-structured document, you can specify a path to the array using this property." }, "dataToExtract": { + "x-nullable": true, "$ref": "#/definitions/DataToExtract" }, "imageAction": { + "x-nullable": true, "$ref": "#/definitions/ImageAction" }, "allowSkillsetToReadFileData": { "type": "boolean", "default": false, + "x-nullable": true, "description": "If true, will create a path //document//file_data that is an object representing the original file data downloaded from your blob data source. This allows you to pass the original file data to a custom skill for processing within the enrichment pipeline, or to the Document Extraction skill." }, "pdfTextRotationAlgorithm": { + "x-nullable": true, "$ref": "#/definitions/PdfTextRotationAlgorithm" }, "executionEnvironment": { + "x-nullable": true, "$ref": "#/definitions/ExecutionEnvironment" }, "queryTimeout": { "type": "string", "default": "00:05:00", + "x-nullable": true, "description": "Increases the timeout beyond the 5-minute default for Azure SQL database data sources, specified in the format \"hh:mm:ss\"." } }, From 3ac21995a83c617d22a3806207e8a2640568ff4d Mon Sep 17 00:00:00 2001 From: Jennifer Marsman Date: Wed, 5 Aug 2020 11:04:50 -0400 Subject: [PATCH 34/38] Removed duplicate x-nullable since someone else also added them --- .../Azure.Search/preview/2020-06-30-Preview/searchservice.json | 2 -- .../Azure.Search/preview/2020-06-30/searchservice.json | 2 -- 2 files changed, 4 deletions(-) diff --git a/specification/search/data-plane/Azure.Search/preview/2020-06-30-Preview/searchservice.json b/specification/search/data-plane/Azure.Search/preview/2020-06-30-Preview/searchservice.json index f05ad5fa5ee4..d4d8eb46a1fd 100644 --- a/specification/search/data-plane/Azure.Search/preview/2020-06-30-Preview/searchservice.json +++ b/specification/search/data-plane/Azure.Search/preview/2020-06-30-Preview/searchservice.json @@ -5776,7 +5776,6 @@ "maxFailedItems": { "type": "integer", "format": "int32", - "x-nullable": true, "default": 0, "x-nullable": true, "description": "The maximum number of items that can fail indexing for indexer execution to still be considered successful. -1 means no limit. Default is 0." @@ -5784,7 +5783,6 @@ "maxFailedItemsPerBatch": { "type": "integer", "format": "int32", - "x-nullable": true, "default": 0, "x-nullable": true, "description": "The maximum number of items in a single batch that can fail indexing for the batch to still be considered successful. -1 means no limit. Default is 0." diff --git a/specification/search/data-plane/Azure.Search/preview/2020-06-30/searchservice.json b/specification/search/data-plane/Azure.Search/preview/2020-06-30/searchservice.json index 09d5cd120f35..7e2beb8b2237 100644 --- a/specification/search/data-plane/Azure.Search/preview/2020-06-30/searchservice.json +++ b/specification/search/data-plane/Azure.Search/preview/2020-06-30/searchservice.json @@ -5776,7 +5776,6 @@ "maxFailedItems": { "type": "integer", "format": "int32", - "x-nullable": true, "default": 0, "x-nullable": true, "description": "The maximum number of items that can fail indexing for indexer execution to still be considered successful. -1 means no limit. Default is 0." @@ -5784,7 +5783,6 @@ "maxFailedItemsPerBatch": { "type": "integer", "format": "int32", - "x-nullable": true, "default": 0, "x-nullable": true, "description": "The maximum number of items in a single batch that can fail indexing for the batch to still be considered successful. -1 means no limit. Default is 0." From 4b3c6332b8081f01f99392f5ad1fc1f3e26e15c9 Mon Sep 17 00:00:00 2001 From: Jennifer Marsman Date: Thu, 6 Aug 2020 19:16:09 -0400 Subject: [PATCH 35/38] Revert "Added x-nullable to configuration parameters" This reverts commit 732f032fdc77c2314ab29a270818d2af1e1ec42f. --- .../2020-06-30-Preview/searchservice.json | 16 ---------------- .../preview/2020-06-30/searchservice.json | 16 ---------------- 2 files changed, 32 deletions(-) diff --git a/specification/search/data-plane/Azure.Search/preview/2020-06-30-Preview/searchservice.json b/specification/search/data-plane/Azure.Search/preview/2020-06-30-Preview/searchservice.json index d4d8eb46a1fd..3db2bee6813e 100644 --- a/specification/search/data-plane/Azure.Search/preview/2020-06-30-Preview/searchservice.json +++ b/specification/search/data-plane/Azure.Search/preview/2020-06-30-Preview/searchservice.json @@ -5791,84 +5791,68 @@ "type": "object", "properties": { "parsingMode": { - "x-nullable": true, "$ref": "#/definitions/ParsingMode" }, "excludedFileNameExtensions": { "type": "string", - "x-nullable": true, "description": "Comma-delimited list of filename extensions to ignore when processing from Azure blob storage. For example, you could exclude \".png, .mp4\" to skip over those files during indexing." }, "indexedFileNameExtensions": { "type": "string", - "x-nullable": true, "description": "Comma-delimited list of filename extensions to select when processing from Azure blob storage. For example, you could focus indexing on specific application files \".docx, .pptx, .msg\" to specifically include those file types." }, "failOnUnsupportedContentType": { "type": "boolean", "default": false, - "x-nullable": true, "description": "For Azure blobs, set to false if you want to continue indexing when an unsupported content type is encountered, and you don't know all the content types (file extensions) in advance." }, "failOnUnprocessableDocument": { "type": "boolean", "default": false, - "x-nullable": true, "description": "For Azure blobs, set to false if you want to continue indexing if a document fails indexing." }, "indexStorageMetadataOnlyForOversizedDocuments": { "type": "boolean", "default": false, - "x-nullable": true, "description": "For Azure blobs, set this property to true to still index storage metadata for blob content that is too large to process. Oversized blobs are treated as errors by default. For limits on blob size, see https://docs.microsoft.com/azure/search/search-limits-quotas-capacity." }, "delimitedTextHeaders": { "type": "string", - "x-nullable": true, "description": "For CSV blobs, specifies a comma-delimited list of column headers, useful for mapping source fields to destination fields in an index." }, "delimitedTextDelimiter": { "type": "string", - "x-nullable": true, "description": "For CSV blobs, specifies the end-of-line single-character delimiter for CSV files where each line starts a new document (for example, \"|\")." }, "firstLineContainsHeaders": { "type": "boolean", "default": true, - "x-nullable": true, "description": "For CSV blobs, indicates that the first (non-blank) line of each blob contains headers." }, "documentRoot": { "type": "string", - "x-nullable": true, "description": "For JSON arrays, given a structured or semi-structured document, you can specify a path to the array using this property." }, "dataToExtract": { - "x-nullable": true, "$ref": "#/definitions/DataToExtract" }, "imageAction": { - "x-nullable": true, "$ref": "#/definitions/ImageAction" }, "allowSkillsetToReadFileData": { "type": "boolean", "default": false, - "x-nullable": true, "description": "If true, will create a path //document//file_data that is an object representing the original file data downloaded from your blob data source. This allows you to pass the original file data to a custom skill for processing within the enrichment pipeline, or to the Document Extraction skill." }, "pdfTextRotationAlgorithm": { - "x-nullable": true, "$ref": "#/definitions/PdfTextRotationAlgorithm" }, "executionEnvironment": { - "x-nullable": true, "$ref": "#/definitions/ExecutionEnvironment" }, "queryTimeout": { "type": "string", "default": "00:05:00", - "x-nullable": true, "description": "Increases the timeout beyond the 5-minute default for Azure SQL database data sources, specified in the format \"hh:mm:ss\"." } }, diff --git a/specification/search/data-plane/Azure.Search/preview/2020-06-30/searchservice.json b/specification/search/data-plane/Azure.Search/preview/2020-06-30/searchservice.json index 7e2beb8b2237..0ee96bce0b00 100644 --- a/specification/search/data-plane/Azure.Search/preview/2020-06-30/searchservice.json +++ b/specification/search/data-plane/Azure.Search/preview/2020-06-30/searchservice.json @@ -5791,84 +5791,68 @@ "type": "object", "properties": { "parsingMode": { - "x-nullable": true, "$ref": "#/definitions/ParsingMode" }, "excludedFileNameExtensions": { "type": "string", - "x-nullable": true, "description": "Comma-delimited list of filename extensions to ignore when processing from Azure blob storage. For example, you could exclude \".png, .mp4\" to skip over those files during indexing." }, "indexedFileNameExtensions": { "type": "string", - "x-nullable": true, "description": "Comma-delimited list of filename extensions to select when processing from Azure blob storage. For example, you could focus indexing on specific application files \".docx, .pptx, .msg\" to specifically include those file types." }, "failOnUnsupportedContentType": { "type": "boolean", "default": false, - "x-nullable": true, "description": "For Azure blobs, set to false if you want to continue indexing when an unsupported content type is encountered, and you don't know all the content types (file extensions) in advance." }, "failOnUnprocessableDocument": { "type": "boolean", "default": false, - "x-nullable": true, "description": "For Azure blobs, set to false if you want to continue indexing if a document fails indexing." }, "indexStorageMetadataOnlyForOversizedDocuments": { "type": "boolean", "default": false, - "x-nullable": true, "description": "For Azure blobs, set this property to true to still index storage metadata for blob content that is too large to process. Oversized blobs are treated as errors by default. For limits on blob size, see https://docs.microsoft.com/azure/search/search-limits-quotas-capacity." }, "delimitedTextHeaders": { "type": "string", - "x-nullable": true, "description": "For CSV blobs, specifies a comma-delimited list of column headers, useful for mapping source fields to destination fields in an index." }, "delimitedTextDelimiter": { "type": "string", - "x-nullable": true, "description": "For CSV blobs, specifies the end-of-line single-character delimiter for CSV files where each line starts a new document (for example, \"|\")." }, "firstLineContainsHeaders": { "type": "boolean", "default": true, - "x-nullable": true, "description": "For CSV blobs, indicates that the first (non-blank) line of each blob contains headers." }, "documentRoot": { "type": "string", - "x-nullable": true, "description": "For JSON arrays, given a structured or semi-structured document, you can specify a path to the array using this property." }, "dataToExtract": { - "x-nullable": true, "$ref": "#/definitions/DataToExtract" }, "imageAction": { - "x-nullable": true, "$ref": "#/definitions/ImageAction" }, "allowSkillsetToReadFileData": { "type": "boolean", "default": false, - "x-nullable": true, "description": "If true, will create a path //document//file_data that is an object representing the original file data downloaded from your blob data source. This allows you to pass the original file data to a custom skill for processing within the enrichment pipeline, or to the Document Extraction skill." }, "pdfTextRotationAlgorithm": { - "x-nullable": true, "$ref": "#/definitions/PdfTextRotationAlgorithm" }, "executionEnvironment": { - "x-nullable": true, "$ref": "#/definitions/ExecutionEnvironment" }, "queryTimeout": { "type": "string", "default": "00:05:00", - "x-nullable": true, "description": "Increases the timeout beyond the 5-minute default for Azure SQL database data sources, specified in the format \"hh:mm:ss\"." } }, From 6a024fe462c3a4b0b720183c1d8c6f96b261a386 Mon Sep 17 00:00:00 2001 From: Jennifer Marsman Date: Fri, 7 Aug 2020 10:40:14 -0400 Subject: [PATCH 36/38] Added default values for excludedFileNameExtensions and indexedFileNameExtensions --- .../Azure.Search/preview/2020-06-30-Preview/searchservice.json | 2 ++ .../Azure.Search/preview/2020-06-30/searchservice.json | 2 ++ 2 files changed, 4 insertions(+) diff --git a/specification/search/data-plane/Azure.Search/preview/2020-06-30-Preview/searchservice.json b/specification/search/data-plane/Azure.Search/preview/2020-06-30-Preview/searchservice.json index 3db2bee6813e..a7eb2cf0b145 100644 --- a/specification/search/data-plane/Azure.Search/preview/2020-06-30-Preview/searchservice.json +++ b/specification/search/data-plane/Azure.Search/preview/2020-06-30-Preview/searchservice.json @@ -5795,10 +5795,12 @@ }, "excludedFileNameExtensions": { "type": "string", + "default": "", "description": "Comma-delimited list of filename extensions to ignore when processing from Azure blob storage. For example, you could exclude \".png, .mp4\" to skip over those files during indexing." }, "indexedFileNameExtensions": { "type": "string", + "default": "", "description": "Comma-delimited list of filename extensions to select when processing from Azure blob storage. For example, you could focus indexing on specific application files \".docx, .pptx, .msg\" to specifically include those file types." }, "failOnUnsupportedContentType": { diff --git a/specification/search/data-plane/Azure.Search/preview/2020-06-30/searchservice.json b/specification/search/data-plane/Azure.Search/preview/2020-06-30/searchservice.json index 0ee96bce0b00..0c3501d0adb7 100644 --- a/specification/search/data-plane/Azure.Search/preview/2020-06-30/searchservice.json +++ b/specification/search/data-plane/Azure.Search/preview/2020-06-30/searchservice.json @@ -5795,10 +5795,12 @@ }, "excludedFileNameExtensions": { "type": "string", + "default": "", "description": "Comma-delimited list of filename extensions to ignore when processing from Azure blob storage. For example, you could exclude \".png, .mp4\" to skip over those files during indexing." }, "indexedFileNameExtensions": { "type": "string", + "default": "", "description": "Comma-delimited list of filename extensions to select when processing from Azure blob storage. For example, you could focus indexing on specific application files \".docx, .pptx, .msg\" to specifically include those file types." }, "failOnUnsupportedContentType": { From e9ae5e749b606bf411dccfba3740fac7357f00dd Mon Sep 17 00:00:00 2001 From: Jennifer Marsman Date: Fri, 11 Sep 2020 12:01:36 -0400 Subject: [PATCH 37/38] Moved indexing parameters configuration to a separate model not under configuration --- .../2020-06-30-Preview/searchservice.json | 148 ++++++++++-------- .../preview/2020-06-30/searchservice.json | 148 ++++++++++-------- 2 files changed, 158 insertions(+), 138 deletions(-) diff --git a/specification/search/data-plane/Azure.Search/preview/2020-06-30-Preview/searchservice.json b/specification/search/data-plane/Azure.Search/preview/2020-06-30-Preview/searchservice.json index a7eb2cf0b145..a33562dbe81d 100644 --- a/specification/search/data-plane/Azure.Search/preview/2020-06-30-Preview/searchservice.json +++ b/specification/search/data-plane/Azure.Search/preview/2020-06-30-Preview/searchservice.json @@ -5789,75 +5789,6 @@ }, "configuration": { "type": "object", - "properties": { - "parsingMode": { - "$ref": "#/definitions/ParsingMode" - }, - "excludedFileNameExtensions": { - "type": "string", - "default": "", - "description": "Comma-delimited list of filename extensions to ignore when processing from Azure blob storage. For example, you could exclude \".png, .mp4\" to skip over those files during indexing." - }, - "indexedFileNameExtensions": { - "type": "string", - "default": "", - "description": "Comma-delimited list of filename extensions to select when processing from Azure blob storage. For example, you could focus indexing on specific application files \".docx, .pptx, .msg\" to specifically include those file types." - }, - "failOnUnsupportedContentType": { - "type": "boolean", - "default": false, - "description": "For Azure blobs, set to false if you want to continue indexing when an unsupported content type is encountered, and you don't know all the content types (file extensions) in advance." - }, - "failOnUnprocessableDocument": { - "type": "boolean", - "default": false, - "description": "For Azure blobs, set to false if you want to continue indexing if a document fails indexing." - }, - "indexStorageMetadataOnlyForOversizedDocuments": { - "type": "boolean", - "default": false, - "description": "For Azure blobs, set this property to true to still index storage metadata for blob content that is too large to process. Oversized blobs are treated as errors by default. For limits on blob size, see https://docs.microsoft.com/azure/search/search-limits-quotas-capacity." - }, - "delimitedTextHeaders": { - "type": "string", - "description": "For CSV blobs, specifies a comma-delimited list of column headers, useful for mapping source fields to destination fields in an index." - }, - "delimitedTextDelimiter": { - "type": "string", - "description": "For CSV blobs, specifies the end-of-line single-character delimiter for CSV files where each line starts a new document (for example, \"|\")." - }, - "firstLineContainsHeaders": { - "type": "boolean", - "default": true, - "description": "For CSV blobs, indicates that the first (non-blank) line of each blob contains headers." - }, - "documentRoot": { - "type": "string", - "description": "For JSON arrays, given a structured or semi-structured document, you can specify a path to the array using this property." - }, - "dataToExtract": { - "$ref": "#/definitions/DataToExtract" - }, - "imageAction": { - "$ref": "#/definitions/ImageAction" - }, - "allowSkillsetToReadFileData": { - "type": "boolean", - "default": false, - "description": "If true, will create a path //document//file_data that is an object representing the original file data downloaded from your blob data source. This allows you to pass the original file data to a custom skill for processing within the enrichment pipeline, or to the Document Extraction skill." - }, - "pdfTextRotationAlgorithm": { - "$ref": "#/definitions/PdfTextRotationAlgorithm" - }, - "executionEnvironment": { - "$ref": "#/definitions/ExecutionEnvironment" - }, - "queryTimeout": { - "type": "string", - "default": "00:05:00", - "description": "Increases the timeout beyond the 5-minute default for Azure SQL database data sources, specified in the format \"hh:mm:ss\"." - } - }, "additionalProperties": { "type": "object" }, @@ -5869,6 +5800,85 @@ "url": "https://docs.microsoft.com/rest/api/searchservice/create-indexer#parameters" } }, + "IndexingParametersConfiguration": { + "type": "object", + "properties": { + "parsingMode": { + "$ref": "#/definitions/ParsingMode" + }, + "excludedFileNameExtensions": { + "type": "string", + "default": "", + "description": "Comma-delimited list of filename extensions to ignore when processing from Azure blob storage. For example, you could exclude \".png, .mp4\" to skip over those files during indexing." + }, + "indexedFileNameExtensions": { + "type": "string", + "default": "", + "description": "Comma-delimited list of filename extensions to select when processing from Azure blob storage. For example, you could focus indexing on specific application files \".docx, .pptx, .msg\" to specifically include those file types." + }, + "failOnUnsupportedContentType": { + "type": "boolean", + "default": false, + "description": "For Azure blobs, set to false if you want to continue indexing when an unsupported content type is encountered, and you don't know all the content types (file extensions) in advance." + }, + "failOnUnprocessableDocument": { + "type": "boolean", + "default": false, + "description": "For Azure blobs, set to false if you want to continue indexing if a document fails indexing." + }, + "indexStorageMetadataOnlyForOversizedDocuments": { + "type": "boolean", + "default": false, + "description": "For Azure blobs, set this property to true to still index storage metadata for blob content that is too large to process. Oversized blobs are treated as errors by default. For limits on blob size, see https://docs.microsoft.com/azure/search/search-limits-quotas-capacity." + }, + "delimitedTextHeaders": { + "type": "string", + "description": "For CSV blobs, specifies a comma-delimited list of column headers, useful for mapping source fields to destination fields in an index." + }, + "delimitedTextDelimiter": { + "type": "string", + "description": "For CSV blobs, specifies the end-of-line single-character delimiter for CSV files where each line starts a new document (for example, \"|\")." + }, + "firstLineContainsHeaders": { + "type": "boolean", + "default": true, + "description": "For CSV blobs, indicates that the first (non-blank) line of each blob contains headers." + }, + "documentRoot": { + "type": "string", + "description": "For JSON arrays, given a structured or semi-structured document, you can specify a path to the array using this property." + }, + "dataToExtract": { + "$ref": "#/definitions/DataToExtract" + }, + "imageAction": { + "$ref": "#/definitions/ImageAction" + }, + "allowSkillsetToReadFileData": { + "type": "boolean", + "default": false, + "description": "If true, will create a path //document//file_data that is an object representing the original file data downloaded from your blob data source. This allows you to pass the original file data to a custom skill for processing within the enrichment pipeline, or to the Document Extraction skill." + }, + "pdfTextRotationAlgorithm": { + "$ref": "#/definitions/PdfTextRotationAlgorithm" + }, + "executionEnvironment": { + "$ref": "#/definitions/ExecutionEnvironment" + }, + "queryTimeout": { + "type": "string", + "default": "00:05:00", + "description": "Increases the timeout beyond the 5-minute default for Azure SQL database data sources, specified in the format \"hh:mm:ss\"." + } + }, + "additionalProperties": { + "type": "object" + }, + "description": "A dictionary of indexer-specific configuration properties. Each name is the name of a specific property. Each value must be of a primitive type.", + "externalDocs": { + "url": "https://docs.microsoft.com/rest/api/searchservice/create-indexer#parameters" + } + }, "ParsingMode": { "type": "string", "enum": [ diff --git a/specification/search/data-plane/Azure.Search/preview/2020-06-30/searchservice.json b/specification/search/data-plane/Azure.Search/preview/2020-06-30/searchservice.json index 0c3501d0adb7..1780d1305eda 100644 --- a/specification/search/data-plane/Azure.Search/preview/2020-06-30/searchservice.json +++ b/specification/search/data-plane/Azure.Search/preview/2020-06-30/searchservice.json @@ -5789,75 +5789,6 @@ }, "configuration": { "type": "object", - "properties": { - "parsingMode": { - "$ref": "#/definitions/ParsingMode" - }, - "excludedFileNameExtensions": { - "type": "string", - "default": "", - "description": "Comma-delimited list of filename extensions to ignore when processing from Azure blob storage. For example, you could exclude \".png, .mp4\" to skip over those files during indexing." - }, - "indexedFileNameExtensions": { - "type": "string", - "default": "", - "description": "Comma-delimited list of filename extensions to select when processing from Azure blob storage. For example, you could focus indexing on specific application files \".docx, .pptx, .msg\" to specifically include those file types." - }, - "failOnUnsupportedContentType": { - "type": "boolean", - "default": false, - "description": "For Azure blobs, set to false if you want to continue indexing when an unsupported content type is encountered, and you don't know all the content types (file extensions) in advance." - }, - "failOnUnprocessableDocument": { - "type": "boolean", - "default": false, - "description": "For Azure blobs, set to false if you want to continue indexing if a document fails indexing." - }, - "indexStorageMetadataOnlyForOversizedDocuments": { - "type": "boolean", - "default": false, - "description": "For Azure blobs, set this property to true to still index storage metadata for blob content that is too large to process. Oversized blobs are treated as errors by default. For limits on blob size, see https://docs.microsoft.com/azure/search/search-limits-quotas-capacity." - }, - "delimitedTextHeaders": { - "type": "string", - "description": "For CSV blobs, specifies a comma-delimited list of column headers, useful for mapping source fields to destination fields in an index." - }, - "delimitedTextDelimiter": { - "type": "string", - "description": "For CSV blobs, specifies the end-of-line single-character delimiter for CSV files where each line starts a new document (for example, \"|\")." - }, - "firstLineContainsHeaders": { - "type": "boolean", - "default": true, - "description": "For CSV blobs, indicates that the first (non-blank) line of each blob contains headers." - }, - "documentRoot": { - "type": "string", - "description": "For JSON arrays, given a structured or semi-structured document, you can specify a path to the array using this property." - }, - "dataToExtract": { - "$ref": "#/definitions/DataToExtract" - }, - "imageAction": { - "$ref": "#/definitions/ImageAction" - }, - "allowSkillsetToReadFileData": { - "type": "boolean", - "default": false, - "description": "If true, will create a path //document//file_data that is an object representing the original file data downloaded from your blob data source. This allows you to pass the original file data to a custom skill for processing within the enrichment pipeline, or to the Document Extraction skill." - }, - "pdfTextRotationAlgorithm": { - "$ref": "#/definitions/PdfTextRotationAlgorithm" - }, - "executionEnvironment": { - "$ref": "#/definitions/ExecutionEnvironment" - }, - "queryTimeout": { - "type": "string", - "default": "00:05:00", - "description": "Increases the timeout beyond the 5-minute default for Azure SQL database data sources, specified in the format \"hh:mm:ss\"." - } - }, "additionalProperties": { "type": "object" }, @@ -5869,6 +5800,85 @@ "url": "https://docs.microsoft.com/rest/api/searchservice/create-indexer#parameters" } }, + "IndexingParametersConfiguration": { + "type": "object", + "properties": { + "parsingMode": { + "$ref": "#/definitions/ParsingMode" + }, + "excludedFileNameExtensions": { + "type": "string", + "default": "", + "description": "Comma-delimited list of filename extensions to ignore when processing from Azure blob storage. For example, you could exclude \".png, .mp4\" to skip over those files during indexing." + }, + "indexedFileNameExtensions": { + "type": "string", + "default": "", + "description": "Comma-delimited list of filename extensions to select when processing from Azure blob storage. For example, you could focus indexing on specific application files \".docx, .pptx, .msg\" to specifically include those file types." + }, + "failOnUnsupportedContentType": { + "type": "boolean", + "default": false, + "description": "For Azure blobs, set to false if you want to continue indexing when an unsupported content type is encountered, and you don't know all the content types (file extensions) in advance." + }, + "failOnUnprocessableDocument": { + "type": "boolean", + "default": false, + "description": "For Azure blobs, set to false if you want to continue indexing if a document fails indexing." + }, + "indexStorageMetadataOnlyForOversizedDocuments": { + "type": "boolean", + "default": false, + "description": "For Azure blobs, set this property to true to still index storage metadata for blob content that is too large to process. Oversized blobs are treated as errors by default. For limits on blob size, see https://docs.microsoft.com/azure/search/search-limits-quotas-capacity." + }, + "delimitedTextHeaders": { + "type": "string", + "description": "For CSV blobs, specifies a comma-delimited list of column headers, useful for mapping source fields to destination fields in an index." + }, + "delimitedTextDelimiter": { + "type": "string", + "description": "For CSV blobs, specifies the end-of-line single-character delimiter for CSV files where each line starts a new document (for example, \"|\")." + }, + "firstLineContainsHeaders": { + "type": "boolean", + "default": true, + "description": "For CSV blobs, indicates that the first (non-blank) line of each blob contains headers." + }, + "documentRoot": { + "type": "string", + "description": "For JSON arrays, given a structured or semi-structured document, you can specify a path to the array using this property." + }, + "dataToExtract": { + "$ref": "#/definitions/DataToExtract" + }, + "imageAction": { + "$ref": "#/definitions/ImageAction" + }, + "allowSkillsetToReadFileData": { + "type": "boolean", + "default": false, + "description": "If true, will create a path //document//file_data that is an object representing the original file data downloaded from your blob data source. This allows you to pass the original file data to a custom skill for processing within the enrichment pipeline, or to the Document Extraction skill." + }, + "pdfTextRotationAlgorithm": { + "$ref": "#/definitions/PdfTextRotationAlgorithm" + }, + "executionEnvironment": { + "$ref": "#/definitions/ExecutionEnvironment" + }, + "queryTimeout": { + "type": "string", + "default": "00:05:00", + "description": "Increases the timeout beyond the 5-minute default for Azure SQL database data sources, specified in the format \"hh:mm:ss\"." + } + }, + "additionalProperties": { + "type": "object" + }, + "description": "A dictionary of indexer-specific configuration properties. Each name is the name of a specific property. Each value must be of a primitive type.", + "externalDocs": { + "url": "https://docs.microsoft.com/rest/api/searchservice/create-indexer#parameters" + } + }, "ParsingMode": { "type": "string", "enum": [ From 8c52a2039638429b1ff8d53d13a39ba0d1bddd94 Mon Sep 17 00:00:00 2001 From: Jennifer Marsman Date: Fri, 11 Sep 2020 14:16:20 -0400 Subject: [PATCH 38/38] Referencing the IndexingParametersConfiguration from configuration --- .../preview/2020-06-30-Preview/searchservice.json | 6 +----- .../Azure.Search/preview/2020-06-30/searchservice.json | 6 +----- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/specification/search/data-plane/Azure.Search/preview/2020-06-30-Preview/searchservice.json b/specification/search/data-plane/Azure.Search/preview/2020-06-30-Preview/searchservice.json index a33562dbe81d..1bab90d7a0d8 100644 --- a/specification/search/data-plane/Azure.Search/preview/2020-06-30-Preview/searchservice.json +++ b/specification/search/data-plane/Azure.Search/preview/2020-06-30-Preview/searchservice.json @@ -5788,11 +5788,7 @@ "description": "The maximum number of items in a single batch that can fail indexing for the batch to still be considered successful. -1 means no limit. Default is 0." }, "configuration": { - "type": "object", - "additionalProperties": { - "type": "object" - }, - "description": "A dictionary of indexer-specific configuration properties. Each name is the name of a specific property. Each value must be of a primitive type." + "$ref": "#/definitions/IndexingParametersConfiguration" } }, "description": "Represents parameters for indexer execution.", diff --git a/specification/search/data-plane/Azure.Search/preview/2020-06-30/searchservice.json b/specification/search/data-plane/Azure.Search/preview/2020-06-30/searchservice.json index 1780d1305eda..828e7300c027 100644 --- a/specification/search/data-plane/Azure.Search/preview/2020-06-30/searchservice.json +++ b/specification/search/data-plane/Azure.Search/preview/2020-06-30/searchservice.json @@ -5788,11 +5788,7 @@ "description": "The maximum number of items in a single batch that can fail indexing for the batch to still be considered successful. -1 means no limit. Default is 0." }, "configuration": { - "type": "object", - "additionalProperties": { - "type": "object" - }, - "description": "A dictionary of indexer-specific configuration properties. Each name is the name of a specific property. Each value must be of a primitive type." + "$ref": "#/definitions/IndexingParametersConfiguration" } }, "description": "Represents parameters for indexer execution.",