Fix bug for supporting multi-page documents in Form Recognizer

sdk/formrecognizer/azure-ai-formrecognizer/CHANGELOG.md

@@ -1,8 +1,7 @@
 # Release History
 
 ## 1.0.0-beta.3 (Unreleased)
-
-FormTraining Client updates:
+- Fix bug in FormRecognizer API's to support multipage document recognition.
 - Adopt the `training` namespace for Form Recognizer Training Clients
 - Rename parameter `fileSourceUrl` to `trainingFilesUrl` on `beginTraining` method in FormTrainingClients
 - Rename parameter `useLabelFile` to `useTrainingLabels` on `beginTraining` method in FormTrainingClients

sdk/formrecognizer/azure-ai-formrecognizer/README.md

@@ -23,13 +23,13 @@ from form documents. It includes the following main functionalities:
 <dependency>
     <groupId>com.azure</groupId>
     <artifactId>azure-ai-formrecognizer</artifactId>
-    <version>1.0.0-beta.1</version>
+    <version>1.0.0-beta.3</version>
 </dependency>
 ```
 [//]: # ({x-version-update-end})
 
 ### Create a Form Recognizer resource
-Form Recognizer supports both [multi-service and single-service access][service_access]. Create a Cognitive Services 
+Form Recognizer supports both [multi-service and single-service access][service_access]. Create a Cognitive Service's 
 resource if you plan to access multiple cognitive services under a single endpoint/key. For Form Recognizer access only,
 create a Form Recognizer resource.
 
@@ -50,10 +50,10 @@ az group create --name my-resource-group --location westus2
 ```bash
 # Create Form Recognizer
 az cognitiveservices account create \
-    --name text-analytics-resource \
+    --name form-recognizer-resource \
     --resource-group my-resource-group \
-    --kind TextAnalytics \
-    --sku F0 \
+    --kind FormRecognizer \
+    --sku S0 \
     --location westus2 \
     --yes
 ```

sdk/formrecognizer/azure-ai-formrecognizer/src/main/java/com/azure/ai/formrecognizer/Transforms.java

@@ -89,7 +89,7 @@ static List<RecognizedForm> toRecognizedForm(AnalyzeResult analyzeResult, boolea
             }
         } else {
             extractedFormList = new ArrayList<>();
-            for (PageResult pageResultItem : pageResults) {
+            forEachWithIndex(pageResults, ((index, pageResultItem) -> {
                 StringBuffer formType = new StringBuffer("form-");
                 int pageNumber = pageResultItem.getPage();
                 Integer clusterId = pageResultItem.getClusterId();
@@ -103,8 +103,8 @@ static List<RecognizedForm> toRecognizedForm(AnalyzeResult analyzeResult, boolea
                     extractedFieldMap,
                     formType.toString(),
                     new PageRange(pageNumber, pageNumber),
-                    new IterableStream<>(Collections.singletonList(formPages.get(pageNumber - 1)))));
-            }
+                    new IterableStream<>(Collections.singletonList(formPages.get(index)))));
+            }));
         }
         return extractedFormList;
     }
@@ -216,24 +216,27 @@ private static Map<String, FormField<?>> getUnlabeledFieldMap(DocumentResult doc
         List<ReadResult> readResults, boolean includeTextDetails) {
         Map<String, FormField<?>> extractedFieldMap = new TreeMap<>();
         // add receipt fields
-        documentResultItem.getFields().forEach((key, fieldValue) -> {
-            if (fieldValue != null) {
-                Integer pageNumber = fieldValue.getPage();
-                FieldText labelText = new FieldText(key, null, pageNumber, null);
-                IterableStream<FormContent> formContentList = null;
-                if (includeTextDetails) {
-                    formContentList = setReferenceElements(fieldValue.getElements(), readResults, pageNumber);
+        if (!CoreUtils.isNullOrEmpty(documentResultItem.getFields())) {
+            documentResultItem.getFields().forEach((key, fieldValue) -> {
+                if (fieldValue != null) {
+                    Integer pageNumber = fieldValue.getPage();
+                    FieldText labelText = new FieldText(key, null, pageNumber, null);
+                    IterableStream<FormContent> formContentList = null;
+                    if (includeTextDetails) {
+                        formContentList = setReferenceElements(fieldValue.getElements(), readResults, pageNumber);
+                    }
+                    FieldText valueText = new FieldText(fieldValue.getText(),
+                        toBoundingBox(fieldValue.getBoundingBox()),
+                        pageNumber, formContentList);
+                    extractedFieldMap.put(key, setFormField(labelText, key, fieldValue, valueText, pageNumber,
+                        readResults));
+                } else {
+                    FieldText labelText = new FieldText(key, null, null, null);
+                    extractedFieldMap.put(key, new FormField<>(DEFAULT_CONFIDENCE_VALUE, labelText,
+                        key, null, null, null));
                 }
-                FieldText valueText = new FieldText(fieldValue.getText(), toBoundingBox(fieldValue.getBoundingBox()),
-                    pageNumber, formContentList);
-                extractedFieldMap.put(key, setFormField(labelText, key, fieldValue, valueText, pageNumber,
-                    readResults));
-            } else {
-                FieldText labelText = new FieldText(key, null, null, null);
-                extractedFieldMap.put(key, new FormField<>(DEFAULT_CONFIDENCE_VALUE, labelText,
-                    key, null, null, null));
-            }
-        });
+            });
+        }
         return extractedFieldMap;
     }
 
@@ -309,8 +312,7 @@ private static float setDefaultConfidenceValue(Float confidence) {
      * {@link com.azure.ai.formrecognizer.implementation.models.FieldValue#getValueObject()}
      * to a SDK level map of {@link FormField}.
      *
-     * @param valueObject The array of field values returned by the service in
-     * {@link FieldValue#getValueObject()} .
+     * @param valueObject The array of field values returned by the service in {@link FieldValue#getValueObject()}.
      *
      * @return The Map of {@link FormField}.
      */
@@ -332,8 +334,7 @@ private static Map<String, FormField<?>> toFormFieldObject(Map<String, FieldValu
      * {@link com.azure.ai.formrecognizer.implementation.models.FieldValue#getValueArray()}
      * to a SDK level List of {@link FormField}.
      *
-     * @param valueArray The array of field values returned by the service in
-     * {@link FieldValue#getValueArray()}.
+     * @param valueArray The array of field values returned by the service in {@link FieldValue#getValueArray()}.
      * @param readResults The text extraction result returned by the service.
      *
      * @return The List of {@link FormField}.

sdk/formrecognizer/azure-ai-formrecognizer/src/test/java/com/azure/ai/formrecognizer/FormRecognizerAsyncClientTest.java

@@ -33,6 +33,7 @@
 import static com.azure.ai.formrecognizer.TestUtils.INVALID_URL;
 import static com.azure.ai.formrecognizer.TestUtils.LAYOUT_FILE_LENGTH;
 import static com.azure.ai.formrecognizer.TestUtils.LAYOUT_LOCAL_URL;
+import static com.azure.ai.formrecognizer.TestUtils.MULTIPAGE_INVOICE_FILE_LENGTH;
 import static com.azure.ai.formrecognizer.TestUtils.RECEIPT_FILE_LENGTH;
 import static com.azure.ai.formrecognizer.TestUtils.RECEIPT_LOCAL_URL;
 import static com.azure.ai.formrecognizer.TestUtils.getReplayableBufferData;
@@ -128,8 +129,7 @@ public void recognizeReceiptDataTextDetailsWithNullData(HttpClient httpClient,
     }
 
     /**
-     * Verifies receipt data from a document using file data as source.
-     * And the content type is not given. The content type will be auto detected.
+     * Verifies content type will be auto detected when using custom form API with input stream data overload.
      */
     @ParameterizedTest(name = DISPLAY_NAME_WITH_ARGUMENTS)
     @MethodSource("com.azure.ai.formrecognizer.TestUtils#getTestParameters")
@@ -193,9 +193,9 @@ public void recognizeReceiptAsUSReceipt(HttpClient httpClient, FormRecognizerSer
      */
     @ParameterizedTest(name = DISPLAY_NAME_WITH_ARGUMENTS)
     @MethodSource("com.azure.ai.formrecognizer.TestUtils#getTestParameters")
-    public void recognizeLayoutData(HttpClient httpClient, FormRecognizerServiceVersion serviceVersion) {
+    public void recognizeContent(HttpClient httpClient, FormRecognizerServiceVersion serviceVersion) {
         client = getFormRecognizerAsyncClient(httpClient, serviceVersion);
-        layoutDataRunner((data) -> {
+        contentFromDataRunner((data) -> {
             SyncPoller<OperationResult, IterableStream<FormPage>> syncPoller =
                 client.beginRecognizeContent(toFluxByteBuffer(data), LAYOUT_FILE_LENGTH, FormContentType.IMAGE_JPEG,
                     null).getSyncPoller();
@@ -209,29 +209,21 @@ public void recognizeLayoutData(HttpClient httpClient, FormRecognizerServiceVers
      */
     @ParameterizedTest(name = DISPLAY_NAME_WITH_ARGUMENTS)
     @MethodSource("com.azure.ai.formrecognizer.TestUtils#getTestParameters")
-    public void recognizeLayoutDataWithNullData(HttpClient httpClient, FormRecognizerServiceVersion serviceVersion) {
-        client = getFormRecognizerAsyncClient(httpClient, serviceVersion);
-        layoutDataRunner((data) -> {
-            SyncPoller<OperationResult, IterableStream<FormPage>> syncPoller =
-                client.beginRecognizeContent(toFluxByteBuffer(data), LAYOUT_FILE_LENGTH, FormContentType.IMAGE_JPEG,
-                    null).getSyncPoller();
-            syncPoller.waitForCompletion();
-
-            assertThrows(RuntimeException.class, () -> client.beginRecognizeContent(null, LAYOUT_FILE_LENGTH,
-                FormContentType.IMAGE_JPEG, null).getSyncPoller());
-        });
+    public void recognizeContentResultWithNullData(HttpClient httpClient, FormRecognizerServiceVersion serviceVersion) {
+        assertThrows(RuntimeException.class, () -> client.beginRecognizeContent(null, LAYOUT_FILE_LENGTH,
+            FormContentType.IMAGE_JPEG, null).getSyncPoller());
     }
 
+
     /**
-     * Verifies layout data for a document using source as input stream data.
-     * And the content type is not given. The content type will be auto detected.
+     * Verifies content type will be auto detected when using content/layout API with input stream data overload.
      */
     @ParameterizedTest(name = DISPLAY_NAME_WITH_ARGUMENTS)
     @MethodSource("com.azure.ai.formrecognizer.TestUtils#getTestParameters")
-    public void recognizeLayoutDataWithContentTypeAutoDetection(HttpClient httpClient,
+    public void recognizeContentResultWithContentTypeAutoDetection(HttpClient httpClient,
         FormRecognizerServiceVersion serviceVersion) {
         client = getFormRecognizerAsyncClient(httpClient, serviceVersion);
-        layoutDataRunner((data) -> {
+        contentFromDataRunner((data) -> {
             SyncPoller<OperationResult, IterableStream<FormPage>> syncPoller =
                 client.beginRecognizeContent(getReplayableBufferData(LAYOUT_LOCAL_URL), LAYOUT_FILE_LENGTH, null,
                     null).getSyncPoller();
@@ -245,9 +237,9 @@ public void recognizeLayoutDataWithContentTypeAutoDetection(HttpClient httpClien
      */
     @ParameterizedTest(name = DISPLAY_NAME_WITH_ARGUMENTS)
     @MethodSource("com.azure.ai.formrecognizer.TestUtils#getTestParameters")
-    public void recognizeLayoutSourceUrl(HttpClient httpClient, FormRecognizerServiceVersion serviceVersion) {
+    public void recognizeContentFromUrl(HttpClient httpClient, FormRecognizerServiceVersion serviceVersion) {
         client = getFormRecognizerAsyncClient(httpClient, serviceVersion);
-        layoutSourceUrlRunner(sourceUrl -> {
+        contentFromUrlRunner(sourceUrl -> {
             SyncPoller<OperationResult, IterableStream<FormPage>> syncPoller =
                 client.beginRecognizeContentFromUrl(sourceUrl).getSyncPoller();
             syncPoller.waitForCompletion();
@@ -260,7 +252,7 @@ public void recognizeLayoutSourceUrl(HttpClient httpClient, FormRecognizerServic
      */
     @ParameterizedTest(name = DISPLAY_NAME_WITH_ARGUMENTS)
     @MethodSource("com.azure.ai.formrecognizer.TestUtils#getTestParameters")
-    public void recognizeLayoutInvalidSourceUrl(HttpClient httpClient, FormRecognizerServiceVersion serviceVersion) {
+    public void recognizeContentInvalidSourceUrl(HttpClient httpClient, FormRecognizerServiceVersion serviceVersion) {
         client = getFormRecognizerAsyncClient(httpClient, serviceVersion);
         invalidSourceUrlRunner((invalidSourceUrl) -> assertThrows(ErrorResponseException.class,
             () -> client.beginRecognizeContentFromUrl(invalidSourceUrl).getSyncPoller()));
@@ -326,9 +318,9 @@ public void recognizeCustomFormLabeledDataWithNullValues(HttpClient httpClient,
         }));
     }
 
+
     /**
-     * Verifies custom form data for a document using source as input stream data and valid labeled model Id.
-     * And the content type is not given. The content type will be auto detected.
+     * Verifies content type will be auto detected when using custom form API with input stream data overload.
      */
     @ParameterizedTest(name = DISPLAY_NAME_WITH_ARGUMENTS)
     @MethodSource("com.azure.ai.formrecognizer.TestUtils#getTestParameters")
@@ -367,4 +359,88 @@ public void recognizeCustomFormUnlabeledData(HttpClient httpClient, FormRecogniz
             validateRecognizedResult(syncPoller.getFinalResult(), false, false);
         }));
     }
+
+    @ParameterizedTest(name = DISPLAY_NAME_WITH_ARGUMENTS)
+    @MethodSource("com.azure.ai.formrecognizer.TestUtils#getTestParameters")
+    public void recognizeCustomFormMultiPageUnlabeled(HttpClient httpClient, FormRecognizerServiceVersion serviceVersion) {
+        client = getFormRecognizerAsyncClient(httpClient, serviceVersion);
+        multipageFromDataRunner(data -> beginTrainingMultipageRunner((trainingFilesUrl) -> {
+            SyncPoller<OperationResult, CustomFormModel> trainingPoller =
+                client.getFormTrainingAsyncClient().beginTraining(trainingFilesUrl, false).getSyncPoller();
+            trainingPoller.waitForCompletion();
+
+            SyncPoller<OperationResult, IterableStream<RecognizedForm>> syncPoller =
+                client.beginRecognizeCustomForms(toFluxByteBuffer(data), trainingPoller.getFinalResult().getModelId(),
+                    MULTIPAGE_INVOICE_FILE_LENGTH, FormContentType.APPLICATION_PDF).getSyncPoller();
+            syncPoller.waitForCompletion();
+            validateMultiPageDataUnlabeled(syncPoller.getFinalResult());
+        }));
+    }
+
+    @ParameterizedTest(name = DISPLAY_NAME_WITH_ARGUMENTS)
+    @MethodSource("com.azure.ai.formrecognizer.TestUtils#getTestParameters")
+    public void recognizeCustomFormUrlMultiPageLabeled(HttpClient httpClient, FormRecognizerServiceVersion serviceVersion) {
+        client = getFormRecognizerAsyncClient(httpClient, serviceVersion);
+        multipageFromUrlRunner(fileUrl -> beginTrainingMultipageRunner((trainingFilesUrl) -> {
+            SyncPoller<OperationResult, CustomFormModel> trainingPoller =
+                client.getFormTrainingAsyncClient().beginTraining(trainingFilesUrl, true).getSyncPoller();
+            trainingPoller.waitForCompletion();
+
+            SyncPoller<OperationResult, IterableStream<RecognizedForm>> syncPoller =
+                client.beginRecognizeCustomFormsFromUrl(fileUrl, trainingPoller.getFinalResult().getModelId()).getSyncPoller();
+            syncPoller.waitForCompletion();
+            validateMultiPageDataLabeled(syncPoller.getFinalResult());
+        }));
+    }
+
+    @ParameterizedTest(name = DISPLAY_NAME_WITH_ARGUMENTS)
+    @MethodSource("com.azure.ai.formrecognizer.TestUtils#getTestParameters")
+    public void recognizeReceiptFromUrlMultiPage(HttpClient httpClient, FormRecognizerServiceVersion serviceVersion) {
+        client = getFormRecognizerAsyncClient(httpClient, serviceVersion);
+        multipageFromUrlRunner(fileUrl -> {
+            SyncPoller<OperationResult, IterableStream<RecognizedReceipt>> syncPoller =
+                client.beginRecognizeReceiptsFromUrl(fileUrl).getSyncPoller();
+            syncPoller.waitForCompletion();
+            validateMultipageReceiptData(syncPoller.getFinalResult());
+        });
+    }
+
+    @ParameterizedTest(name = DISPLAY_NAME_WITH_ARGUMENTS)
+    @MethodSource("com.azure.ai.formrecognizer.TestUtils#getTestParameters")
+    public void recognizeReceiptFromDataMultiPage(HttpClient httpClient, FormRecognizerServiceVersion serviceVersion) {
+        client = getFormRecognizerAsyncClient(httpClient, serviceVersion);
+        multipageFromDataRunner(data -> {
+            SyncPoller<OperationResult, IterableStream<RecognizedReceipt>> syncPoller =
+                client.beginRecognizeReceipts(toFluxByteBuffer(data), MULTIPAGE_INVOICE_FILE_LENGTH,
+                    FormContentType.APPLICATION_PDF).getSyncPoller();
+            syncPoller.waitForCompletion();
+            validateMultipageReceiptData(syncPoller.getFinalResult());
+        });
+    }
+
+    @ParameterizedTest(name = DISPLAY_NAME_WITH_ARGUMENTS)
+    @MethodSource("com.azure.ai.formrecognizer.TestUtils#getTestParameters")
+    public void recognizeContentFromUrlMultiPage(HttpClient httpClient, FormRecognizerServiceVersion serviceVersion) {
+        client = getFormRecognizerAsyncClient(httpClient, serviceVersion);
+        multipageFromUrlRunner((fileUrl) -> {
+            SyncPoller<OperationResult, IterableStream<FormPage>> syncPoller =
+                client.beginRecognizeContentFromUrl(fileUrl).getSyncPoller();
+            syncPoller.waitForCompletion();
+            validateLayoutDataResults(syncPoller.getFinalResult(), false);
+        });
+    }
+
+    @ParameterizedTest(name = DISPLAY_NAME_WITH_ARGUMENTS)
+    @MethodSource("com.azure.ai.formrecognizer.TestUtils#getTestParameters")
+    public void recognizeContentFromDataMultiPage(HttpClient httpClient, FormRecognizerServiceVersion serviceVersion) {
+        client = getFormRecognizerAsyncClient(httpClient, serviceVersion);
+        multipageFromDataRunner(data -> {
+            SyncPoller<OperationResult, IterableStream<FormPage>> syncPoller =
+                client.beginRecognizeContent(toFluxByteBuffer(data), MULTIPAGE_INVOICE_FILE_LENGTH,
+                    FormContentType.APPLICATION_PDF).getSyncPoller();
+            syncPoller.waitForCompletion();
+            validateLayoutDataResults(syncPoller.getFinalResult(), false);
+        });
+    }
+
 }