Fix erroneous batch 2 $export 75% complete count when the job is COMPLETE

hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/6_6_0/4860-bulk-export-progress-less-than-100-percent.yaml

@@ -0,0 +1,5 @@
+---
+type: fix
+issue: 4860
+title: "Running an $export that completes successfully results in a progress percentage of less than 100%.
+        This has now been fixed."

hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/batch2/Batch2CoordinatorIT.java

@@ -219,6 +219,15 @@ public void testFirstStepToSecondStep_singleChunkFasttracks() throws Interrupted
 		// Since there was only one chunk, the job should proceed without requiring a maintenance pass
 		myBatch2JobHelper.awaitJobCompletion(batchJobId);
 		myLastStepLatch.awaitExpected();
+
+		final List<JobInstance> jobInstances = myJobPersistence.fetchInstances(10, 0);
+
+		assertEquals(1, jobInstances.size());
+
+		final JobInstance jobInstance = jobInstances.get(0);
+
+		assertEquals(StatusEnum.COMPLETED, jobInstance.getStatus());
+		assertEquals(1.0, jobInstance.getProgress());
 	}
 
 	private void createThreeStepReductionJob(
@@ -360,6 +369,15 @@ private void complete(
 				testInfo + i
 			));
 		}
+
+		final List<JobInstance> jobInstances = myJobPersistence.fetchInstances(10, 0);
+
+		assertEquals(1, jobInstances.size());
+
+		final JobInstance jobInstance = jobInstances.get(0);
+
+		assertEquals(StatusEnum.COMPLETED, jobInstance.getStatus());
+		assertEquals(1.0, jobInstance.getProgress());
 	}
 
 	@Test

hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/coordinator/ReductionStepDataSink.java

@@ -86,7 +86,7 @@ public void accept(WorkChunkData<OT> theData) {
 			 * here. Until then though, this is safer.
 			 */
 
-			progress.updateInstance(instance);
+			progress.updateInstanceForReductionStep(instance);
 
 			instance.setReport(dataString);
 			instance.setStatus(StatusEnum.COMPLETED);

hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/progress/InstanceProgress.java

@@ -105,13 +105,29 @@ private void updateRecordsProcessed(WorkChunk theChunk) {
 		}
 	}
 
+	/**
+	 * Signal to the progress calculator to skip the incomplete work chunk count when determining the completed percentage.
+	 * <p/>
+	 * This is a hack:  The reason we do this is to get around a race condition in which all work chunks are complete but
+	 * the last chunk is * still in QUEUED status and will only be marked COMPLETE later.
+	 *
+	 * @param theInstance The Batch 2 {@link JobInstance} that we're updating
+	 */
+	public void updateInstanceForReductionStep(JobInstance theInstance) {
+		updateInstance(theInstance, true);
+	}
+
+	public void updateInstance(JobInstance theInstance) {
+		updateInstance(theInstance, false);
+	}
+
 	/**
 	 * Update the job instance with status information.
 	 * We shouldn't read any values from theInstance here -- just write.
 	 *
 	 * @param theInstance the instance to update with progress statistics
 	 */
-	public void updateInstance(JobInstance theInstance) {
+	public void updateInstance(JobInstance theInstance, boolean theCalledFromReducer) {
 		if (myEarliestStartTime != null) {
 			theInstance.setStartTime(myEarliestStartTime);
 		}
@@ -122,7 +138,9 @@ public void updateInstance(JobInstance theInstance) {
 		theInstance.setCombinedRecordsProcessed(myRecordsProcessed);
 
 		if (getChunkCount() > 0) {
-			double percentComplete = (double) (myCompleteChunkCount) / (double) getChunkCount();
+			final int chunkCount = getChunkCount();
+			final int conditionalChunkCount = theCalledFromReducer ? (chunkCount - myIncompleteChunkCount) : chunkCount;
+			final double percentComplete = (double) (myCompleteChunkCount) / (double) conditionalChunkCount;
 			theInstance.setProgress(percentComplete);
 		}