diff --git a/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/6_6_0/4860-bulk-export-progress-less-than-100-percent.yaml b/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/6_6_0/4860-bulk-export-progress-less-than-100-percent.yaml new file mode 100644 index 000000000000..2da2365fb737 --- /dev/null +++ b/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/6_6_0/4860-bulk-export-progress-less-than-100-percent.yaml @@ -0,0 +1,5 @@ +--- +type: fix +issue: 4860 +title: "Running an $export that completes successfully results in a progress percentage of less than 100%. + This has now been fixed." diff --git a/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/batch2/Batch2CoordinatorIT.java b/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/batch2/Batch2CoordinatorIT.java index 38730c336348..be9f2a4b00c2 100644 --- a/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/batch2/Batch2CoordinatorIT.java +++ b/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/batch2/Batch2CoordinatorIT.java @@ -219,6 +219,15 @@ public void testFirstStepToSecondStep_singleChunkFasttracks() throws Interrupted // Since there was only one chunk, the job should proceed without requiring a maintenance pass myBatch2JobHelper.awaitJobCompletion(batchJobId); myLastStepLatch.awaitExpected(); + + final List jobInstances = myJobPersistence.fetchInstances(10, 0); + + assertEquals(1, jobInstances.size()); + + final JobInstance jobInstance = jobInstances.get(0); + + assertEquals(StatusEnum.COMPLETED, jobInstance.getStatus()); + assertEquals(1.0, jobInstance.getProgress()); } private void createThreeStepReductionJob( @@ -360,6 +369,15 @@ private void complete( testInfo + i )); } + + final List jobInstances = myJobPersistence.fetchInstances(10, 0); + + assertEquals(1, jobInstances.size()); + + final JobInstance jobInstance = jobInstances.get(0); + + assertEquals(StatusEnum.COMPLETED, jobInstance.getStatus()); + assertEquals(1.0, jobInstance.getProgress()); } @Test diff --git a/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/coordinator/ReductionStepDataSink.java b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/coordinator/ReductionStepDataSink.java index 7c348888219e..639d8f9e4cbf 100644 --- a/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/coordinator/ReductionStepDataSink.java +++ b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/coordinator/ReductionStepDataSink.java @@ -86,7 +86,7 @@ public void accept(WorkChunkData theData) { * here. Until then though, this is safer. */ - progress.updateInstance(instance); + progress.updateInstanceForReductionStep(instance); instance.setReport(dataString); instance.setStatus(StatusEnum.COMPLETED); diff --git a/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/progress/InstanceProgress.java b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/progress/InstanceProgress.java index e7ade344c4cf..d5386c08bb67 100644 --- a/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/progress/InstanceProgress.java +++ b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/progress/InstanceProgress.java @@ -105,13 +105,29 @@ private void updateRecordsProcessed(WorkChunk theChunk) { } } + /** + * Signal to the progress calculator to skip the incomplete work chunk count when determining the completed percentage. + *

+ * This is a hack: The reason we do this is to get around a race condition in which all work chunks are complete but + * the last chunk is * still in QUEUED status and will only be marked COMPLETE later. + * + * @param theInstance The Batch 2 {@link JobInstance} that we're updating + */ + public void updateInstanceForReductionStep(JobInstance theInstance) { + updateInstance(theInstance, true); + } + + public void updateInstance(JobInstance theInstance) { + updateInstance(theInstance, false); + } + /** * Update the job instance with status information. * We shouldn't read any values from theInstance here -- just write. * * @param theInstance the instance to update with progress statistics */ - public void updateInstance(JobInstance theInstance) { + public void updateInstance(JobInstance theInstance, boolean theCalledFromReducer) { if (myEarliestStartTime != null) { theInstance.setStartTime(myEarliestStartTime); } @@ -122,7 +138,9 @@ public void updateInstance(JobInstance theInstance) { theInstance.setCombinedRecordsProcessed(myRecordsProcessed); if (getChunkCount() > 0) { - double percentComplete = (double) (myCompleteChunkCount) / (double) getChunkCount(); + final int chunkCount = getChunkCount(); + final int conditionalChunkCount = theCalledFromReducer ? (chunkCount - myIncompleteChunkCount) : chunkCount; + final double percentComplete = (double) (myCompleteChunkCount) / (double) conditionalChunkCount; theInstance.setProgress(percentComplete); }