Reducer LOB usage in Batch2 and Search

hapi-deployable-pom/pom.xml

@@ -228,6 +228,7 @@
 								</goals>
 								<phase>process-sources</phase>
 								<configuration>
+									<skipUpdateLicense>true</skipUpdateLicense>
 									<addJavaLicenseAfterPackage>false</addJavaLicenseAfterPackage>
 									<licenseName>apache_v2</licenseName>
 									<canUpdateDescription>true</canUpdateDescription>

hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/7_2_0/5748-avoid-lob-usage-in-batch2-and-search.yaml

@@ -0,0 +1,9 @@
+---
+type: perf
+issue: 5748
+backport: 6.8.6
+title: "In the JPA server, several database columns related to Batch2 jobs and searching
+   have been reworked so that they no will longer use LOB datatypes going forward. This
+   is a significant advantage on Postgresql databases as it removes a significant use
+   of the inefficient `pg_largeobject` table, and should yield performance boosts for
+   MSSQL as well."

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/IBatch2JobInstanceRepository.java

@@ -55,14 +55,15 @@ int updateInstanceStatusIfIn(
 	int updateWorkChunksPurgedTrue(@Param("id") String theInstanceId);
 
 	@Query(
-			"SELECT b from Batch2JobInstanceEntity b WHERE b.myDefinitionId = :defId AND b.myParamsJson = :params AND b.myStatus IN( :stats )")
+			"SELECT b from Batch2JobInstanceEntity b WHERE b.myDefinitionId = :defId AND (b.myParamsJson = :params OR b.myParamsJsonVc = :params) AND b.myStatus IN( :stats )")
 	List<Batch2JobInstanceEntity> findInstancesByJobIdParamsAndStatus(
 			@Param("defId") String theDefinitionId,
 			@Param("params") String theParams,
 			@Param("stats") Set<StatusEnum> theStatus,
 			Pageable thePageable);
 
-	@Query("SELECT b from Batch2JobInstanceEntity b WHERE b.myDefinitionId = :defId AND b.myParamsJson = :params")
+	@Query(
+			"SELECT b from Batch2JobInstanceEntity b WHERE b.myDefinitionId = :defId AND (b.myParamsJson = :params OR b.myParamsJsonVc = :params)")
 	List<Batch2JobInstanceEntity> findInstancesByJobIdAndParams(
 			@Param("defId") String theDefinitionId, @Param("params") String theParams, Pageable thePageable);
 

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/IBatch2WorkChunkRepository.java

@@ -65,7 +65,7 @@ Stream<Batch2WorkChunkEntity> fetchChunksForStep(
 
 	@Modifying
 	@Query("UPDATE Batch2WorkChunkEntity e SET e.myStatus = :status, e.myEndTime = :et, "
-			+ "e.myRecordsProcessed = :rp, e.myErrorCount = e.myErrorCount + :errorRetries, e.mySerializedData = null, "
+			+ "e.myRecordsProcessed = :rp, e.myErrorCount = e.myErrorCount + :errorRetries, e.mySerializedData = null, e.mySerializedDataVc = null, "
 			+ "e.myWarningMessage = :warningMessage WHERE e.myId = :id")
 	void updateChunkStatusAndClearDataForEndSuccess(
 			@Param("id") String theChunkId,
@@ -77,7 +77,7 @@ void updateChunkStatusAndClearDataForEndSuccess(
 
 	@Modifying
 	@Query(
-			"UPDATE Batch2WorkChunkEntity e SET e.myStatus = :status, e.myEndTime = :et, e.mySerializedData = null, e.myErrorMessage = :em WHERE e.myId IN(:ids)")
+			"UPDATE Batch2WorkChunkEntity e SET e.myStatus = :status, e.myEndTime = :et, e.mySerializedData = null, e.mySerializedDataVc = null, e.myErrorMessage = :em WHERE e.myId IN(:ids)")
 	void updateAllChunksForInstanceStatusClearDataAndSetError(
 			@Param("ids") List<String> theChunkIds,
 			@Param("et") Date theEndTime,

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/Batch2JobInstanceEntity.java

@@ -21,6 +21,7 @@
 
 import ca.uhn.fhir.batch2.model.JobDefinition;
 import ca.uhn.fhir.batch2.model.StatusEnum;
+import ca.uhn.fhir.jpa.model.util.JpaConstants;
 import org.apache.commons.lang3.builder.ToStringBuilder;
 import org.apache.commons.lang3.builder.ToStringStyle;
 
@@ -43,6 +44,7 @@
 import static ca.uhn.fhir.batch2.model.JobDefinition.ID_MAX_LENGTH;
 import static ca.uhn.fhir.jpa.entity.Batch2WorkChunkEntity.ERROR_MSG_MAX_LENGTH;
 import static ca.uhn.fhir.jpa.entity.Batch2WorkChunkEntity.WARNING_MSG_MAX_LENGTH;
+import static ca.uhn.fhir.jpa.model.entity.ResourceHistoryTable.RES_TEXT_VC_MAX_LENGTH;
 import static org.apache.commons.lang3.StringUtils.left;
 
 @Entity
@@ -93,13 +95,18 @@ public class Batch2JobInstanceEntity implements Serializable {
 	@Column(name = "FAST_TRACKING", nullable = true)
 	private Boolean myFastTracking;
 
+	// TODO: VC column added in 7.2.0 - Remove non-VC column later
 	@Column(name = "PARAMS_JSON", length = PARAMS_JSON_MAX_LENGTH, nullable = true)
 	private String myParamsJson;
 
-	@Lob
+	@Lob // TODO: VC column added in 7.2.0 - Remove non-VC column later
 	@Column(name = "PARAMS_JSON_LOB", nullable = true)
 	private String myParamsJsonLob;
 
+	@Column(name = "PARAMS_JSON_VC", nullable = true, length = RES_TEXT_VC_MAX_LENGTH)
+	@org.hibernate.annotations.Type(type = JpaConstants.ORG_HIBERNATE_TYPE_TEXT_TYPE)
+	private String myParamsJsonVc;
+
 	@Column(name = "CMB_RECS_PROCESSED", nullable = true)
 	private Integer myCombinedRecordsProcessed;
 
@@ -134,11 +141,15 @@ public class Batch2JobInstanceEntity implements Serializable {
 	 * Any output from the job can be held in this column
 	 * Even serialized json
 	 */
-	@Lob
+	@Lob // TODO: VC column added in 7.2.0 - Remove non-VC column later
 	@Basic(fetch = FetchType.LAZY)
 	@Column(name = "REPORT", nullable = true, length = Integer.MAX_VALUE - 1)
 	private String myReport;
 
+	@Column(name = "REPORT_VC", nullable = true, length = RES_TEXT_VC_MAX_LENGTH)
+	@org.hibernate.annotations.Type(type = JpaConstants.ORG_HIBERNATE_TYPE_TEXT_TYPE)
+	private String myReportVc;
+
 	public String getCurrentGatedStepId() {
 		return myCurrentGatedStepId;
 	}
@@ -252,20 +263,19 @@ public void setStatus(StatusEnum theStatus) {
 	}
 
 	public String getParams() {
+		if (myParamsJsonVc != null) {
+			return myParamsJsonVc;
+		}
 		if (myParamsJsonLob != null) {
 			return myParamsJsonLob;
 		}
 		return myParamsJson;
 	}
 
 	public void setParams(String theParams) {
+		myParamsJsonVc = theParams;
 		myParamsJsonLob = null;
 		myParamsJson = null;
-		if (theParams != null && theParams.length() > PARAMS_JSON_MAX_LENGTH) {
-			myParamsJsonLob = theParams;
-		} else {
-			myParamsJson = theParams;
-		}
 	}
 
 	public boolean getWorkChunksPurged() {
@@ -301,11 +311,12 @@ public void setEstimatedTimeRemaining(String theEstimatedTimeRemaining) {
 	}
 
 	public String getReport() {
-		return myReport;
+		return myReportVc != null ? myReportVc : myReport;
 	}
 
 	public void setReport(String theReport) {
-		myReport = theReport;
+		myReportVc = theReport;
+		myReport = null;
 	}
 
 	public String getWarningMessages() {
@@ -336,7 +347,7 @@ public String toString() {
 				.append("progress", myProgress)
 				.append("errorMessage", myErrorMessage)
 				.append("estimatedTimeRemaining", myEstimatedTimeRemaining)
-				.append("report", myReport)
+				.append("report", getReport())
 				.append("warningMessages", myWarningMessages)
 				.toString();
 	}

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/Batch2WorkChunkEntity.java

@@ -20,6 +20,7 @@
 package ca.uhn.fhir.jpa.entity;
 
 import ca.uhn.fhir.batch2.model.WorkChunkStatusEnum;
+import ca.uhn.fhir.jpa.model.util.JpaConstants;
 import org.apache.commons.lang3.builder.ToStringBuilder;
 import org.apache.commons.lang3.builder.ToStringStyle;
 
@@ -44,6 +45,7 @@
 
 import static ca.uhn.fhir.batch2.model.JobDefinition.ID_MAX_LENGTH;
 import static ca.uhn.fhir.jpa.entity.Batch2JobInstanceEntity.STATUS_MAX_LENGTH;
+import static ca.uhn.fhir.jpa.model.entity.ResourceHistoryTable.RES_TEXT_VC_MAX_LENGTH;
 import static org.apache.commons.lang3.StringUtils.left;
 
 @Entity
@@ -92,11 +94,15 @@ public class Batch2WorkChunkEntity implements Serializable {
 	@Column(name = "TGT_STEP_ID", length = ID_MAX_LENGTH, nullable = false)
 	private String myTargetStepId;
 
-	@Lob
+	@Lob // TODO: VC column added in 7.2.0 - Remove non-VC column later
 	@Basic(fetch = FetchType.LAZY)
 	@Column(name = "CHUNK_DATA", nullable = true, length = Integer.MAX_VALUE - 1)
 	private String mySerializedData;
 
+	@Column(name = "CHUNK_DATA_VC", nullable = true, length = RES_TEXT_VC_MAX_LENGTH)
+	@org.hibernate.annotations.Type(type = JpaConstants.ORG_HIBERNATE_TYPE_TEXT_TYPE)
+	private String mySerializedDataVc;
+
 	@Column(name = "STAT", length = STATUS_MAX_LENGTH, nullable = false)
 	@Enumerated(EnumType.STRING)
 	private WorkChunkStatusEnum myStatus;
@@ -263,11 +269,12 @@ public void setTargetStepId(String theTargetStepId) {
 	}
 
 	public String getSerializedData() {
-		return mySerializedData;
+		return mySerializedDataVc != null ? mySerializedDataVc : mySerializedData;
 	}
 
 	public void setSerializedData(String theSerializedData) {
-		mySerializedData = theSerializedData;
+		mySerializedData = null;
+		mySerializedDataVc = theSerializedData;
 	}
 
 	public WorkChunkStatusEnum getStatus() {
@@ -309,7 +316,7 @@ public String toString() {
 				.append("updateTime", myUpdateTime)
 				.append("recordsProcessed", myRecordsProcessed)
 				.append("targetStepId", myTargetStepId)
-				.append("serializedData", mySerializedData)
+				.append("serializedData", getSerializedData())
 				.append("status", myStatus)
 				.append("errorMessage", myErrorMessage)
 				.append("warningMessage", myWarningMessage)

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/BulkImportJobFileEntity.java

@@ -20,6 +20,7 @@
 package ca.uhn.fhir.jpa.entity;
 
 import ca.uhn.fhir.jpa.bulk.imprt.model.BulkImportJobFileJson;
+import ca.uhn.fhir.jpa.model.util.JpaConstants;
 
 import java.io.Serializable;
 import java.nio.charset.StandardCharsets;
@@ -36,6 +37,7 @@
 import javax.persistence.SequenceGenerator;
 import javax.persistence.Table;
 
+import static ca.uhn.fhir.jpa.model.entity.ResourceHistoryTable.RES_TEXT_VC_MAX_LENGTH;
 import static org.apache.commons.lang3.StringUtils.left;
 
 @Entity
@@ -66,10 +68,14 @@ public class BulkImportJobFileEntity implements Serializable {
 	@Column(name = "FILE_DESCRIPTION", nullable = true, length = MAX_DESCRIPTION_LENGTH)
 	private String myFileDescription;
 
-	@Lob
-	@Column(name = "JOB_CONTENTS", nullable = false)
+	@Lob // TODO: VC column added in 7.2.0 - Remove non-VC column later
+	@Column(name = "JOB_CONTENTS", nullable = true)
 	private byte[] myContents;
 
+	@Column(name = "JOB_CONTENTS_VC", nullable = true, length = RES_TEXT_VC_MAX_LENGTH)
+	@org.hibernate.annotations.Type(type = JpaConstants.ORG_HIBERNATE_TYPE_TEXT_TYPE)
+	private String myContentsVc;
+
 	@Column(name = "TENANT_NAME", nullable = true, length = PartitionEntity.MAX_NAME_LENGTH)
 	private String myTenantName;
 
@@ -98,11 +104,16 @@ public void setFileSequence(int theFileSequence) {
 	}
 
 	public String getContents() {
-		return new String(myContents, StandardCharsets.UTF_8);
+		if (myContentsVc != null) {
+			return myContentsVc;
+		} else {
+			return new String(myContents, StandardCharsets.UTF_8);
+		}
 	}
 
 	public void setContents(String theContents) {
-		myContents = theContents.getBytes(StandardCharsets.UTF_8);
+		myContentsVc = theContents;
+		myContents = null;
 	}
 
 	public BulkImportJobFileJson toJson() {

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/Search.java

@@ -21,6 +21,7 @@
 
 import ca.uhn.fhir.interceptor.model.RequestPartitionId;
 import ca.uhn.fhir.jpa.model.search.SearchStatusEnum;
+import ca.uhn.fhir.jpa.model.util.JpaConstants;
 import ca.uhn.fhir.jpa.searchparam.SearchParameterMap;
 import ca.uhn.fhir.model.api.Include;
 import ca.uhn.fhir.rest.param.DateRangeParam;
@@ -30,6 +31,7 @@
 import org.apache.commons.lang3.SerializationUtils;
 import org.apache.commons.lang3.builder.ToStringBuilder;
 import org.hibernate.annotations.OptimisticLock;
+import org.hibernate.annotations.Type;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -64,6 +66,7 @@
 import javax.persistence.UniqueConstraint;
 import javax.persistence.Version;
 
+import static ca.uhn.fhir.jpa.model.entity.ResourceHistoryTable.RES_TEXT_VC_MAX_LENGTH;
 import static org.apache.commons.lang3.StringUtils.left;
 
 @Entity
@@ -139,14 +142,22 @@ public class Search implements ICachedSearchDetails, Serializable {
 
 	@Column(name = "RESOURCE_TYPE", length = 200, nullable = true)
 	private String myResourceType;
+
 	/**
 	 * Note that this field may have the request partition IDs prepended to it
 	 */
-	@Lob()
+	@Lob // TODO: VC column added in 7.2.0 - Remove non-VC column later
 	@Basic(fetch = FetchType.LAZY)
 	@Column(name = "SEARCH_QUERY_STRING", nullable = true, updatable = false, length = MAX_SEARCH_QUERY_STRING)
 	private String mySearchQueryString;
 
+	/**
+	 * Note that this field may have the request partition IDs prepended to it
+	 */
+	@Column(name = "SEARCH_QUERY_STRING_VC", nullable = true, length = RES_TEXT_VC_MAX_LENGTH)
+	@org.hibernate.annotations.Type(type = JpaConstants.ORG_HIBERNATE_TYPE_TEXT_TYPE)
+	private String mySearchQueryStringVc;
+
 	@Column(name = "SEARCH_QUERY_STRING_HASH", nullable = true, updatable = false)
 	private Integer mySearchQueryStringHash;
 
@@ -169,10 +180,14 @@ public class Search implements ICachedSearchDetails, Serializable {
 	@Column(name = "OPTLOCK_VERSION", nullable = true)
 	private Integer myVersion;
 
-	@Lob
+	@Lob // TODO: VC column added in 7.2.0 - Remove non-VC column later
 	@Column(name = "SEARCH_PARAM_MAP", nullable = true)
 	private byte[] mySearchParameterMap;
 
+	@Column(name = "SEARCH_PARAM_MAP_BIN", nullable = true, length = JpaConstants.ORG_HIBERNATE_TYPE_BINARY_TYPE_LENGTH)
+	@Type(type = JpaConstants.ORG_HIBERNATE_TYPE_BINARY_TYPE)
+	private byte[] mySearchParameterMapBin;
+
 	@Transient
 	private transient SearchParameterMap mySearchParameterMapTransient;
 
@@ -347,7 +362,7 @@ public void setResourceType(String theResourceType) {
 	 * Note that this field may have the request partition IDs prepended to it
 	 */
 	public String getSearchQueryString() {
-		return mySearchQueryString;
+		return mySearchQueryStringVc != null ? mySearchQueryStringVc : mySearchQueryString;
 	}
 
 	public void setSearchQueryString(String theSearchQueryString, RequestPartitionId theRequestPartitionId) {
@@ -359,12 +374,13 @@ public void setSearchQueryString(String theSearchQueryString, RequestPartitionId
 			// We want this field to always have a wide distribution of values in order
 			// to avoid optimizers avoiding using it if it has lots of nulls, so in the
 			// case of null, just put a value that will never be hit
-			mySearchQueryString = UUID.randomUUID().toString();
+			mySearchQueryStringVc = UUID.randomUUID().toString();
 		} else {
-			mySearchQueryString = searchQueryString;
+			mySearchQueryStringVc = searchQueryString;
 		}
 
-		mySearchQueryStringHash = mySearchQueryString.hashCode();
+		mySearchQueryString = null;
+		mySearchQueryStringHash = mySearchQueryStringVc.hashCode();
 	}
 
 	public SearchTypeEnum getSearchType() {
@@ -463,16 +479,21 @@ public Optional<SearchParameterMap> getSearchParameterMap() {
 			return Optional.of(mySearchParameterMapTransient);
 		}
 		SearchParameterMap searchParameterMap = null;
-		if (mySearchParameterMap != null) {
-			searchParameterMap = SerializationUtils.deserialize(mySearchParameterMap);
+		byte[] searchParameterMapSerialized = mySearchParameterMapBin;
+		if (searchParameterMapSerialized == null) {
+			searchParameterMapSerialized = mySearchParameterMap;
+		}
+		if (searchParameterMapSerialized != null) {
+			searchParameterMap = SerializationUtils.deserialize(searchParameterMapSerialized);
 			mySearchParameterMapTransient = searchParameterMap;
 		}
 		return Optional.ofNullable(searchParameterMap);
 	}
 
 	public void setSearchParameterMap(SearchParameterMap theSearchParameterMap) {
 		mySearchParameterMapTransient = theSearchParameterMap;
-		mySearchParameterMap = SerializationUtils.serialize(theSearchParameterMap);
+		mySearchParameterMapBin = SerializationUtils.serialize(theSearchParameterMap);
+		mySearchParameterMap = null;
 	}
 
 	@Override