Skip to content

Commit

Permalink
Enhance feature for instances
Browse files Browse the repository at this point in the history
  • Loading branch information
@ShmElena
ShmElena committed Oct 2, 2024
1 parent 41274c2 commit 8796ac0
Show file tree
Hide file tree
Showing 10 changed files with 127 additions and 55 deletions.
46 changes: 11 additions & 35 deletions src/main/java/org/folio/dataexp/service/DownloadRecordService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,15 +3,12 @@

import java.io.IOException;
import java.util.List;
import java.util.Set;
import java.util.UUID;
import lombok.AllArgsConstructor;
import lombok.extern.log4j.Log4j2;
import org.folio.dataexp.domain.entity.MarcRecordEntity;
import org.folio.dataexp.exception.export.DownloadRecordException;
import org.folio.dataexp.repository.MappingProfileEntityRepository;
import org.folio.dataexp.service.export.ExportStrategyFactory;
import org.folio.dataexp.service.export.S3ExportsUploader;
import org.folio.dataexp.service.export.strategies.AuthorityExportStrategy;
import org.folio.dataexp.service.export.strategies.JsonToMarcConverter;
import org.folio.spring.FolioExecutionContext;
import org.springframework.core.io.ByteArrayResource;
Expand All @@ -22,8 +19,7 @@
@Log4j2
public class DownloadRecordService {

private final AuthorityExportStrategy authorityExportStrategy;
private final MappingProfileEntityRepository mappingProfileEntityRepository;
private final ExportStrategyFactory exportStrategyFactory;
private final JsonToMarcConverter jsonToMarcConverter;
private final S3ExportsUploader s3Uploader;
private final InputFileProcessor inputFileProcessor;
Expand All @@ -33,21 +29,11 @@ public ByteArrayResource processRecordDownload(final UUID recordId,
boolean isUtf,
final String formatPostfix,
final String idType) {
if ("AUTHORITY".equals(idType)) {
return processAuthorityDownload(recordId, isUtf, formatPostfix);
}
else {
log.error("processRecordDownload:: unsupported record id type: {}", idType);
throw new DownloadRecordException("Unsupported record id type: %s".formatted(idType));
}
}

public ByteArrayResource processAuthorityDownload(final UUID authorityId, boolean isUtf, final String formatPostfix) {
log.info("processAuthorityDownload:: start downloading authority with id: {}, isUtf: {}", authorityId, isUtf);
var dirName = authorityId.toString() + formatPostfix;
log.info("processRecordDownload:: start downloading record with id: {}, isUtf: {}", recordId, isUtf);
var dirName = recordId.toString() + formatPostfix;
var marcFileContent = getContentIfFileExists(dirName);
if (marcFileContent.isEmpty()) {
marcFileContent = generateAuthorityFileContent(authorityId, isUtf);
marcFileContent = generateRecordFileContent(recordId, isUtf, idType);
uploadMarcFile(dirName, marcFileContent);
}
return new ByteArrayResource(marcFileContent.getBytes());
Expand All @@ -62,29 +48,19 @@ private String getContentIfFileExists(final String dirName) {
}
}

private String generateAuthorityFileContent(final UUID authorityId, boolean isUtf) {
var marcAuthority = getMarcAuthority(authorityId);
var mappingProfile = mappingProfileEntityRepository.getReferenceById(
UUID.fromString("5d636597-a59d-4391-a270-4e79d5ba70e3")).getMappingProfile();
private String generateRecordFileContent(final UUID recordId, boolean isUtf, final String idType) {
var exportStrategy = exportStrategyFactory.getExportStrategy(idType);
var marcRecord = exportStrategy.getMarcRecord(recordId);
var mappingProfile = exportStrategy.getDefaultMappingProfile();
try {
return jsonToMarcConverter.convertJsonRecordToMarcRecord(marcAuthority.getContent(), List.of(),
return jsonToMarcConverter.convertJsonRecordToMarcRecord(marcRecord.getContent(), List.of(),
mappingProfile, isUtf);
} catch (IOException e) {
log.error("createAuthorityFileContent :: Error generating content for authority with ID: {}", authorityId);
log.error("generateRecordFileContent :: Error generating content for record with ID: {}", recordId);
throw new DownloadRecordException(e.getMessage());
}
}

private MarcRecordEntity getMarcAuthority(final UUID authorityId) {
var marcAuthorities = authorityExportStrategy.getMarcAuthorities(Set.of(authorityId));
if (marcAuthorities.isEmpty()) {
log.error("getMarcAuthority:: Couldn't find authority in db for ID: {}", authorityId);
throw new DownloadRecordException("Couldn't find authority in db for ID: %s".formatted(authorityId));
}
marcAuthorities = authorityExportStrategy.handleDuplicatedDeletedAndUseLastGeneration(marcAuthorities);
return marcAuthorities.get(0);
}

private void uploadMarcFile(final String dirName, final String marcFileContent) {
try {
s3Uploader.uploadSingleRecordById(dirName, marcFileContent);
Expand Down
11 changes: 11 additions & 0 deletions src/main/java/org/folio/dataexp/service/export/ExportStrategyFactory.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@

import lombok.AllArgsConstructor;
import org.folio.dataexp.domain.dto.ExportRequest;
import org.folio.dataexp.exception.export.DownloadRecordException;
import org.folio.dataexp.service.export.strategies.AbstractExportStrategy;
import org.folio.dataexp.service.export.strategies.AuthorityExportAllStrategy;
import org.folio.dataexp.service.export.strategies.AuthorityExportStrategy;
import org.folio.dataexp.service.export.strategies.ExportStrategy;
Expand Down Expand Up @@ -40,4 +42,13 @@ public ExportStrategy getExportStrategy(ExportRequest exportRequest) {
return instancesExportStrategy;
}

public AbstractExportStrategy getExportStrategy(String recordIdType) {
if ("AUTHORITY".equals(recordIdType)) {
return authorityExportStrategy;
}
if ("INSTANCE".equals(recordIdType)) {
return instancesExportStrategy;
}
throw new DownloadRecordException("Unsupported record id type: %s".formatted(recordIdType));
}
}
5 changes: 4 additions & 1 deletion src/main/java/org/folio/dataexp/service/export/strategies/AbstractExportStrategy.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,12 +55,12 @@ public abstract class AbstractExportStrategy implements ExportStrategy {

private InstanceEntityRepository instanceEntityRepository;
private ExportIdEntityRepository exportIdEntityRepository;
private MappingProfileEntityRepository mappingProfileEntityRepository;
private JobProfileEntityRepository jobProfileEntityRepository;
private JobExecutionService jobExecutionService;
private JsonToMarcConverter jsonToMarcConverter;

protected ErrorLogService errorLogService;
protected MappingProfileEntityRepository mappingProfileEntityRepository;
protected MarcAuthorityRecordAllRepository marcAuthorityRecordAllRepository;
protected FolioExecutionContext folioExecutionContext;

Expand Down Expand Up @@ -122,6 +122,9 @@ public void setStatusBaseExportStatistic(JobExecutionExportFilesEntity exportFil

abstract List<MarcRecordEntity> getMarcRecords(Set<UUID> externalIds, MappingProfile mappingProfile, ExportRequest exportRequest,
UUID jobExecutionId);
public abstract MarcRecordEntity getMarcRecord(UUID externalId);

public abstract MappingProfile getDefaultMappingProfile();

abstract GeneratedMarcResult getGeneratedMarc(Set<UUID> ids, MappingProfile mappingProfile, ExportRequest exportRequest,
UUID jobExecutionId, ExportStrategyStatistic exportStatistic);
Expand Down
2 changes: 1 addition & 1 deletion src/main/java/org/folio/dataexp/service/export/strategies/AuthorityExportAllStrategy.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ public AuthorityExportAllStrategy(ConsortiaService consortiaService, ErrorLogEnt
}

@Override
public List<MarcRecordEntity> getMarcAuthorities(Set<UUID> externalIds) {
protected List<MarcRecordEntity> getMarcAuthorities(Set<UUID> externalIds) {
return marcAuthorityRecordRepository.findAllByExternalIdIn(context.getTenantId(), externalIds);
}

Expand Down
22 changes: 20 additions & 2 deletions src/main/java/org/folio/dataexp/service/export/strategies/AuthorityExportStrategy.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
import org.folio.dataexp.domain.dto.ExportRequest;
import org.folio.dataexp.domain.dto.MappingProfile;
import org.folio.dataexp.domain.entity.MarcRecordEntity;
import org.folio.dataexp.exception.export.DownloadRecordException;
import org.folio.dataexp.repository.ErrorLogEntityCqlRepository;
import org.folio.dataexp.repository.MarcAuthorityRecordRepository;
import org.folio.dataexp.service.ConsortiaService;
Expand Down Expand Up @@ -79,10 +80,27 @@ List<MarcRecordEntity> getMarcRecords(Set<UUID> externalIds, MappingProfile mapp
return new ArrayList<>();
}

public List<MarcRecordEntity> getMarcAuthorities(Set<UUID> externalIds) {
protected List<MarcRecordEntity> getMarcAuthorities(Set<UUID> externalIds) {
return marcAuthorityRecordRepository.findNonDeletedByExternalIdIn(context.getTenantId(), externalIds);
}

@Override
public MarcRecordEntity getMarcRecord(final UUID recordId) {
List<MarcRecordEntity> marcAuthorities = getMarcAuthorities(Set.of(recordId));
if (marcAuthorities.isEmpty()) {
log.error("getMarcRecord:: Couldn't find authority in db for ID: {}", recordId);
throw new DownloadRecordException("Couldn't find authority in db for ID: %s".formatted(recordId));
}
marcAuthorities = handleDuplicatedDeletedAndUseLastGeneration(marcAuthorities);
return marcAuthorities.get(0);
}

@Override
public MappingProfile getDefaultMappingProfile() {
return mappingProfileEntityRepository.getReferenceById(
UUID.fromString("5d636597-a59d-4391-a270-4e79d5ba70e3")).getMappingProfile();
}

private void handleDeleted(List<MarcRecordEntity> marcAuthorities, UUID jobExecutionId, ExportRequest exportRequest,
Set<String> alreadySavedErrors) {
var iterator = marcAuthorities.iterator();
Expand Down Expand Up @@ -122,7 +140,7 @@ private void handleDeleted(List<MarcRecordEntity> marcAuthorities, UUID jobExecu
}
}

public List<MarcRecordEntity> handleDuplicatedDeletedAndUseLastGeneration(List<MarcRecordEntity> marcAuthorities) {
private List<MarcRecordEntity> handleDuplicatedDeletedAndUseLastGeneration(List<MarcRecordEntity> marcAuthorities) {
return marcAuthorities.stream().collect(
groupingBy(MarcRecordEntity::getExternalId,
maxBy(comparing(MarcRecordEntity::getGeneration)))).values().stream()
Expand Down
10 changes: 10 additions & 0 deletions src/main/java/org/folio/dataexp/service/export/strategies/HoldingsExportStrategy.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -349,4 +349,14 @@ private void addItemsToHolding(JSONObject holdingJson, UUID holdingId) {
});
holdingJson.put(ITEMS_KEY, itemJsonArray);
}

@Override
public MarcRecordEntity getMarcRecord(UUID externalId) {
throw new UnsupportedOperationException("The functionality is not required for holdings.");
}

@Override
public MappingProfile getDefaultMappingProfile() {
throw new UnsupportedOperationException("The functionality is not required for holdings.");
}
}
17 changes: 17 additions & 0 deletions src/main/java/org/folio/dataexp/service/export/strategies/InstancesExportStrategy.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
import org.folio.dataexp.domain.entity.InstanceEntity;
import org.folio.dataexp.domain.entity.MarcRecordEntity;
import org.folio.dataexp.exception.TransformationRuleException;
import org.folio.dataexp.exception.export.DownloadRecordException;
import org.folio.dataexp.repository.InstanceCentralTenantRepository;
import org.folio.dataexp.repository.InstanceEntityRepository;
import org.folio.dataexp.repository.InstanceWithHridEntityRepository;
Expand Down Expand Up @@ -174,6 +175,22 @@ public void saveConvertJsonRecordToMarcRecordError(MarcRecordEntity marcRecordEn
super.saveConvertJsonRecordToMarcRecordError(marcRecordEntity, jobExecutionId, e);
}

@Override
public MarcRecordEntity getMarcRecord(UUID recordId) {
var marcInstances = marcRecordEntityRepository.findByExternalIdInAndRecordTypeIsAndStateIn(Set.of(recordId), INSTANCE_MARC_TYPE, Set.of("ACTUAL"));
if (marcInstances.isEmpty()) {
log.error("getMarcRecord:: Couldn't find instance in db for ID: {}", recordId);
throw new DownloadRecordException("Couldn't find instance in db for ID: %s".formatted(recordId));
}
return marcInstances.get(0);
}

@Override
public MappingProfile getDefaultMappingProfile() {
return mappingProfileEntityRepository.getReferenceById(
UUID.fromString("25d81cbe-9686-11ea-bb37-0242ac130002")).getMappingProfile();
}

protected Optional<ExportIdentifiersForDuplicateErrors> getDefaultIdentifiers(UUID id) {
var exportIdentifiers = new ExportIdentifiersForDuplicateErrors();
exportIdentifiers.setIdentifierHridMessage("Instance with ID : " + id);
Expand Down
57 changes: 42 additions & 15 deletions src/test/java/org/folio/dataexp/service/DownloadRecordTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,8 +39,11 @@ class DownloadRecordTest extends BaseDataExportInitializer {
@MockBean
private MappingProfileEntityRepository mappingProfileEntityRepository;

private final static UUID LOCAL_MARC_AUTHORITY_UUID = UUID.fromString("17eed93e-f9e2-4cb2-a52b-e9155acfc119");
private final static UUID LOCAL_AUTHORITY_UUID = UUID.fromString("4a090b0f-9da3-40f1-ab17-33d6a1e3abae");
private final static UUID MISSING_AUTHORITY_UUID = UUID.fromString("17eed93e-f9e2-4cb2-a52b-e9155acfc119");
private final static UUID INSTANCE_UUID = UUID.fromString("71717177-f243-4e4a-bf1c-9e1e62b3171d");
private final static UUID AUTHORITY_UUID = UUID.fromString("4a090b0f-9da3-40f1-ab17-33d6a1e3abae");
private final static String AUTHORITY_ID_TYPE = "AUTHORITY";
private final static String INSTANCE_ID_TYPE = "INSTANCE";

private final MappingProfileEntity mappingProfileEntity = new MappingProfileEntity();

Expand All @@ -51,62 +54,86 @@ void init() {
mappingProfileEntity.setMappingProfile(mappingProfile);
when(mappingProfileEntityRepository.getReferenceById(UUID.fromString("5d636597-a59d-4391-a270-4e79d5ba70e3")))
.thenReturn(mappingProfileEntity);
when(mappingProfileEntityRepository.getReferenceById(UUID.fromString("25d81cbe-9686-11ea-bb37-0242ac130002")))
.thenReturn(mappingProfileEntity);
}

@Test
void whenAuthorityIsMissing_downloadShouldFail() {
try (var context = new FolioExecutionContextSetter(folioExecutionContext)) {
Exception exception = assertThrows(DownloadRecordException.class, () -> {
downloadRecordService.processAuthorityDownload(LOCAL_MARC_AUTHORITY_UUID, true, "-utf");
downloadRecordService.processRecordDownload(MISSING_AUTHORITY_UUID, true, "-utf", AUTHORITY_ID_TYPE);
});
assertEquals("Couldn't find authority in db for ID: 17eed93e-f9e2-4cb2-a52b-e9155acfc119",
exception.getMessage());
}
}

@Test
void whenUnknownRecordId_downloadShouldFail() {
void whenUnknownRecordIdType_downloadShouldFail() {
try (var context = new FolioExecutionContextSetter(folioExecutionContext)) {
Exception exception = assertThrows(DownloadRecordException.class, () -> {
downloadRecordService.processRecordDownload(LOCAL_MARC_AUTHORITY_UUID, true, "-utf", "HOLDING");
downloadRecordService.processRecordDownload(MISSING_AUTHORITY_UUID, true, "-utf", "HOLDING");
});
assertEquals("Unsupported record id type: HOLDING",
exception.getMessage());
}
}

@ParameterizedTest
@MethodSource("providedData")
void whenMarcFileDoesntExist_generateFileAndSaveInS3(boolean isUtf, String postfix, String fileContent) {
@MethodSource("providedAuthorityData")
void whenAuthorityMarcFileDoesntExist_generateFileAndSaveInS3(boolean isUtf, String postfix, String fileContent) {
try (var context = new FolioExecutionContextSetter(folioExecutionContext)) {
var filePath = "mod-data-export/download/4a090b0f-9da3-40f1-ab17-33d6a1e3abae%s/4a090b0f-9da3-40f1-ab17-33d6a1e3abae%s.mrc".formatted(postfix, postfix);
var expectedResult = new ByteArrayResource(fileContent.getBytes());

var actualResult = downloadRecordService.processRecordDownload(AUTHORITY_UUID, isUtf, postfix, AUTHORITY_ID_TYPE);

assertEquals(expectedResult, actualResult);
assertEquals(filePath, s3Client.list(filePath).get(0));
}
}

@ParameterizedTest
@MethodSource("providedInstanceData")
void whenInstanceMarcFileDoesntExist_generateFileAndSaveInS3(boolean isUtf, String postfix, String fileContent) {
try (var context = new FolioExecutionContextSetter(folioExecutionContext)) {
var filePath = "mod-data-export/download/4a090b0f-9da3-40f1-ab17-33d6a1e3abae-%s/4a090b0f-9da3-40f1-ab17-33d6a1e3abae-%s.mrc".formatted(postfix, postfix);
var filePath = "mod-data-export/download/71717177-f243-4e4a-bf1c-9e1e62b3171d%s/71717177-f243-4e4a-bf1c-9e1e62b3171d%s.mrc".formatted(postfix, postfix);
var expectedResult = new ByteArrayResource(fileContent.getBytes());

var actualResult = downloadRecordService.processRecordDownload(LOCAL_AUTHORITY_UUID, isUtf, postfix, "AUTHORITY");
var actualResult = downloadRecordService.processRecordDownload(INSTANCE_UUID, isUtf, postfix, INSTANCE_ID_TYPE);

assertEquals(expectedResult, actualResult);
assertEquals(filePath, s3Client.list(filePath).get(0));
}
}

@ParameterizedTest
@MethodSource("providedData")
@MethodSource("providedAuthorityData")
void whenMarcFileExists_retrieveItFromS3(boolean isUtf, String postfix, String fileContent) {
try (var context = new FolioExecutionContextSetter(folioExecutionContext)) {
var filePath = "mod-data-export/download/4a090b0f-9da3-40f1-ab17-33d6a1e3abae-%s/4a090b0f-9da3-40f1-ab17-33d6a1e3abae-%s.mrc".formatted(postfix, postfix);
var filePath = "mod-data-export/download/4a090b0f-9da3-40f1-ab17-33d6a1e3abae%s/4a090b0f-9da3-40f1-ab17-33d6a1e3abae%s.mrc".formatted(postfix, postfix);
s3Client.write(filePath, new BufferedInputStream(new ByteArrayInputStream(fileContent.getBytes())));
var expectedResult = new ByteArrayResource(fileContent.getBytes());

var actualResult = downloadRecordService.processAuthorityDownload(LOCAL_AUTHORITY_UUID, isUtf, postfix);
var actualResult = downloadRecordService.processRecordDownload(AUTHORITY_UUID, isUtf, postfix, AUTHORITY_ID_TYPE);

assertEquals(expectedResult, actualResult);
assertEquals(filePath, s3Client.list(filePath).get(0));
}
}

private static Stream<Arguments> providedData() {
private static Stream<Arguments> providedAuthorityData() {
return Stream.of(
Arguments.of(true, "-utf", "00237cam a2200073 i 4500001001400000008004100014373002900055999007900084\u001Ein00000001098\u001E210701t20222022nyua c 001 0 eng d\u001E \u001Faπανεπιστήμιο\u001Eff\u001Fs17eed93e-f9e2-4cb2-a52b-e9155acfc119\u001Fi4a090b0f-9da3-40f1-ab17-33d6a1e3abae\u001E\u001D"),
Arguments.of(false,"-marc8", "00235cam 2200073 i 4500001001400000008004100014373002700055999007900082\u001Ein00000001098\u001E210701t20222022nyua c 001 0 eng d\u001E \u001Fa\u001B(Ssapfslvx\"jolr\u001B(B\u001B(B\u001Eff\u001Fs17eed93e-f9e2-4cb2-a52b-e9155acfc119\u001Fi4a090b0f-9da3-40f1-ab17-33d6a1e3abae\u001E\u001D")
);
}

private static Stream<Arguments> providedInstanceData() {
return Stream.of(
Arguments.of(true, "utf", "00237cam a2200073 i 4500001001400000008004100014373002900055999007900084\u001Ein00000001098\u001E210701t20222022nyua c 001 0 eng d\u001E \u001Faπανεπιστήμιο\u001Eff\u001Fs17eed93e-f9e2-4cb2-a52b-e9155acfc119\u001Fi4a090b0f-9da3-40f1-ab17-33d6a1e3abae\u001E\u001D"),
Arguments.of(false,"marc8", "00235cam 2200073 i 4500001001400000008004100014373002700055999007900082\u001Ein00000001098\u001E210701t20222022nyua c 001 0 eng d\u001E \u001Fa\u001B(Ssapfslvx\"jolr\u001B(B\u001B(B\u001Eff\u001Fs17eed93e-f9e2-4cb2-a52b-e9155acfc119\u001Fi4a090b0f-9da3-40f1-ab17-33d6a1e3abae\u001E\u001D")
Arguments.of(true, "-utf", "00221cam a2200061 i 4500001001400000008006600014999007900080\u001Ein00000001098\u001E210701t20222022nyua c 001 0 eng d πανεπιστήμιο\u001Eff\u001Fs7171713e-f9e2-4cb2-a52b-e9155acfc119\u001Fi71717177-f243-4e4a-bf1c-9e1e62b3171d\u001E\u001D"),
Arguments.of(false,"-marc8", "00219cam 2200061 i 4500001001400000008006400014999007900078\u001Ein00000001098\u001E210701t20222022nyua c 001 0 eng d \u001B(Ssapfslvx\"jolr\u001B(B\u001B(B\u001Eff\u001Fs7171713e-f9e2-4cb2-a52b-e9155acfc119\u001Fi71717177-f243-4e4a-bf1c-9e1e62b3171d\u001E\u001D")
);
}
}
10 changes: 10 additions & 0 deletions src/test/java/org/folio/dataexp/service/export/strategies/AbstractExportStrategyTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -283,5 +283,15 @@ Map<UUID,MarcFields> getAdditionalMarcFieldsByExternalId(List<MarcRecordEntity>
protected LocalStorageWriter createLocalStorageWrite(JobExecutionExportFilesEntity exportFilesEntity) {
return localStorageWriter;
}

@Override
public MarcRecordEntity getMarcRecord(UUID externalId) {
throw new UnsupportedOperationException("The functionality is not required for testing.");
}

@Override
public MappingProfile getDefaultMappingProfile() {
throw new UnsupportedOperationException("The functionality is not required for testing.");
}
}
}
Loading

0 comments on commit 8796ac0

Please sign in to comment.