Closes #2506: Removal of harvested DOI (#2523)

* Closes #2507: Datacite DOI harvester

* Refs #2507: Minor adjustments

* Closes #2504: Endpoint to run datacite harvester

* Closes #2506: Removal of harvested DOI

* review comments

* merge conflicts

* merge conflicts

Author: diasf

dataverse-webapp/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportServiceBean.java

@@ -7,6 +7,7 @@
 import edu.harvard.iq.dataverse.api.dto.DatasetDTO;
 import edu.harvard.iq.dataverse.engine.command.DataverseRequest;
 import edu.harvard.iq.dataverse.engine.command.impl.CreateHarvestedDatasetCommand;
+import edu.harvard.iq.dataverse.engine.command.impl.DeleteDatasetCommand;
 import edu.harvard.iq.dataverse.engine.command.impl.DestroyDatasetCommand;
 import edu.harvard.iq.dataverse.persistence.datafile.DataFile;
 import edu.harvard.iq.dataverse.persistence.dataset.Dataset;
@@ -112,6 +113,30 @@ public Dataset doImportHarvestedDataset(DataverseRequest dataverseRequest, Harve
         }
     }
 
+    @TransactionAttribute(REQUIRES_NEW)
+    public void doDeleteHarvestedDataset(DataverseRequest request, HarvestingClient harvestingClient, String identifier) throws ImportException {
+        Dataset dataset = datasetDao.getDatasetByHarvestInfo(harvestingClient.getDataverse(), identifier);
+        if (dataset != null) {
+            // Purge all the SOLR documents associated with this client from the
+            // index server:
+            indexService.deleteHarvestedDocuments(dataset);
+
+            // files from harvested datasets are removed unceremoniously,
+            // directly in the database. no need to bother calling the
+            // DeleteFileCommand on them.
+            for (DataFile harvestedFile : dataset.getFiles()) {
+                DataFile merged = em.merge(harvestedFile);
+                em.remove(merged);
+            }
+
+            dataset.setFiles(null);
+            Dataset merged = em.merge(dataset);
+            engineSvc.submit(new DeleteDatasetCommand(request, merged));
+        } else {
+            throw new ImportException("No dataset found for " + identifier + ", skipping delete. ");
+        }
+    }
+
     private Dataset importDatasetDTOJson(DataverseRequest dataverseRequest, HarvestingClient harvestingClient, String harvestIdentifier, String json) throws ImportException {
         try {
             Dataset ds = harvestedJsonParser.parseDataset(json);

dataverse-webapp/src/main/java/edu/harvard/iq/dataverse/harvest/client/DataciteDOIHarvester.java

@@ -18,7 +18,6 @@
 @LocalBean
 public class DataciteDOIHarvester implements Harvester<DataciteHarvesterParams> {
 
-
     @Inject
     private DataCiteRestApiClient dataCiteRestApiClient;
 
@@ -42,7 +41,7 @@ public Class<DataciteHarvesterParams> getParamsClass() {
 
     @Override
     public HarvesterResult harvest(DataverseRequest dataverseRequest, HarvestingClient harvestingClient, Logger hdLogger, DataciteHarvesterParams params) throws ImportException {
-        if (params.getDoiImport().isEmpty()) {
+        if (params.getDoiImport().isEmpty() && params.getDoiRemove().isEmpty()) {
             throw new ImportException("Missing DOI's");
         }
 
@@ -52,24 +51,36 @@ public HarvesterResult harvest(DataverseRequest dataverseRequest, HarvestingClie
             importDOI(rs, dataverseRequest, harvestingClient, hdLogger, doi);
         }
 
+        for (DataciteHarvesterParams.DOIValue doi: params.getDoiRemove()) {
+            removeDOI(rs, dataverseRequest, harvestingClient, hdLogger, doi);
+        }
+
         return rs;
     }
 
     // -------------------- PRIVATE --------------------
 
-    private void importDOI(HarvesterResult rs, DataverseRequest dataverseRequest, HarvestingClient harvestingClient, Logger hdLogger, DataciteHarvesterParams.DOIValue doi) throws ImportException {
+    private void importDOI(HarvesterResult rs, DataverseRequest dataverseRequest, HarvestingClient harvestingClient, Logger hdLogger, DataciteHarvesterParams.DOIValue doi) {
         try {
             DatasetDTO dto = dataciteDatasetMapper.toDataset(dataCiteRestApiClient.findDoi(doi.getAuthority(), doi.getId()));
             importService.doImportHarvestedDataset(dataverseRequest, harvestingClient, doi.getFull(), dto);
             rs.incrementHarvested();
         } catch (Exception e) {
             rs.incrementFailed();
-            String errorMessage = "Failed to import DOI "
+            hdLogger.log(Level.SEVERE, "Failed to import DOI (" + doi.getFull() + ") "
                     + harvestingClient.getName()
                     + "; "
-                    + e.getMessage();
-            hdLogger.log(Level.SEVERE, errorMessage);
-            throw new ImportException(errorMessage, e);
+                    + e.getMessage());
+        }
+    }
+
+    private void removeDOI(HarvesterResult rs, DataverseRequest dataverseRequest, HarvestingClient harvestingClient, Logger hdLogger, DataciteHarvesterParams.DOIValue doi) {
+        try {
+            importService.doDeleteHarvestedDataset(dataverseRequest, harvestingClient, doi.getFull());
+            rs.incrementDeleted();
+        } catch (Exception e) {
+            rs.incrementFailed();
+            hdLogger.log(Level.SEVERE, "Failed to delete DOI " + doi.getFull() + " (" + harvestingClient.getName() + "): " + e.getMessage());
         }
     }
 }

dataverse-webapp/src/main/java/edu/harvard/iq/dataverse/harvest/client/DataciteHarvesterParams.java

@@ -1,6 +1,5 @@
 package edu.harvard.iq.dataverse.harvest.client;
 
-import com.fasterxml.jackson.annotation.JsonCreator;
 import io.vavr.control.Option;
 
 import java.util.Collections;
@@ -15,18 +14,28 @@ public class DataciteHarvesterParams extends HarvesterParams {
 
     private List<DOIValue> doiImport;
 
+    private List<DOIValue> doiRemove;
+
     // -------------------- GETTERS --------------------
 
     public List<DOIValue> getDoiImport() {
         return Option.of(doiImport).getOrElse(Collections.emptyList());
     }
 
+    public List<DOIValue> getDoiRemove() {
+        return Option.of(doiRemove).getOrElse(Collections.emptyList());
+    }
+
     // -------------------- SETTERS --------------------
 
     public void setDoiImport(List<DOIValue> doiImport) {
         this.doiImport = doiImport;
     }
 
+    public void setDoiRemove(List<DOIValue> doiRemove) {
+        this.doiRemove = doiRemove;
+    }
+
     // -------------------- INNER CLASSES --------------------
 
     static public class DOIValue {

dataverse-webapp/src/main/java/edu/harvard/iq/dataverse/harvest/client/OAIHarvester.java

@@ -1,23 +1,14 @@
 package edu.harvard.iq.dataverse.harvest.client;
 
-import edu.harvard.iq.dataverse.DatasetDao;
-import edu.harvard.iq.dataverse.EjbDataverseEngine;
 import edu.harvard.iq.dataverse.api.imports.HarvestImporterType;
 import edu.harvard.iq.dataverse.api.imports.HarvestImporterTypeResolver;
 import edu.harvard.iq.dataverse.api.imports.ImportException;
 import edu.harvard.iq.dataverse.api.imports.ImportServiceBean;
 import edu.harvard.iq.dataverse.engine.command.DataverseRequest;
-import edu.harvard.iq.dataverse.engine.command.exception.CommandException;
-import edu.harvard.iq.dataverse.engine.command.exception.IllegalCommandException;
-import edu.harvard.iq.dataverse.engine.command.exception.PermissionException;
-import edu.harvard.iq.dataverse.engine.command.impl.DeleteDatasetCommand;
 import edu.harvard.iq.dataverse.harvest.client.oai.OaiHandler;
 import edu.harvard.iq.dataverse.harvest.client.oai.OaiHandlerException;
-import edu.harvard.iq.dataverse.persistence.datafile.DataFile;
-import edu.harvard.iq.dataverse.persistence.dataset.Dataset;
 import edu.harvard.iq.dataverse.persistence.harvest.HarvestType;
 import edu.harvard.iq.dataverse.persistence.harvest.HarvestingClient;
-import edu.harvard.iq.dataverse.search.index.IndexServiceBean;
 import org.apache.commons.io.FileUtils;
 import org.dspace.xoai.model.oaipmh.Header;
 import org.dspace.xoai.serviceprovider.exceptions.HarvestException;
@@ -46,21 +37,12 @@ public class OAIHarvester implements Harvester<HarvesterParams.EmptyHarvesterPar
     @PersistenceContext(unitName = "VDCNet-ejbPU")
     private EntityManager em;
 
-    @EJB
-    DatasetDao datasetDao;
-
     @EJB
     ImportServiceBean importService;
 
-    @EJB
-    IndexServiceBean indexService;
-
     @EJB
     private HarvestImporterTypeResolver harvestImporterTypeResolver;
 
-    @EJB
-    EjbDataverseEngine engineService;
-
     // -------------------- LOGIC --------------------
 
     @Override
@@ -136,16 +118,9 @@ private void processRecord(HarvesterResult result, DataverseRequest dataverseReq
             if (record.isDeleted()) {
                 hdLogger.info("Deleting harvesting dataset for " + identifier + ", per the OAI server's instructions.");
 
-                Dataset dataset = datasetDao.getDatasetByHarvestInfo(oaiHandler.getHarvestingClient().getDataverse(), identifier);
-                if (dataset != null) {
-                    result.incrementDeleted();
-                    hdLogger.info("Deleting dataset " + dataset.getGlobalIdString());
-                    deleteHarvestedDataset(dataset, dataverseRequest, hdLogger);
-                    // TODO:
-                    // check the status of that Delete - see if it actually succeeded
-                } else {
-                    hdLogger.info("No dataset found for " + identifier + ", skipping delete. ");
-                }
+                importService.doDeleteHarvestedDataset(dataverseRequest, oaiHandler.getHarvestingClient(), identifier);
+                result.incrementDeleted();
+
             } else {
                 hdLogger.info("Successfully retrieved GetRecord response.");
                 HarvestImporterType importType = harvestImporterTypeResolver.resolveImporterType(oaiHandler.getMetadataFormat())
@@ -175,35 +150,6 @@ private void processRecord(HarvesterResult result, DataverseRequest dataverseReq
         }
     }
 
-    private void deleteHarvestedDataset(Dataset dataset, DataverseRequest request, Logger hdLogger) {
-        // Purge all the SOLR documents associated with this client from the
-        // index server:
-        indexService.deleteHarvestedDocuments(dataset);
-
-        try {
-            // files from harvested datasets are removed unceremoniously,
-            // directly in the database. no need to bother calling the
-            // DeleteFileCommand on them.
-            for (DataFile harvestedFile : dataset.getFiles()) {
-                DataFile merged = em.merge(harvestedFile);
-                em.remove(merged);
-                harvestedFile = null;
-            }
-            dataset.setFiles(null);
-            Dataset merged = em.merge(dataset);
-            engineService.submit(new DeleteDatasetCommand(request, merged));
-        } catch (IllegalCommandException ex) {
-            // TODO: log the result
-        } catch (PermissionException ex) {
-            // TODO: log the result
-        } catch (CommandException ex) {
-            // TODO: log the result
-        }
-
-        // TODO: log the success result
-    }
-
-
     private void logBeginOaiHarvest(Logger hdLogger, HarvestingClient harvestingClient) {
         hdLogger.log(Level.INFO, "BEGIN HARVEST, oaiUrl="
                 + harvestingClient.getHarvestingUrl()