Closes #2507: Datacite DOI harvester

Author: diasf

dataverse-persistence/src/main/java/edu/harvard/iq/dataverse/common/DatasetFieldConstant.java

@@ -39,6 +39,9 @@ public class DatasetFieldConstant {
     public final static String title = "title";
     public final static String subTitle = "subtitle"; //SEK 6-7-2016 to match what is in DB
     public final static String alternativeTitle = "alternativeTitle"; //missing from class
+    public final static String titleTranslation = "titleTranslation";
+    public final static String titleTranslationText = "titleTranslationText";
+    public final static String titleTranslationLanguage = "titleTranslationLanguage";
     public final static String datasetId = "datasetId";
     public final static String authorName = "authorName";
     public final static String authorAffiliation = "authorAffiliation";

dataverse-persistence/src/main/java/edu/harvard/iq/dataverse/persistence/dataset/Dataset.java

@@ -11,6 +11,7 @@
 import edu.harvard.iq.dataverse.persistence.datafile.license.FileTermsOfUse;
 import edu.harvard.iq.dataverse.persistence.dataverse.link.DatasetLinkingDataverse;
 import edu.harvard.iq.dataverse.persistence.guestbook.Guestbook;
+import edu.harvard.iq.dataverse.persistence.harvest.HarvestStyle;
 import edu.harvard.iq.dataverse.persistence.harvest.HarvestingClient;
 import io.vavr.control.Option;
 import org.apache.commons.lang3.StringUtils;
@@ -615,22 +616,24 @@ public String getRemoteArchiveURL() {
         if (!isHarvested()) {
             return null;
         }
-        if (HarvestingClient.HARVEST_STYLE_DATAVERSE.equals(getHarvestedFrom().getHarvestStyle())) {
+        if (HarvestStyle.DATAVERSE.equals(getHarvestedFrom().getHarvestStyle())) {
             return getHarvestedFrom().getArchiveUrl() + "/dataset.xhtml?persistentId=" + getGlobalIdString();
-        } else if (HarvestingClient.HARVEST_STYLE_VDC.equals(getHarvestedFrom().getHarvestStyle())) {
+        } else if (HarvestStyle.VDC.equals(getHarvestedFrom().getHarvestStyle())) {
             String rootArchiveUrl = getHarvestedFrom().getHarvestingUrl();
             int c = rootArchiveUrl.indexOf("/OAIHandler");
             return c > 0
                     ? rootArchiveUrl.substring(0, c) + "/faces/study/StudyPage.xhtml?globalId=" + getGlobalIdString()
                     : null;
-        } else if (HarvestingClient.HARVEST_STYLE_ICPSR.equals(getHarvestedFrom().getHarvestStyle())) {
+        } else if (HarvestStyle.ICPSR.equals(getHarvestedFrom().getHarvestStyle())) {
             // For the ICPSR, it turns out that the best thing to do is to
             // rely on the DOI to send the user to the right landing page for
             // the study:
             //String icpsrId = identifier;
             //return getOwner().getHarvestingClient().getArchiveUrl() + "/icpsrweb/ICPSR/studies/"+icpsrId+"?q="+icpsrId+"&searchSource=icpsr-landing";
             return "http://doi.org/" + getAuthority() + "/" + getIdentifier();
-        } else if (HarvestingClient.HARVEST_STYLE_NESSTAR.equals(getHarvestedFrom().getHarvestStyle())) {
+        } else if (HarvestStyle.DOI.equals(getHarvestedFrom().getHarvestStyle())) {
+            return getHarvestedFrom().getArchiveUrl() + "/" + getAuthority() + "/" + getIdentifier();
+        } else if (HarvestStyle.NESSTAR.equals(getHarvestedFrom().getHarvestStyle())) {
             String nServerURL = getHarvestedFrom().getArchiveUrl();
             // chop any trailing slashes in the server URL - or they will result
             // in multiple slashes in the final URL pointing to the study
@@ -641,9 +644,9 @@ public String getRemoteArchiveURL() {
             //SEK 09/13/18
             return nServerURL + "/webview/?mode=documentation&submode=abstract&studydoc=" + nServerURLencoded + "%2Fobj%2FfStudy%2F"
                     + getIdentifier() + "&top=yes";
-        } else if (HarvestingClient.HARVEST_STYLE_ROPER.equals(getHarvestedFrom().getHarvestStyle())) {
+        } else if (HarvestStyle.ROPER.equals(getHarvestedFrom().getHarvestStyle())) {
             return getHarvestedFrom().getArchiveUrl() + "/CFIDE/cf/action/catalog/abstract.cfm?archno=" + getIdentifier();
-        } else if (HarvestingClient.HARVEST_STYLE_HGL.equals(getHarvestedFrom().getHarvestStyle())) {
+        } else if (HarvestStyle.HGL.equals(getHarvestedFrom().getHarvestStyle())) {
             // a bit of a hack, true.
             // HGL documents, when turned into Dataverse studies/datasets
             // all 1 datafile; the location ("storage identifier") of the file

dataverse-persistence/src/main/java/edu/harvard/iq/dataverse/persistence/harvest/HarvestStyle.java

@@ -0,0 +1,26 @@
+package edu.harvard.iq.dataverse.persistence.harvest;
+
+/**
+ * Different harvesting "styles". These define how we format and
+ * display metadata harvested from various remote resources.
+ */
+public enum HarvestStyle {
+    DATAVERSE("Dataverse v4+"),
+    VDC("DVN, v2-3"),
+    ICPSR("ICPSR"),
+    NESSTAR("Nesstar archive"),
+    ROPER("Roper Archive"),
+    HGL("HGL"),
+    DOI("DOI"),
+    DEFAULT("Generic OAI resource (DC)");
+
+    final String description;
+
+    HarvestStyle(String description) {
+        this.description = description;
+    }
+
+    public String getDescription() {
+        return description;
+    }
+}

dataverse-persistence/src/main/java/edu/harvard/iq/dataverse/persistence/harvest/HarvestType.java

@@ -0,0 +1,5 @@
+package edu.harvard.iq.dataverse.persistence.harvest;
+
+public enum HarvestType {
+    OAI, DATACITE_DOI
+}

dataverse-persistence/src/main/java/edu/harvard/iq/dataverse/persistence/harvest/HarvestingClient.java

@@ -8,6 +8,8 @@
 import javax.persistence.CascadeType;
 import javax.persistence.Column;
 import javax.persistence.Entity;
+import javax.persistence.EnumType;
+import javax.persistence.Enumerated;
 import javax.persistence.GeneratedValue;
 import javax.persistence.GenerationType;
 import javax.persistence.Id;
@@ -23,13 +25,10 @@
 import javax.validation.constraints.Size;
 import java.io.Serializable;
 import java.text.SimpleDateFormat;
-import java.util.Arrays;
 import java.util.Calendar;
 import java.util.Date;
 import java.util.GregorianCalendar;
-import java.util.LinkedHashMap;
 import java.util.List;
-import java.util.Map;
 
 /**
  * @author Leonid Andreev
@@ -58,58 +57,14 @@ public void setId(Long id) {
         this.id = id;
     }
 
-    public static final String HARVEST_TYPE_OAI = "oai";
-    public static final String HARVEST_TYPE_NESSTAR = "nesstar";
-
-
-    /*
-     * Different harvesting "styles". These define how we format and
-     * display meatada harvested from various remote resources.
-     */
-    public static final String HARVEST_STYLE_DATAVERSE = "dataverse";
-    // pre-4.0 remote Dataverse:
-    public static final String HARVEST_STYLE_VDC = "vdc";
-    public static final String HARVEST_STYLE_ICPSR = "icpsr";
-    public static final String HARVEST_STYLE_NESSTAR = "nesstar";
-    public static final String HARVEST_STYLE_ROPER = "roper";
-    public static final String HARVEST_STYLE_HGL = "hgl";
-    public static final String HARVEST_STYLE_DEFAULT = "default";
-
-    public static final String HARVEST_STYLE_DESCRIPTION_DATAVERSE = "Dataverse v4+";
-    // pre-4.0 remote Dataverse:
-    public static final String HARVEST_STYLE_DESCRIPTION_VDC = "DVN, v2-3";
-    public static final String HARVEST_STYLE_DESCRIPTION_ICPSR = "ICPSR";
-    public static final String HARVEST_STYLE_DESCRIPTION_NESSTAR = "Nesstar archive";
-    public static final String HARVEST_STYLE_DESCRIPTION_ROPER = "Roper Archive";
-    public static final String HARVEST_STYLE_DESCRIPTION_HGL = "HGL";
-    public static final String HARVEST_STYLE_DESCRIPTION_DEFAULT = "Generic OAI resource (DC)";
-
-
-    public static final List<String> HARVEST_STYLE_LIST = Arrays.asList(HARVEST_STYLE_DATAVERSE, HARVEST_STYLE_VDC, HARVEST_STYLE_ICPSR, HARVEST_STYLE_NESSTAR, HARVEST_STYLE_ROPER, HARVEST_STYLE_HGL, HARVEST_STYLE_DEFAULT);
-    public static final List<String> HARVEST_STYLE_DESCRIPTION_LIST = Arrays.asList(HARVEST_STYLE_DESCRIPTION_DATAVERSE, HARVEST_STYLE_DESCRIPTION_VDC, HARVEST_STYLE_DESCRIPTION_ICPSR, HARVEST_STYLE_DESCRIPTION_NESSTAR, HARVEST_STYLE_DESCRIPTION_ROPER, HARVEST_STYLE_DESCRIPTION_HGL, HARVEST_STYLE_DESCRIPTION_DEFAULT);
-
-    public static final Map<String, String> HARVEST_STYLE_INFOMAP = new LinkedHashMap<String, String>();
-
-    static {
-        for (int i = 0; i < HARVEST_STYLE_LIST.size(); i++) {
-            HARVEST_STYLE_INFOMAP.put(HARVEST_STYLE_LIST.get(i), HARVEST_STYLE_DESCRIPTION_LIST.get(i));
-        }
-    }
-
-
-    public static final String REMOTE_ARCHIVE_URL_LEVEL_DATAVERSE = "dataverse";
-    public static final String REMOTE_ARCHIVE_URL_LEVEL_DATASET = "dataset";
-    public static final String REMOTE_ARCHIVE_URL_LEVEL_FILE = "file";
-
     public static final String SCHEDULE_PERIOD_DAILY = "daily";
     public static final String SCHEDULE_PERIOD_WEEKLY = "weekly";
 
     public HarvestingClient() {
-        this.harvestType = HARVEST_TYPE_OAI; // default harvestType
-        this.harvestStyle = HARVEST_STYLE_DATAVERSE; // default harvestStyle
+        this.harvestType = HarvestType.OAI; // default harvestType
+        this.harvestStyle = HarvestStyle.DATAVERSE; // default harvestStyle
     }
 
-
     @ManyToOne
     @JoinColumn(name = "dataverse_id")
     private Dataverse dataverse;
@@ -148,27 +103,25 @@ public void setName(String name) {
         this.name = name;
     }
 
-    private String harvestType;
+    @Enumerated(EnumType.STRING)
+    private HarvestType harvestType;
 
-    public String getHarvestType() {
+    public HarvestType getHarvestType() {
         return harvestType;
     }
 
-    public void setHarvestType(String harvestType) {
+    public void setHarvestType(HarvestType harvestType) {
         this.harvestType = harvestType;
     }
 
-    public boolean isOai() {
-        return HARVEST_TYPE_OAI.equals(harvestType);
-    }
-
-    private String harvestStyle;
+    @Enumerated(EnumType.STRING)
+    private HarvestStyle harvestStyle;
 
-    public String getHarvestStyle() {
+    public HarvestStyle getHarvestStyle() {
         return harvestStyle;
     }
 
-    public void setHarvestStyle(String harvestStyle) {
+    public void setHarvestStyle(HarvestStyle harvestStyle) {
         this.harvestStyle = harvestStyle;
     }
 

dataverse-persistence/src/main/resources/db/migration/V76__harvest_client_enums.sql

@@ -0,0 +1,11 @@
+-- Harvest type enum
+UPDATE harvestingclient SET harvesttype = 'OAI' WHERE harvesttype ='oai';
+
+-- Harvest style enums
+UPDATE harvestingclient SET harveststyle = 'DATAVERSE' WHERE harveststyle = 'dataverse';
+UPDATE harvestingclient SET harveststyle = 'VDC' WHERE harveststyle = 'vdc';
+UPDATE harvestingclient SET harveststyle = 'ICPSR' WHERE harveststyle = 'icpsr';
+UPDATE harvestingclient SET harveststyle = 'NESSTAR' WHERE harveststyle = 'nesstar';
+UPDATE harvestingclient SET harveststyle = 'ROPER' WHERE harveststyle = 'roper';
+UPDATE harvestingclient SET harveststyle = 'HGL' WHERE harveststyle = 'hgl';
+UPDATE harvestingclient SET harveststyle = 'DEFAULT' WHERE harveststyle = 'default';

dataverse-persistence/src/test/resources/dbinit.sql

@@ -21,7 +21,7 @@ INSERT INTO builtinuser (id, encryptedpassword, passwordencryptionversion, usern
 -------------------- HARVESTING CLIENTS --------------------
 
 INSERT INTO harvestingclient(id, archiveurl, deleted, harveststyle, harvesttype, harvestingnow, harvestingset, harvestingurl, metadataprefix, name, scheduledayofweek, schedulehourofday, scheduleperiod, scheduled, dataverse_id)
-VALUES (12, '', false, '', '', false, '', '', '', 'harvest_client_name', 1, 1, '', false, 1);
+VALUES (12, '', false, 'DATAVERSE', 'OAI', false, '', '', '', 'harvest_client_name', 1, 1, '', false, 1);
 
 -------------------- DATAVERSES --------------------
 

dataverse-webapp/src/main/java/edu/harvard/iq/dataverse/api/HarvestingClients.java

@@ -5,12 +5,14 @@
 import edu.harvard.iq.dataverse.engine.command.DataverseRequest;
 import edu.harvard.iq.dataverse.engine.command.impl.CreateHarvestingClientCommand;
 import edu.harvard.iq.dataverse.engine.command.impl.UpdateHarvestingClientCommand;
+import edu.harvard.iq.dataverse.harvest.client.HarvesterParams;
 import edu.harvard.iq.dataverse.harvest.client.HarvesterServiceBean;
 import edu.harvard.iq.dataverse.harvest.client.HarvestingClientDao;
 import edu.harvard.iq.dataverse.persistence.dataverse.Dataverse;
 import edu.harvard.iq.dataverse.persistence.harvest.HarvestingClient;
 import edu.harvard.iq.dataverse.persistence.user.AuthenticatedUser;
 import edu.harvard.iq.dataverse.util.json.JsonParseException;
+import io.vavr.control.Option;
 
 import javax.ejb.EJB;
 import javax.ejb.Stateless;
@@ -210,7 +212,7 @@ public Response startHarvestingJob(@PathParam("nickName") String clientNickname,
             }
 
             DataverseRequest dataverseRequest = createDataverseRequest(superuser);
-            harvesterService.doAsyncHarvest(dataverseRequest, harvestingClient);
+            harvesterService.doAsyncHarvest(dataverseRequest, harvestingClient, HarvesterParams.empty());
 
         } catch (Exception e) {
             return error(Response.Status.BAD_REQUEST, "Exception thrown when running harvesting client\"" + clientNickname + "\" via REST API; " + e.getMessage());
@@ -226,7 +228,7 @@ public static JsonObjectBuilder harvestingConfigAsJson(HarvestingClient harvesti
 
         return jsonObjectBuilder().add("nickName", harvestingConfig.getName()).
                 add("dataverseAlias", harvestingConfig.getDataverse().getAlias()).
-                add("type", harvestingConfig.getHarvestType()).
+                add("type", Option.of(harvestingConfig.getHarvestType()).map(Enum::name).getOrNull()).
                 add("harvestUrl", harvestingConfig.getHarvestingUrl()).
                 add("archiveUrl", harvestingConfig.getArchiveUrl()).
                 add("metadataFormat", harvestingConfig.getMetadataPrefix()).