Skip to content

Commit 00d99d8

Browse files
diasfdiasf
authored
Closes #2504: Endpoint to run datacite harvester (#2522)
* Closes #2507: Datacite DOI harvester * Refs #2507: Minor adjustments * Closes #2504: Endpoint to run datacite harvester * review comments
1 parent 289e25c commit 00d99d8

File tree

4 files changed

+228
-14
lines changed

4 files changed

+228
-14
lines changed

‎dataverse-webapp/src/main/java/edu/harvard/iq/dataverse/api/HarvestingClients.java

+30-14
Original file line numberDiff line numberDiff line change
@@ -8,18 +8,21 @@
88
import edu.harvard.iq.dataverse.harvest.client.HarvesterParams;
99
import edu.harvard.iq.dataverse.harvest.client.HarvesterServiceBean;
1010
import edu.harvard.iq.dataverse.harvest.client.HarvestingClientDao;
11+
import edu.harvard.iq.dataverse.harvest.client.HarvestingClientsService;
1112
import edu.harvard.iq.dataverse.persistence.dataverse.Dataverse;
1213
import edu.harvard.iq.dataverse.persistence.harvest.HarvestingClient;
1314
import edu.harvard.iq.dataverse.persistence.user.AuthenticatedUser;
1415
import edu.harvard.iq.dataverse.util.json.JsonParseException;
1516
import io.vavr.control.Option;
17+
import io.vavr.control.Try;
1618

1719
import javax.ejb.EJB;
1820
import javax.ejb.Stateless;
1921
import javax.json.Json;
2022
import javax.json.JsonArrayBuilder;
2123
import javax.json.JsonObject;
2224
import javax.json.JsonObjectBuilder;
25+
import javax.ws.rs.DELETE;
2326
import javax.ws.rs.GET;
2427
import javax.ws.rs.POST;
2528
import javax.ws.rs.PUT;
@@ -29,7 +32,6 @@
2932
import javax.ws.rs.core.Response;
3033
import java.io.IOException;
3134
import java.io.StringReader;
32-
import java.util.ArrayList;
3335
import java.util.List;
3436
import java.util.logging.Logger;
3537

@@ -45,7 +47,9 @@ public class HarvestingClients extends AbstractApiBean {
4547
@EJB
4648
HarvesterServiceBean harvesterService;
4749
@EJB
48-
HarvestingClientDao harvestingClientService;
50+
HarvestingClientDao harvestingClientDao;
51+
@EJB
52+
HarvestingClientsService harvestingClientsService;
4953

5054
private static final Logger logger = Logger.getLogger(HarvestingClients.class.getName());
5155

@@ -65,7 +69,7 @@ public Response harvestingClients(@QueryParam("key") String apiKey) throws Wrapp
6569

6670
List<HarvestingClient> harvestingClients;
6771
try {
68-
harvestingClients = harvestingClientService.getAllHarvestingClients();
72+
harvestingClients = harvestingClientDao.getAllHarvestingClients();
6973
} catch (Exception ex) {
7074
return error(Response.Status.INTERNAL_SERVER_ERROR, "Caught an exception looking up configured harvesting clients; " + ex.getMessage());
7175
}
@@ -89,7 +93,7 @@ public Response harvestingClient(@PathParam("nickName") String nickName, @QueryP
8993

9094
HarvestingClient harvestingClient;
9195
try {
92-
harvestingClient = harvestingClientService.findByNickname(nickName);
96+
harvestingClient = harvestingClientDao.findByNickname(nickName);
9397
} catch (Exception ex) {
9498
logger.warning("Exception caught looking up harvesting client " + nickName + ": " + ex.getMessage());
9599
return error(Response.Status.BAD_REQUEST, "Internal error: failed to look up harvesting client " + nickName + ".");
@@ -129,10 +133,6 @@ public Response createHarvestingClient(String jsonBody, @PathParam("nickName") S
129133
}
130134

131135
harvestingClient.setDataverse(ownerDataverse);
132-
if (ownerDataverse.getHarvestingClientConfigs() == null) {
133-
ownerDataverse.setHarvestingClientConfigs(new ArrayList<>());
134-
}
135-
ownerDataverse.getHarvestingClientConfigs().add(harvestingClient);
136136

137137
DataverseRequest req = createDataverseRequest(superuser);
138138
HarvestingClient managedHarvestingClient = execCommand(new CreateHarvestingClientCommand(req, harvestingClient));
@@ -148,6 +148,21 @@ public Response createHarvestingClient(String jsonBody, @PathParam("nickName") S
148148

149149
}
150150

151+
@DELETE
152+
@ApiWriteOperation
153+
@Path("{nickName}")
154+
public Response deleteHarvestingClient(@PathParam("nickName") String nickName, @QueryParam("key") String apiKey) throws WrappedResponse {
155+
156+
findSuperuserOrDie();
157+
158+
HarvestingClient harvestingClient = Try.of(() -> harvestingClientDao.findByNickname(nickName))
159+
.getOrElseThrow(() -> new WrappedResponse(notFound("Harvesting client " + nickName + " not found.")));
160+
161+
harvestingClientDao.setDeleteInProgress(harvestingClient.getId());
162+
harvestingClientsService.deleteClient(harvestingClient);
163+
return accepted();
164+
}
165+
151166
@PUT
152167
@ApiWriteOperation
153168
@Path("{nickName}")
@@ -157,7 +172,7 @@ public Response modifyHarvestingClient(String jsonBody, @PathParam("nickName") S
157172

158173
HarvestingClient harvestingClient;
159174
try {
160-
harvestingClient = harvestingClientService.findByNickname(nickName);
175+
harvestingClient = harvestingClientDao.findByNickname(nickName);
161176
} catch (Exception ex) {
162177
// We don't care what happened; we'll just assume we couldn't find it.
163178
harvestingClient = null;
@@ -200,24 +215,25 @@ public Response modifyHarvestingClient(String jsonBody, @PathParam("nickName") S
200215
@POST
201216
@ApiWriteOperation
202217
@Path("{nickName}/run")
203-
public Response startHarvestingJob(@PathParam("nickName") String clientNickname, @QueryParam("key") String apiKey) throws WrappedResponse {
218+
public Response startHarvestingJob(String paramsJson, @PathParam("nickName") String clientNickname, @QueryParam("key") String apiKey) throws WrappedResponse {
204219

205-
AuthenticatedUser superuser = findSuperuserOrDie();
220+
AuthenticatedUser superuser = findSuperuserOrDie();
206221

207222
try {
208-
HarvestingClient harvestingClient = harvestingClientService.findByNickname(clientNickname);
223+
HarvestingClient harvestingClient = harvestingClientDao.findByNickname(clientNickname);
209224

210225
if (harvestingClient == null) {
211226
return error(Response.Status.NOT_FOUND, "No such dataverse: " + clientNickname);
212227
}
213228

214229
DataverseRequest dataverseRequest = createDataverseRequest(superuser);
215-
harvesterService.doAsyncHarvest(dataverseRequest, harvestingClient, HarvesterParams.empty());
230+
HarvesterParams params = harvesterService.parseParams(harvestingClient, paramsJson);
231+
harvesterService.doAsyncHarvest(dataverseRequest, harvestingClient, params);
216232

217233
} catch (Exception e) {
218234
return error(Response.Status.BAD_REQUEST, "Exception thrown when running harvesting client\"" + clientNickname + "\" via REST API; " + e.getMessage());
219235
}
220-
return this.accepted();
236+
return accepted();
221237
}
222238

223239
public static JsonObjectBuilder harvestingConfigAsJson(HarvestingClient harvestingConfig) {

‎dataverse-webapp/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvesterServiceBean.java

+10
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,12 @@
11
package edu.harvard.iq.dataverse.harvest.client;
22

3+
import com.google.gson.Gson;
34
import edu.harvard.iq.dataverse.DatasetDao;
45
import edu.harvard.iq.dataverse.engine.command.DataverseRequest;
56
import edu.harvard.iq.dataverse.persistence.dataverse.Dataverse;
67
import edu.harvard.iq.dataverse.persistence.harvest.HarvestType;
78
import edu.harvard.iq.dataverse.persistence.harvest.HarvestingClient;
9+
import org.apache.commons.lang.StringUtils;
810

911
import javax.annotation.PostConstruct;
1012
import javax.ejb.Asynchronous;
@@ -132,6 +134,14 @@ public <T extends HarvesterParams> void doHarvest(DataverseRequest dataverseRequ
132134
}
133135
}
134136

137+
public HarvesterParams parseParams(HarvestingClient client, String paramsJson) {
138+
if (StringUtils.isBlank(paramsJson)) {
139+
return HarvesterParams.empty();
140+
}
141+
142+
return new Gson().fromJson(paramsJson, resolveHarvester(client).getParamsClass());
143+
}
144+
135145
// -------------------- PRIVATE --------------------
136146

137147
// TODO: I doubt we need a full stacktrace in the harvest log - ??

‎dataverse-webapp/src/test/java/edu/harvard/iq/dataverse/harvest/client/DataciteHarvesterParamsTest.java

+65
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
package edu.harvard.iq.dataverse.harvest.client;
2+
3+
import com.google.gson.Gson;
4+
import org.junit.Test;
5+
6+
import static org.assertj.core.api.Assertions.assertThat;
7+
8+
public class DataciteHarvesterParamsTest {
9+
10+
@Test
11+
public void parseJson__empty() {
12+
// given
13+
String json = "{}";
14+
15+
// when
16+
DataciteHarvesterParams params = new Gson().fromJson(json, DataciteHarvesterParams.class);
17+
18+
// then
19+
assertThat(params.getDoiImport()).isEmpty();
20+
}
21+
22+
@Test
23+
public void parseJson__emptyArray() {
24+
// given
25+
String json = "{ \"doiImport\": [] }";
26+
27+
// when
28+
DataciteHarvesterParams params = new Gson().fromJson(json, DataciteHarvesterParams.class);
29+
30+
// then
31+
assertThat(params.getDoiImport()).isEmpty();
32+
}
33+
34+
@Test
35+
public void parseJson__array_of_doi_parts() {
36+
// given
37+
String json = "{ \"doiImport\": [ {\"authority\": \"10.5072\", \"id\": \"FK2/BYM3IW\"}, {\"authority\": \"1902.1\", \"id\": \"111012\"} ] }";
38+
39+
// when
40+
DataciteHarvesterParams params = new Gson().fromJson(json, DataciteHarvesterParams.class);
41+
42+
// then
43+
assertThat(params.getDoiImport()).hasSize(2);
44+
}
45+
46+
@Test
47+
public void parseJson__class() {
48+
// given
49+
String json = "{ \"doiImport\": [ {\"authority\": \"10.5072\", \"id\": \"FK2/BYM3IW\"}, {\"authority\": \"1902.1\", \"id\": \"111012\"} ] }";
50+
51+
// when
52+
HarvesterParams params = new Gson().fromJson(json, getParams());
53+
54+
// then
55+
assertThat(params).isInstanceOf(DataciteHarvesterParams.class);
56+
}
57+
58+
private Class<HarvesterParams> getParams() {
59+
return getSpecific();
60+
}
61+
62+
private <T extends HarvesterParams> Class<T> getSpecific() {
63+
return (Class<T>) DataciteHarvesterParams.class;
64+
}
65+
}

‎dataverse-webapp/src/test/java/edu/harvard/iq/dataverse/harvest/client/HarvesterParamsTest.java

+123
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,123 @@
1+
package edu.harvard.iq.dataverse.harvest.client;
2+
3+
import com.google.common.collect.ImmutableMap;
4+
import com.google.gson.Gson;
5+
import org.assertj.core.api.InstanceOfAssertFactories;
6+
import org.junit.Test;
7+
8+
import java.util.Map;
9+
10+
import static org.assertj.core.api.Assertions.assertThat;
11+
12+
public class HarvesterParamsTest {
13+
14+
private final Map<String, Class<?>> paramClasses = ImmutableMap.<String, Class<?>>builder()
15+
.put("Test1", HarvesterParamsTest1.class)
16+
.put("Test2", HarvesterParamsTest2.class)
17+
.put("Empty", HarvesterParams.EmptyHarvesterParams.class)
18+
.build();
19+
20+
@Test
21+
public void parseJson() {
22+
// given
23+
String json = "{ \"param1Test1\": \"val1\", \"param2Test1\": \"val2\"}";
24+
25+
// when
26+
HarvesterParams params = new Gson().fromJson(json, getParams("Test1"));
27+
28+
// then
29+
assertThat(params).isInstanceOf(HarvesterParamsTest1.class);
30+
assertThat(params).asInstanceOf(InstanceOfAssertFactories.type(HarvesterParamsTest1.class))
31+
.satisfies(paramsTest1 -> {
32+
assertThat(paramsTest1.getParam1Test1()).isEqualTo("val1");
33+
assertThat(paramsTest1.getParam2Test1()).isEqualTo("val2");
34+
});
35+
}
36+
37+
@Test
38+
public void parseJson__wrong_class() {
39+
// given
40+
String json = "{ \"param1Test1\": \"val1\", \"param2Test1\": \"val2\"}";
41+
42+
// when
43+
HarvesterParams params = new Gson().fromJson(json, getParams("Test2"));
44+
45+
// then
46+
assertThat(params).isInstanceOf(HarvesterParamsTest2.class);
47+
assertThat(params).asInstanceOf(InstanceOfAssertFactories.type(HarvesterParamsTest2.class))
48+
.satisfies(paramsTest2 -> {
49+
assertThat(paramsTest2.getParam1Test2()).isNull();
50+
assertThat(paramsTest2.getParam2Test2()).isNull();
51+
});
52+
}
53+
54+
@Test
55+
public void parseJson__emptyJson() {
56+
// given
57+
String json = "{}";
58+
59+
// when
60+
HarvesterParams params = new Gson().fromJson(json, getParams("Empty"));
61+
62+
// then
63+
assertThat(params).isInstanceOf(HarvesterParams.EmptyHarvesterParams.class);
64+
}
65+
66+
@Test
67+
public void parseJson__emptyString() {
68+
// given
69+
String json = "";
70+
71+
// when
72+
HarvesterParams params = new Gson().fromJson(json, getParams("Empty"));
73+
74+
// then
75+
assertThat(params).isNull();
76+
}
77+
78+
@Test
79+
public void parseJson__null() {
80+
// given
81+
String json = null;
82+
83+
// when
84+
HarvesterParams params = new Gson().fromJson(json, getParams("Empty"));
85+
86+
// then
87+
assertThat(params).isNull();
88+
}
89+
90+
private Class<HarvesterParams> getParams(String type) {
91+
return getSpecific(type);
92+
}
93+
94+
private <T extends HarvesterParams> Class<T> getSpecific(String type) {
95+
return (Class<T>) paramClasses.get(type);
96+
}
97+
98+
public static class HarvesterParamsTest1 extends HarvesterParams {
99+
private String param1Test1;
100+
private String param2Test1;
101+
102+
public String getParam1Test1() {
103+
return param1Test1;
104+
}
105+
106+
public String getParam2Test1() {
107+
return param2Test1;
108+
}
109+
}
110+
111+
public static class HarvesterParamsTest2 extends HarvesterParams {
112+
private String param1Test2;
113+
private String param2Test2;
114+
115+
public String getParam1Test2() {
116+
return param1Test2;
117+
}
118+
119+
public String getParam2Test2() {
120+
return param2Test2;
121+
}
122+
}
123+
}

0 commit comments

Comments
 (0)