From cb2947053d721b6984c9d84169293383e9bdc547 Mon Sep 17 00:00:00 2001 From: Olivier Lamothe Date: Thu, 1 Jul 2021 09:09:27 -0400 Subject: [PATCH 1/5] feat: add support for batch update documents https://coveord.atlassian.net/browse/CDX-432 --- .../com/coveo/pushapiclient/BatchUpdate.java | 13 ++++++ .../coveo/pushapiclient/FileContainer.java | 9 ++++ .../coveo/pushapiclient/PlatformClient.java | 42 ++++++++++++++++--- .../java/com/coveo/pushapiclient/Source.java | 9 ++++ .../com/coveo/testlocally/TestingLocally.java | 14 ++++++- 5 files changed, 80 insertions(+), 7 deletions(-) create mode 100644 src/main/java/com/coveo/pushapiclient/BatchUpdate.java create mode 100644 src/main/java/com/coveo/pushapiclient/FileContainer.java diff --git a/src/main/java/com/coveo/pushapiclient/BatchUpdate.java b/src/main/java/com/coveo/pushapiclient/BatchUpdate.java new file mode 100644 index 00000000..cfe560e5 --- /dev/null +++ b/src/main/java/com/coveo/pushapiclient/BatchUpdate.java @@ -0,0 +1,13 @@ +package com.coveo.pushapiclient; + +import java.util.ArrayList; + +public class BatchUpdate { + private final ArrayList addOrUpdate; + private final ArrayList delete; + + public BatchUpdate(ArrayList addOrUpdate, ArrayList delete) { + this.addOrUpdate = addOrUpdate; + this.delete = delete; + } +} diff --git a/src/main/java/com/coveo/pushapiclient/FileContainer.java b/src/main/java/com/coveo/pushapiclient/FileContainer.java new file mode 100644 index 00000000..dcb0ea56 --- /dev/null +++ b/src/main/java/com/coveo/pushapiclient/FileContainer.java @@ -0,0 +1,9 @@ +package com.coveo.pushapiclient; + +import java.util.HashMap; + +public class FileContainer { + public String uploadUri; + public String fileId; + public HashMap requiredHeaders; +} diff --git a/src/main/java/com/coveo/pushapiclient/PlatformClient.java b/src/main/java/com/coveo/pushapiclient/PlatformClient.java index 3f6535ef..3188f76c 100644 --- a/src/main/java/com/coveo/pushapiclient/PlatformClient.java +++ b/src/main/java/com/coveo/pushapiclient/PlatformClient.java @@ -130,16 +130,46 @@ public void deleteDocument(String sourceId, String documentId, Boolean deleteChi // TODO } - public void createFileContainer() { - // TODO + public HttpResponse createFileContainer() throws IOException, InterruptedException { + String[] headers = this.getHeaders(this.getAuthorizationHeader(), this.getContentTypeApplicationJSONHeader()); + URI uri = URI.create(this.getBasePushURL() + "/files"); + + HttpRequest request = HttpRequest.newBuilder() + .headers(headers) + .uri(uri) + .POST(HttpRequest.BodyPublishers.ofString("")) + .build(); + + return this.httpClient.send(request, HttpResponse.BodyHandlers.ofString()); } - public void uploadContentToFileContainer(String sourceId, Object fileContainer) { - // TODO + public HttpResponse uploadContentToFileContainer(String sourceId, FileContainer fileContainer, BatchUpdate batchUpdate) throws IOException, InterruptedException { + String[] headers = fileContainer.requiredHeaders.entrySet() + .stream() + .flatMap(entry -> Stream.of(entry.getKey(), entry.getValue())) + .toArray(String[]::new); + URI uri = URI.create(fileContainer.uploadUri); + + HttpRequest request = HttpRequest.newBuilder() + .headers(headers) + .uri(uri) + .PUT(HttpRequest.BodyPublishers.ofString(new Gson().toJson(batchUpdate))) + .build(); + + return this.httpClient.send(request, HttpResponse.BodyHandlers.ofString()); } - public void pushFileContainerContent(String sourceId, Object fileContainer) { - // TODO + public HttpResponse pushFileContainerContent(String sourceId, FileContainer fileContainer) throws IOException, InterruptedException { + String[] headers = this.getHeaders(this.getAuthorizationHeader(), this.getContentTypeApplicationJSONHeader()); + URI uri = URI.create(this.getBasePushURL() + String.format("/sources/%s/documents/batch?fileId=%s", sourceId, fileContainer.fileId)); + + HttpRequest request = HttpRequest.newBuilder() + .headers(headers) + .uri(uri) + .PUT(HttpRequest.BodyPublishers.ofString("")) + .build(); + + return this.httpClient.send(request, HttpResponse.BodyHandlers.ofString()); } private String getBaseSourceURL() { diff --git a/src/main/java/com/coveo/pushapiclient/Source.java b/src/main/java/com/coveo/pushapiclient/Source.java index 84813148..b972c625 100644 --- a/src/main/java/com/coveo/pushapiclient/Source.java +++ b/src/main/java/com/coveo/pushapiclient/Source.java @@ -1,5 +1,7 @@ package com.coveo.pushapiclient; +import com.google.gson.Gson; + import java.io.IOException; import java.net.http.HttpResponse; @@ -37,4 +39,11 @@ public HttpResponse manageSecurityIdentities(String securityProviderId, public HttpResponse addOrUpdateDocument(String sourceId, DocumentBuilder docBuilder) throws IOException, InterruptedException { return this.platformClient.pushDocument(sourceId, docBuilder.marshal(), docBuilder.getDocument().uri); } + + public HttpResponse batchUpdateDocuments(String sourceId, BatchUpdate batchUpdate) throws IOException, InterruptedException { + HttpResponse resFileContainer = this.platformClient.createFileContainer(); + FileContainer fileContainer = new Gson().fromJson(resFileContainer.body(), FileContainer.class); + this.platformClient.uploadContentToFileContainer(sourceId, fileContainer, batchUpdate); + return this.platformClient.pushFileContainerContent(sourceId, fileContainer); + } } diff --git a/src/main/java/com/coveo/testlocally/TestingLocally.java b/src/main/java/com/coveo/testlocally/TestingLocally.java index 650f0582..ec2d0fce 100644 --- a/src/main/java/com/coveo/testlocally/TestingLocally.java +++ b/src/main/java/com/coveo/testlocally/TestingLocally.java @@ -6,6 +6,7 @@ import java.io.IOException; import java.net.http.HttpResponse; +import java.util.ArrayList; import java.util.Date; import java.util.HashMap; @@ -28,9 +29,20 @@ public static void main(String[] args) { public static void testPushDocument(String sourceId, Source source) { DocumentBuilder doc = new DocumentBuilder("https://perdu.com", "the title").withData("this is searchable").withDate(new Date()); - System.out.println(doc.marshal()); + + ArrayList docToAdd = new ArrayList<>(); + ArrayList docToRemove = new ArrayList<>(); + for (int i = 0; i < 10; i++) { + docToAdd.add(new DocumentBuilder(String.format("https://perdu.com/%s", i), String.format("the title %s", i)).withData(String.format("this is searchable %s", i))); + } + for (int i = 10; i < 20; i++) { + docToRemove.add(new DocumentBuilder(String.format("https://perdu.com/%s", i), String.format("the title %s", i)).withData(String.format("this is searchable %s", i))); + } + + try { source.addOrUpdateDocument(sourceId, doc); + source.batchUpdateDocuments(sourceId, new BatchUpdate(docToAdd, docToRemove)); } catch (IOException | InterruptedException e) { System.out.println(e); } From 3efac090c870519c4df875d60cfc58de92b9d165 Mon Sep 17 00:00:00 2001 From: Olivier Lamothe Date: Thu, 1 Jul 2021 09:37:39 -0400 Subject: [PATCH 2/5] use marshal to json element --- .../com/coveo/pushapiclient/BatchUpdate.java | 17 +++++++++++++---- .../coveo/pushapiclient/BatchUpdateRecord.java | 6 ++++++ .../coveo/pushapiclient/DocumentBuilder.java | 8 ++++++-- .../com/coveo/pushapiclient/PlatformClient.java | 3 ++- .../java/com/coveo/pushapiclient/Source.java | 2 +- .../com/coveo/testlocally/TestingLocally.java | 3 +-- 6 files changed, 29 insertions(+), 10 deletions(-) create mode 100644 src/main/java/com/coveo/pushapiclient/BatchUpdateRecord.java diff --git a/src/main/java/com/coveo/pushapiclient/BatchUpdate.java b/src/main/java/com/coveo/pushapiclient/BatchUpdate.java index cfe560e5..e16559b5 100644 --- a/src/main/java/com/coveo/pushapiclient/BatchUpdate.java +++ b/src/main/java/com/coveo/pushapiclient/BatchUpdate.java @@ -1,13 +1,22 @@ package com.coveo.pushapiclient; -import java.util.ArrayList; +import com.google.gson.JsonObject; + +import java.util.List; public class BatchUpdate { - private final ArrayList addOrUpdate; - private final ArrayList delete; + private List addOrUpdate; + private List delete; - public BatchUpdate(ArrayList addOrUpdate, ArrayList delete) { + public BatchUpdate(List addOrUpdate, List delete) { this.addOrUpdate = addOrUpdate; this.delete = delete; } + + public BatchUpdateRecord marshal() { + return new BatchUpdateRecord( + this.addOrUpdate.stream().map(documentBuilder -> documentBuilder.marshalJsonObject()).toArray(JsonObject[]::new), + this.delete.stream().map(documentBuilder -> documentBuilder.marshalJsonObject()).toArray(JsonObject[]::new) + ); + } } diff --git a/src/main/java/com/coveo/pushapiclient/BatchUpdateRecord.java b/src/main/java/com/coveo/pushapiclient/BatchUpdateRecord.java new file mode 100644 index 00000000..e2d0031b --- /dev/null +++ b/src/main/java/com/coveo/pushapiclient/BatchUpdateRecord.java @@ -0,0 +1,6 @@ +package com.coveo.pushapiclient; + +import com.google.gson.JsonObject; + +public record BatchUpdateRecord(JsonObject[] addOrUpdate, JsonObject[] delete) { +} diff --git a/src/main/java/com/coveo/pushapiclient/DocumentBuilder.java b/src/main/java/com/coveo/pushapiclient/DocumentBuilder.java index edda89ab..85aab58c 100644 --- a/src/main/java/com/coveo/pushapiclient/DocumentBuilder.java +++ b/src/main/java/com/coveo/pushapiclient/DocumentBuilder.java @@ -147,13 +147,17 @@ public DocumentBuilder withDeniedPermissions() { } public String marshal() { + return this.marshalJsonObject().toString(); + } + + public JsonObject marshalJsonObject() { JsonObject jsonDocument = new Gson().toJsonTree(this.document).getAsJsonObject(); this.document.metadata.forEach((key, value) -> { jsonDocument.add(key, new Gson().toJsonTree(value)); }); jsonDocument.remove("metadata"); - return jsonDocument.toString(); - + jsonDocument.addProperty("documentId", this.document.uri); + return jsonDocument; } private String dateFormat(DateTime dt) { diff --git a/src/main/java/com/coveo/pushapiclient/PlatformClient.java b/src/main/java/com/coveo/pushapiclient/PlatformClient.java index 3188f76c..5261a22a 100644 --- a/src/main/java/com/coveo/pushapiclient/PlatformClient.java +++ b/src/main/java/com/coveo/pushapiclient/PlatformClient.java @@ -143,13 +143,14 @@ public HttpResponse createFileContainer() throws IOException, Interrupte return this.httpClient.send(request, HttpResponse.BodyHandlers.ofString()); } - public HttpResponse uploadContentToFileContainer(String sourceId, FileContainer fileContainer, BatchUpdate batchUpdate) throws IOException, InterruptedException { + public HttpResponse uploadContentToFileContainer(String sourceId, FileContainer fileContainer, BatchUpdateRecord batchUpdate) throws IOException, InterruptedException { String[] headers = fileContainer.requiredHeaders.entrySet() .stream() .flatMap(entry -> Stream.of(entry.getKey(), entry.getValue())) .toArray(String[]::new); URI uri = URI.create(fileContainer.uploadUri); + HttpRequest request = HttpRequest.newBuilder() .headers(headers) .uri(uri) diff --git a/src/main/java/com/coveo/pushapiclient/Source.java b/src/main/java/com/coveo/pushapiclient/Source.java index b972c625..4adbb5b3 100644 --- a/src/main/java/com/coveo/pushapiclient/Source.java +++ b/src/main/java/com/coveo/pushapiclient/Source.java @@ -43,7 +43,7 @@ public HttpResponse addOrUpdateDocument(String sourceId, DocumentBuilder public HttpResponse batchUpdateDocuments(String sourceId, BatchUpdate batchUpdate) throws IOException, InterruptedException { HttpResponse resFileContainer = this.platformClient.createFileContainer(); FileContainer fileContainer = new Gson().fromJson(resFileContainer.body(), FileContainer.class); - this.platformClient.uploadContentToFileContainer(sourceId, fileContainer, batchUpdate); + this.platformClient.uploadContentToFileContainer(sourceId, fileContainer, batchUpdate.marshal()); return this.platformClient.pushFileContainerContent(sourceId, fileContainer); } } diff --git a/src/main/java/com/coveo/testlocally/TestingLocally.java b/src/main/java/com/coveo/testlocally/TestingLocally.java index 8adf9e0b..75f77491 100644 --- a/src/main/java/com/coveo/testlocally/TestingLocally.java +++ b/src/main/java/com/coveo/testlocally/TestingLocally.java @@ -36,8 +36,6 @@ public static void testPushDocument(String sourceId, Source source) { put("my_field_3", 1234); put("my_field_4", new String[]{"a", "b", "c"}); }}); - System.out.println(doc.marshal()); - System.out.println(docWithMetadata.marshal()); ArrayList docToAdd = new ArrayList<>(); ArrayList docToRemove = new ArrayList<>(); @@ -52,6 +50,7 @@ public static void testPushDocument(String sourceId, Source source) { try { source.addOrUpdateDocument(sourceId, doc); source.addOrUpdateDocument(sourceId, docWithMetadata); + source.batchUpdateDocuments(sourceId, new BatchUpdate(docToAdd, docToRemove)); } catch (IOException | InterruptedException e) { System.out.println(e); } From 096b9c1b054261d9274a60d44f07cbbfa0ee142d Mon Sep 17 00:00:00 2001 From: Olivier Lamothe Date: Thu, 1 Jul 2021 09:51:45 -0400 Subject: [PATCH 3/5] use map for FileContainer --- src/main/java/com/coveo/pushapiclient/FileContainer.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/coveo/pushapiclient/FileContainer.java b/src/main/java/com/coveo/pushapiclient/FileContainer.java index dcb0ea56..4d07bdee 100644 --- a/src/main/java/com/coveo/pushapiclient/FileContainer.java +++ b/src/main/java/com/coveo/pushapiclient/FileContainer.java @@ -1,9 +1,9 @@ package com.coveo.pushapiclient; -import java.util.HashMap; +import java.util.Map; public class FileContainer { public String uploadUri; public String fileId; - public HashMap requiredHeaders; + public Map requiredHeaders; } From 06d46cf1db137d0a3d47d255b555aed1eb72f106 Mon Sep 17 00:00:00 2001 From: Olivier Lamothe Date: Thu, 1 Jul 2021 10:00:44 -0400 Subject: [PATCH 4/5] use record --- src/main/java/com/coveo/pushapiclient/BatchUpdate.java | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/src/main/java/com/coveo/pushapiclient/BatchUpdate.java b/src/main/java/com/coveo/pushapiclient/BatchUpdate.java index e16559b5..abeb0e63 100644 --- a/src/main/java/com/coveo/pushapiclient/BatchUpdate.java +++ b/src/main/java/com/coveo/pushapiclient/BatchUpdate.java @@ -4,15 +4,7 @@ import java.util.List; -public class BatchUpdate { - private List addOrUpdate; - private List delete; - - public BatchUpdate(List addOrUpdate, List delete) { - this.addOrUpdate = addOrUpdate; - this.delete = delete; - } - +public record BatchUpdate(List addOrUpdate, List delete) { public BatchUpdateRecord marshal() { return new BatchUpdateRecord( this.addOrUpdate.stream().map(documentBuilder -> documentBuilder.marshalJsonObject()).toArray(JsonObject[]::new), From 74a1b9f3b8bb7e39b307e7d69297a48eef2bc6aa Mon Sep 17 00:00:00 2001 From: Olivier Lamothe Date: Mon, 5 Jul 2021 13:15:41 -0400 Subject: [PATCH 5/5] merge --- src/main/java/com/coveo/testlocally/TestingLocally.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/main/java/com/coveo/testlocally/TestingLocally.java b/src/main/java/com/coveo/testlocally/TestingLocally.java index 0747c4cc..d0d60318 100644 --- a/src/main/java/com/coveo/testlocally/TestingLocally.java +++ b/src/main/java/com/coveo/testlocally/TestingLocally.java @@ -51,8 +51,6 @@ public static void testPushDocument(String sourceId, Source source) { docToRemove.add(new DocumentBuilder(String.format("https://perdu.com/%s", i), String.format("the title %s", i)).withData(String.format("this is searchable %s", i))); } - - try { source.addOrUpdateDocument(sourceId, simpleDoc); source.addOrUpdateDocument(sourceId, docWithMetadata);