Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
36 commits
Select commit Hold shift + click to select a range
e0296ef
create `PushSource` and `catalogSource` classes
y-lakhdar May 19, 2023
419d177
add boilerplate for stream service
y-lakhdar May 19, 2023
4e8528c
Merge branch 'main' of github.com:coveo/push-api-client.java into LEN…
y-lakhdar May 19, 2023
1c5adec
revert master merge
y-lakhdar May 19, 2023
f5a2557
add required POST calls
y-lakhdar May 19, 2023
673b76a
update streamService
y-lakhdar May 19, 2023
5ef6772
apply ApiUrl corrections
y-lakhdar May 23, 2023
93c9867
Merge branch 'LEN-839' of github.com:coveo/push-api-client.java into …
y-lakhdar May 23, 2023
8e1e3e7
document StreamService
y-lakhdar May 23, 2023
de36933
Merge branch 'main' of github.com:coveo/push-api-client.java into LEN…
y-lakhdar May 25, 2023
7f9ad5e
Update src/main/java/com/coveo/pushapiclient/StreamService.java
y-lakhdar May 26, 2023
3499404
Update src/main/java/com/coveo/pushapiclient/StreamService.java
y-lakhdar May 26, 2023
f3ec845
Apply suggestions from code review
y-lakhdar May 26, 2023
508564e
refactor unit tests
y-lakhdar May 26, 2023
954ad44
Merge branch 'LEN-838' of github.com:coveo/push-api-client.java into …
y-lakhdar May 26, 2023
0e9acac
Merge branch 'main' into LEN-838
y-lakhdar May 29, 2023
b121952
create batch update accumulator
y-lakhdar May 29, 2023
476a09d
rework queue
y-lakhdar May 29, 2023
141889d
complete unit tests
y-lakhdar May 30, 2023
aa68240
remove comments
y-lakhdar May 30, 2023
369525a
update comment
y-lakhdar May 30, 2023
7d5dc2c
draft PushService
y-lakhdar May 30, 2023
c90446c
apply corrections
y-lakhdar May 30, 2023
cd43eb9
Merge branch 'LEN-838' of github.com:coveo/push-api-client.java into …
y-lakhdar May 30, 2023
9a7c88b
Merge branch 'LEN-838' of github.com:coveo/push-api-client.java into …
y-lakhdar May 30, 2023
cad5aac
add gitignore
y-lakhdar May 30, 2023
bd5ab59
Merge branch 'LENS-856' of github.com:coveo/push-api-client.java into…
y-lakhdar May 30, 2023
0b5baa1
fix merge conflict
y-lakhdar May 30, 2023
ce594de
Merge branch 'LENS-856' of github.com:coveo/push-api-client.java into…
y-lakhdar May 30, 2023
40a73dd
add unit tests
y-lakhdar May 30, 2023
b5bbee0
Apply suggestions from code review
y-lakhdar Jun 2, 2023
4d6ff14
Merge branch 'main' of github.com:coveo/push-api-client.java into LEN…
y-lakhdar Jun 2, 2023
018998b
Merge branch 'LENS-837' of github.com:coveo/push-api-client.java into…
y-lakhdar Jun 2, 2023
e429d3e
applying corrections
y-lakhdar Jun 2, 2023
d50d170
Merge branch 'main' into LENS-837
y-lakhdar Jun 8, 2023
cce0ca3
Merge branch 'main' into LENS-837
y-lakhdar Jun 8, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 6 additions & 10 deletions src/main/java/com/coveo/pushapiclient/DocumentUploadQueue.java
Original file line number Diff line number Diff line change
Expand Up @@ -61,18 +61,16 @@ public void add(DocumentBuilder document) throws IOException, InterruptedExcepti
if (this.size + sizeOfDoc >= this.maxQueueSize) {
this.flush();
}
if (document != null) {
documentToAddList.add(document);
this.size += sizeOfDoc;
}
documentToAddList.add(document);
this.size += sizeOfDoc;
}

/**
* Adds a {@link DeleteDocument} to the upload queue and flushes the queue if
* Adds the {@link DeleteDocument} to the upload queue and flushes the queue if
* it exceeds the maximum content length.
* See {@link DocumentUploadQueue#flush}.
*
* @param document The document to be delete from the index.
* @param document The document to be deleted from the index.
* @throws IOException If an I/O error occurs during the upload.
* @throws InterruptedException If the upload process is interrupted.
*/
Expand All @@ -85,10 +83,8 @@ public void add(DeleteDocument document) throws IOException, InterruptedExceptio
if (this.size + sizeOfDoc >= this.maxQueueSize) {
this.flush();
}
if (document != null) {
documentToDeleteList.add(document);
this.size += sizeOfDoc;
}
documentToDeleteList.add(document);
this.size += sizeOfDoc;
}

public BatchUpdate getBatch() {
Expand Down
51 changes: 51 additions & 0 deletions src/main/java/com/coveo/pushapiclient/PushService.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
package com.coveo.pushapiclient;

import java.io.IOException;
import java.net.http.HttpResponse;

import com.google.gson.Gson;

public class PushService {
private final PushEnabledSource source;
private final PlatformClient platformClient;
private PushServiceInternal service;

public PushService(PushEnabledSource source) {
String apiKey = source.getApiKey();
String organizationId = source.getOrganizationId();
PlatformUrl platformUrl = source.getPlatformUrl();
UploadStrategy uploader = this.getUploadStrategy();
DocumentUploadQueue queue = new DocumentUploadQueue(uploader);

this.platformClient = new PlatformClient(apiKey, organizationId, platformUrl);
this.service = new PushServiceInternal(queue);
this.source = source;
}

public void addOrUpdate(DocumentBuilder document) throws IOException, InterruptedException {
// TODO: LENS-843: include partial document updates
this.service.addOrUpdate(document);
}

public void delete(DeleteDocument document) throws IOException, InterruptedException {
this.service.delete(document);
}

public void close() throws IOException, InterruptedException {
this.service.close();
}

private UploadStrategy getUploadStrategy() {
return (batchUpdate) -> {
String sourceId = this.getSourceId();
HttpResponse<String> resFileContainer = this.platformClient.createFileContainer();
FileContainer fileContainer = new Gson().fromJson(resFileContainer.body(), FileContainer.class);
this.platformClient.uploadContentToFileContainer(fileContainer, new Gson().toJson(batchUpdate.marshal()));
return this.platformClient.pushFileContainerContent(sourceId, fileContainer);
};
}

private String getSourceId() {
return this.source.getId();
}
}
24 changes: 24 additions & 0 deletions src/main/java/com/coveo/pushapiclient/PushServiceInternal.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
package com.coveo.pushapiclient;

import java.io.IOException;

public class PushServiceInternal {
private DocumentUploadQueue queue;

public PushServiceInternal(DocumentUploadQueue queue) {
this.queue = queue;
}

public void addOrUpdate(DocumentBuilder document) throws IOException, InterruptedException {
this.queue.add(document);
}

public void delete(DeleteDocument document) throws IOException, InterruptedException {
this.queue.add(document);
}

public void close() throws IOException, InterruptedException {
queue.flush();
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -104,10 +104,17 @@ public void testShouldReturnBatch() throws IOException, InterruptedException {
}

@Test
public void testFlushShouldNotUploadDocumentaWhenRequiredSizeIsNotMet() throws IOException, InterruptedException {
public void testFlushShouldNotUploadDocumentsWhenRequiredSizeIsNotMet() throws IOException, InterruptedException {
// Adding 2MB document to the queue => queue has now 3MB of free space
// (5MB - 2MB = 3MB)
queue.add(documentToAdd);
// Adding 2MB document to the queue => queue has now 1MB of free space
// (3MB - 2MB = 1MB)
queue.add(documentToDelete);

// The maximum queue size has not been reached yet (1MB left of free space).
// Therefore, the accumulated documents will not be automatically flushed.
// Unless the user runs `.flush()` the queue will keep the 4MB of documents
verify(uploadStrategy, times(0)).apply(any(BatchUpdate.class));
}

Expand All @@ -127,10 +134,11 @@ public void testShouldAutomaticallyFlushAccumulatedDocuments() throws IOExceptio

// Adding 3 documents of 2MB to the queue. After adding the first 2 documents,
// the queue size will reach 6MB, which exceeds the maximum queue size
// limit. Therefore, the 2 first added documents will automatically be uploaded
// to the source.
// limit by 1MB. Therefore, the 2 first added documents will automatically be
// uploaded to the source.
queue.add(firstBulkyDocument);
queue.add(secondBulkyDocument);
verify(uploadStrategy, times(0)).apply(any(BatchUpdate.class));

// The 3rd document added to the queue will be included in a separate batch,
// which will not be uploaded unless the `flush()` method is called or until the
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,75 @@
package com.coveo.pushapiclient;

import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.MockitoAnnotations;

import com.coveo.pushapiclient.exceptions.NoOpenStreamException;

import java.io.IOException;
import java.net.http.HttpResponse;

import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;

public class PushServiceInternalTest {
@Mock
private DocumentUploadQueue queue;

@InjectMocks
private PushServiceInternal service;

@Mock
private HttpResponse<String> httpResponse;

private AutoCloseable closeable;
private DocumentBuilder documentA;
private DocumentBuilder documentB;
private DeleteDocument documentC;

@Before
public void setUp() throws Exception {
documentA = new DocumentBuilder("https://my.document.uri?ref=1", "My first document title");
documentB = new DocumentBuilder("https://my.document.uri?ref=2", "My second document title");
documentC = new DeleteDocument("https://my.document.uri?ref=3");

closeable = MockitoAnnotations.openMocks(this);

}

@After
public void closeService() throws Exception {
closeable.close();
}

@Test
public void testShouldAddNewDocumentToQueue() throws IOException, InterruptedException {
service.addOrUpdate(documentA);
service.addOrUpdate(documentB);

verify(this.queue, times(1)).add(documentA);
verify(this.queue, times(1)).add(documentB);
}

@Test
public void testAddShouldAddDocumentToDeleteToQueue() throws IOException, InterruptedException {
service.delete(documentC);

verify(queue, times(1)).add(documentC);
}

@Test
public void testCloseShouldFlushBufferedDocuments()
throws IOException, InterruptedException, NoOpenStreamException {
service.addOrUpdate(documentA);
service.addOrUpdate(documentB);
service.delete(documentC);
service.close();

verify(queue, times(1)).flush();
}

}