From 08e7cec030923464e25462e8e8ecb416173515b3 Mon Sep 17 00:00:00 2001 From: David Brooke <38883189+dmbrooke@users.noreply.github.com> Date: Thu, 31 Aug 2023 16:35:30 -0400 Subject: [PATCH 01/12] First set of changes --- .../java/com/coveo/pushapiclient/ApiCore.java | 36 +++++++++++++++- .../coveo/pushapiclient/PlatformClient.java | 41 ++++++++++++++++--- 2 files changed, 71 insertions(+), 6 deletions(-) diff --git a/src/main/java/com/coveo/pushapiclient/ApiCore.java b/src/main/java/com/coveo/pushapiclient/ApiCore.java index 50f575f6..8ad3072d 100644 --- a/src/main/java/com/coveo/pushapiclient/ApiCore.java +++ b/src/main/java/com/coveo/pushapiclient/ApiCore.java @@ -9,19 +9,53 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -// TODO: LENS-934 - Support throttling class ApiCore { private final HttpClient httpClient; private final Logger logger; + private final int retryAfter; + private final int maxRetries; public ApiCore() { this.httpClient = HttpClient.newHttpClient(); this.logger = LogManager.getLogger(ApiCore.class); + this.retryAfter = 5000; + this.maxRetries = 50; } public ApiCore(HttpClient httpClient, Logger logger) { this.httpClient = httpClient; this.logger = logger; + this.retryAfter = 5000; + this.maxRetries = 50; + } + + public ApiCore(HttpClient httpClient, Logger logger, int retryAfter, int maxRetries) { + this.httpClient = httpClient; + this.logger = logger; + this.retryAfter = retryAfter; + this.maxRetries = maxRetries; + } + + public HttpResponse callApiWithRetries( + URI uri, String[] headers, int timeMultiple) + throws Exception { + long delayInMilliseconds = retryAfter * 1000L; + int nbRetries = 0; + + while (true) { + HttpResponse response = this.post(uri, headers); + nbRetries++; + + if (response.statusCode() == 429 && nbRetries <= maxRetries) { + Thread.sleep(delayInMilliseconds); + delayInMilliseconds = delayInMilliseconds * timeMultiple; + } else { + if (response.statusCode() >= 400) { + throw new Exception("HTTP error " + response.statusCode() + " : " + response.body()); + } + return response; + } + } } public HttpResponse post(URI uri, String[] headers) diff --git a/src/main/java/com/coveo/pushapiclient/PlatformClient.java b/src/main/java/com/coveo/pushapiclient/PlatformClient.java index a70bbb06..caa08427 100644 --- a/src/main/java/com/coveo/pushapiclient/PlatformClient.java +++ b/src/main/java/com/coveo/pushapiclient/PlatformClient.java @@ -18,6 +18,8 @@ public class PlatformClient { private final String organizationId; private final ApiCore api; private final PlatformUrl platformUrl; + private final int retryAfter; + private final int maxRetries; /** * Construct a PlatformClient @@ -28,7 +30,19 @@ public class PlatformClient { * @param organizationId The Coveo Organization identifier. */ public PlatformClient(String apiKey, String organizationId) { - this(apiKey, organizationId, new PlatformUrlBuilder().build()); + this(apiKey, organizationId, new PlatformUrlBuilder().build(), 5000, 50); + } + + /** + * Construct a PlatformClient + * + * @param apiKey An apiKey capable of pushing documents and managing sources in a Coveo + * organization. + * @see Manage API Keys + * @param organizationId The Coveo Organization identifier. + */ + public PlatformClient(String apiKey, String organizationId, int retryAfter, int maxRetries) { + this(apiKey, organizationId, new PlatformUrlBuilder().build(), retryAfter, maxRetries); } /** @@ -40,11 +54,18 @@ public PlatformClient(String apiKey, String organizationId) { * @param organizationId The Coveo Organization identifier. * @param platformUrl The PlatformUrl. */ - public PlatformClient(String apiKey, String organizationId, PlatformUrl platformUrl) { + public PlatformClient( + String apiKey, + String organizationId, + PlatformUrl platformUrl, + int retryAfter, + int maxRetries) { this.apiKey = apiKey; this.organizationId = organizationId; this.api = new ApiCore(); this.platformUrl = platformUrl; + this.retryAfter = retryAfter; + this.maxRetries = maxRetries; } /** @@ -56,11 +77,14 @@ public PlatformClient(String apiKey, String organizationId, PlatformUrl platform * @param organizationId The Coveo Organization identifier. * @param httpClient The HttpClient. */ - public PlatformClient(String apiKey, String organizationId, HttpClient httpClient) { + public PlatformClient( + String apiKey, String organizationId, HttpClient httpClient, int retryAfter, int maxRetries) { this.apiKey = apiKey; this.organizationId = organizationId; - this.api = new ApiCore(httpClient, LogManager.getLogger(ApiCore.class)); + this.api = new ApiCore(httpClient, LogManager.getLogger(ApiCore.class), retryAfter, maxRetries); this.platformUrl = new PlatformUrlBuilder().build(); + this.retryAfter = retryAfter; + this.maxRetries = maxRetries; } /** @@ -73,11 +97,18 @@ public PlatformClient(String apiKey, String organizationId, HttpClient httpClien * @param environment The Environment to be used. */ @Deprecated - public PlatformClient(String apiKey, String organizationId, Environment environment) { + public PlatformClient( + String apiKey, + String organizationId, + Environment environment, + int retryAfter, + int maxRetries) { this.apiKey = apiKey; this.organizationId = organizationId; this.api = new ApiCore(); this.platformUrl = new PlatformUrlBuilder().withEnvironment(environment).build(); + this.retryAfter = retryAfter; + this.maxRetries = maxRetries; } /** From 85d10e4669a3de1e9de437ce7b0484e949b334a0 Mon Sep 17 00:00:00 2001 From: David Brooke <38883189+dmbrooke@users.noreply.github.com> Date: Thu, 31 Aug 2023 17:03:03 -0400 Subject: [PATCH 02/12] Use retry mechanism on all requests --- .../java/com/coveo/pushapiclient/ApiCore.java | 30 +++++++++---------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/src/main/java/com/coveo/pushapiclient/ApiCore.java b/src/main/java/com/coveo/pushapiclient/ApiCore.java index 8ad3072d..1cbe8ff5 100644 --- a/src/main/java/com/coveo/pushapiclient/ApiCore.java +++ b/src/main/java/com/coveo/pushapiclient/ApiCore.java @@ -37,13 +37,18 @@ public ApiCore(HttpClient httpClient, Logger logger, int retryAfter, int maxRetr } public HttpResponse callApiWithRetries( - URI uri, String[] headers, int timeMultiple) - throws Exception { + String method, URI uri, String[] headers, BodyPublisher body, int timeMultiple) + throws IOException, InterruptedException { long delayInMilliseconds = retryAfter * 1000L; int nbRetries = 0; while (true) { - HttpResponse response = this.post(uri, headers); + this.logger.debug(method.toUpperCase() + " " + uri); + HttpRequest request = + HttpRequest.newBuilder().headers(headers).uri(uri).method(method, body).build(); + HttpResponse response = + this.httpClient.send(request, HttpResponse.BodyHandlers.ofString()); + this.logResponse(response); nbRetries++; if (response.statusCode() == 429 && nbRetries <= maxRetries) { @@ -51,7 +56,8 @@ public HttpResponse callApiWithRetries( delayInMilliseconds = delayInMilliseconds * timeMultiple; } else { if (response.statusCode() >= 400) { - throw new Exception("HTTP error " + response.statusCode() + " : " + response.body()); + throw new InterruptedException( + "HTTP error " + response.statusCode() + " : " + response.body()); } return response; } @@ -66,9 +72,7 @@ public HttpResponse post(URI uri, String[] headers) public HttpResponse post(URI uri, String[] headers, BodyPublisher body) throws IOException, InterruptedException { this.logger.debug("POST " + uri); - HttpRequest request = HttpRequest.newBuilder().headers(headers).uri(uri).POST(body).build(); - HttpResponse response = - this.httpClient.send(request, HttpResponse.BodyHandlers.ofString()); + HttpResponse response = this.callApiWithRetries("post", uri, headers, body, 2); this.logResponse(response); return response; } @@ -76,9 +80,7 @@ public HttpResponse post(URI uri, String[] headers, BodyPublisher body) public HttpResponse put(URI uri, String[] headers, BodyPublisher body) throws IOException, InterruptedException { this.logger.debug("PUT " + uri); - HttpRequest request = HttpRequest.newBuilder().headers(headers).uri(uri).PUT(body).build(); - HttpResponse response = - this.httpClient.send(request, HttpResponse.BodyHandlers.ofString()); + HttpResponse response = this.callApiWithRetries("put", uri, headers, body, 2); this.logResponse(response); return response; } @@ -86,9 +88,8 @@ public HttpResponse put(URI uri, String[] headers, BodyPublisher body) public HttpResponse delete(URI uri, String[] headers) throws IOException, InterruptedException { this.logger.debug("DELETE " + uri); - HttpRequest request = HttpRequest.newBuilder().headers(headers).uri(uri).DELETE().build(); HttpResponse response = - this.httpClient.send(request, HttpResponse.BodyHandlers.ofString()); + this.callApiWithRetries("delete", uri, headers, HttpRequest.BodyPublishers.ofString(""), 2); this.logResponse(response); return response; } @@ -96,10 +97,7 @@ public HttpResponse delete(URI uri, String[] headers) public HttpResponse delete(URI uri, String[] headers, BodyPublisher body) throws IOException, InterruptedException { this.logger.debug("DELETE " + uri); - HttpRequest request = - HttpRequest.newBuilder().headers(headers).uri(uri).method("DELETE", body).build(); - HttpResponse response = - this.httpClient.send(request, HttpResponse.BodyHandlers.ofString()); + HttpResponse response = this.callApiWithRetries("delete", uri, headers, body, 2); this.logResponse(response); return response; } From 90990317dfb2fbd2ea3c1e85c57c72067a4162d3 Mon Sep 17 00:00:00 2001 From: David Brooke <38883189+dmbrooke@users.noreply.github.com> Date: Fri, 1 Sep 2023 09:45:06 -0400 Subject: [PATCH 03/12] More constructors, some docs comments --- .../java/com/coveo/pushapiclient/ApiCore.java | 12 ++-- .../coveo/pushapiclient/PlatformClient.java | 56 ++++++++++++++----- .../com/coveo/pushapiclient/PushSource.java | 44 +++++++++++++++ 3 files changed, 89 insertions(+), 23 deletions(-) diff --git a/src/main/java/com/coveo/pushapiclient/ApiCore.java b/src/main/java/com/coveo/pushapiclient/ApiCore.java index 1cbe8ff5..24d22c61 100644 --- a/src/main/java/com/coveo/pushapiclient/ApiCore.java +++ b/src/main/java/com/coveo/pushapiclient/ApiCore.java @@ -10,23 +10,19 @@ import org.apache.logging.log4j.Logger; class ApiCore { + public static final int DEFAULT_RETRY_AFTER = 5000; + public static final int DEFAULT_MAX_RETRIES = 50; private final HttpClient httpClient; private final Logger logger; private final int retryAfter; private final int maxRetries; public ApiCore() { - this.httpClient = HttpClient.newHttpClient(); - this.logger = LogManager.getLogger(ApiCore.class); - this.retryAfter = 5000; - this.maxRetries = 50; + this(HttpClient.newHttpClient(), LogManager.getLogger(ApiCore.class)); } public ApiCore(HttpClient httpClient, Logger logger) { - this.httpClient = httpClient; - this.logger = logger; - this.retryAfter = 5000; - this.maxRetries = 50; + this(httpClient, logger, DEFAULT_RETRY_AFTER, DEFAULT_MAX_RETRIES); } public ApiCore(HttpClient httpClient, Logger logger, int retryAfter, int maxRetries) { diff --git a/src/main/java/com/coveo/pushapiclient/PlatformClient.java b/src/main/java/com/coveo/pushapiclient/PlatformClient.java index caa08427..eb38341d 100644 --- a/src/main/java/com/coveo/pushapiclient/PlatformClient.java +++ b/src/main/java/com/coveo/pushapiclient/PlatformClient.java @@ -14,12 +14,13 @@ /** PlatformClient handles network requests to the Coveo platform */ public class PlatformClient { + public static final int DEFAULT_RETRY_AFTER = 5000; + public static final int DEFAULT_MAX_RETRIES = 50; + private final String apiKey; private final String organizationId; private final ApiCore api; private final PlatformUrl platformUrl; - private final int retryAfter; - private final int maxRetries; /** * Construct a PlatformClient @@ -30,7 +31,24 @@ public class PlatformClient { * @param organizationId The Coveo Organization identifier. */ public PlatformClient(String apiKey, String organizationId) { - this(apiKey, organizationId, new PlatformUrlBuilder().build(), 5000, 50); + this( + apiKey, + organizationId, + new PlatformUrlBuilder().build(), + DEFAULT_RETRY_AFTER, + DEFAULT_MAX_RETRIES); + } + + /** + * Construct a PlatformClient + * + * @param apiKey An apiKey capable of pushing documents and managing sources in a Coveo + * organization. + * @see Manage API Keys + * @param organizationId The Coveo Organization identifier. + */ + public PlatformClient(String apiKey, String organizationId, PlatformUrl platformUrl) { + this(apiKey, organizationId, platformUrl, DEFAULT_RETRY_AFTER, DEFAULT_MAX_RETRIES); } /** @@ -40,6 +58,8 @@ public PlatformClient(String apiKey, String organizationId) { * organization. * @see Manage API Keys * @param organizationId The Coveo Organization identifier. + * @param retryAfter The amount of time, in milliseconds, to wait between request attempts. + * @param maxRetries The maximum number of attempts to make for a request. */ public PlatformClient(String apiKey, String organizationId, int retryAfter, int maxRetries) { this(apiKey, organizationId, new PlatformUrlBuilder().build(), retryAfter, maxRetries); @@ -53,6 +73,8 @@ public PlatformClient(String apiKey, String organizationId, int retryAfter, int * @see Manage API Keys * @param organizationId The Coveo Organization identifier. * @param platformUrl The PlatformUrl. + * @param retryAfter The amount of time, in milliseconds, to wait between request attempts. + * @param maxRetries The maximum number of attempts to make for a request. */ public PlatformClient( String apiKey, @@ -64,8 +86,6 @@ public PlatformClient( this.organizationId = organizationId; this.api = new ApiCore(); this.platformUrl = platformUrl; - this.retryAfter = retryAfter; - this.maxRetries = maxRetries; } /** @@ -77,14 +97,27 @@ public PlatformClient( * @param organizationId The Coveo Organization identifier. * @param httpClient The HttpClient. */ + public PlatformClient(String apiKey, String organizationId, HttpClient httpClient) { + this(apiKey, organizationId, httpClient, DEFAULT_RETRY_AFTER, DEFAULT_MAX_RETRIES); + } + + /** + * Construct a PlatformClient + * + * @param apiKey An apiKey capable of pushing documents and managing sources in a Coveo + * organization. + * @see Manage API Keys + * @param organizationId The Coveo Organization identifier. + * @param httpClient The HttpClient. + * @param retryAfter The amount of time, in milliseconds, to wait between request attempts. + * @param maxRetries The maximum number of attempts to make for a request. + */ public PlatformClient( String apiKey, String organizationId, HttpClient httpClient, int retryAfter, int maxRetries) { this.apiKey = apiKey; this.organizationId = organizationId; this.api = new ApiCore(httpClient, LogManager.getLogger(ApiCore.class), retryAfter, maxRetries); this.platformUrl = new PlatformUrlBuilder().build(); - this.retryAfter = retryAfter; - this.maxRetries = maxRetries; } /** @@ -97,18 +130,11 @@ public PlatformClient( * @param environment The Environment to be used. */ @Deprecated - public PlatformClient( - String apiKey, - String organizationId, - Environment environment, - int retryAfter, - int maxRetries) { + public PlatformClient(String apiKey, String organizationId, Environment environment) { this.apiKey = apiKey; this.organizationId = organizationId; this.api = new ApiCore(); this.platformUrl = new PlatformUrlBuilder().withEnvironment(environment).build(); - this.retryAfter = retryAfter; - this.maxRetries = maxRetries; } /** diff --git a/src/main/java/com/coveo/pushapiclient/PushSource.java b/src/main/java/com/coveo/pushapiclient/PushSource.java index 1f3a8e28..a19ae72f 100644 --- a/src/main/java/com/coveo/pushapiclient/PushSource.java +++ b/src/main/java/com/coveo/pushapiclient/PushSource.java @@ -125,6 +125,37 @@ public static PushSource fromPlatformUrl( return new PushSource(apiKey, organizationId, sourceId, platformUrl); } + /** + * Create a Push source instance + * + * @param apiKey The API key used for all operations regarding your source. + *

Ensure your API key has the required privileges for the operation you will be performing + * * + *

For more information about which privileges are required, see Privilege Reference. + * @param organizationId The unique identifier of your organization. + *

The Organization Id can be retrieved in the URL of your Coveo organization. + * @param sourceId The unique identifier of the target Push source. + *

The Source Id can be retrieved when you edit your source in the Coveo Administration + * Console + * @param platformUrl The object containing additional information on the URL endpoint. You can + * use the {@link PlatformUrl} when your organization is located in a non-default Coveo + * environement and/or region. When not specified, the default platform URL values will be + * used: {@link PlatformUrl#DEFAULT_ENVIRONMENT} and {@link PlatformUrl#DEFAULT_REGION} + * @param retryAfter The amount of time, in milliseconds, to wait between request attempts. + * @param maxRetries The maximum number of attempts to make for a request. + */ + public static PushSource fromPlatformUrl( + String apiKey, + String organizationId, + String sourceId, + PlatformUrl platformUrl, + int retryAfter, + int maxRetries) { + return new PushSource(apiKey, organizationId, sourceId, platformUrl, retryAfter, maxRetries); + } + private PushSource( String apiKey, String organizationId, String sourceId, PlatformUrl platformUrl) { this.apiKey = apiKey; @@ -132,6 +163,19 @@ private PushSource( this.platformClient = new PlatformClient(apiKey, organizationId, platformUrl); } + private PushSource( + String apiKey, + String organizationId, + String sourceId, + PlatformUrl platformUrl, + int retryAfter, + int maxRetries) { + this.apiKey = apiKey; + this.urlExtractor = new ApiUrl(organizationId, sourceId, platformUrl); + this.platformClient = + new PlatformClient(apiKey, organizationId, platformUrl, retryAfter, maxRetries); + } + /** * Create or update a security identity. * From 3a10d3a50704248deaee187c901a75d548b0b0fb Mon Sep 17 00:00:00 2001 From: David Brooke <38883189+dmbrooke@users.noreply.github.com> Date: Fri, 1 Sep 2023 15:02:32 -0400 Subject: [PATCH 04/12] Actually leverage OOP :facepalm: --- .../java/com/coveo/pushapiclient/ApiCore.java | 52 ++++++++----------- .../coveo/pushapiclient/BackoffOptions.java | 29 +++++++++++ .../pushapiclient/BackoffOptionsBuilder.java | 27 ++++++++++ .../coveo/pushapiclient/PlatformClient.java | 24 +++------ .../com/coveo/pushapiclient/PushSource.java | 13 ++--- 5 files changed, 90 insertions(+), 55 deletions(-) create mode 100644 src/main/java/com/coveo/pushapiclient/BackoffOptions.java create mode 100644 src/main/java/com/coveo/pushapiclient/BackoffOptionsBuilder.java diff --git a/src/main/java/com/coveo/pushapiclient/ApiCore.java b/src/main/java/com/coveo/pushapiclient/ApiCore.java index 24d22c61..78499255 100644 --- a/src/main/java/com/coveo/pushapiclient/ApiCore.java +++ b/src/main/java/com/coveo/pushapiclient/ApiCore.java @@ -10,46 +10,43 @@ import org.apache.logging.log4j.Logger; class ApiCore { - public static final int DEFAULT_RETRY_AFTER = 5000; - public static final int DEFAULT_MAX_RETRIES = 50; private final HttpClient httpClient; private final Logger logger; - private final int retryAfter; - private final int maxRetries; + private final BackoffOptions options; public ApiCore() { this(HttpClient.newHttpClient(), LogManager.getLogger(ApiCore.class)); } public ApiCore(HttpClient httpClient, Logger logger) { - this(httpClient, logger, DEFAULT_RETRY_AFTER, DEFAULT_MAX_RETRIES); + this(httpClient, logger, new BackoffOptionsBuilder().build()); } - public ApiCore(HttpClient httpClient, Logger logger, int retryAfter, int maxRetries) { + public ApiCore(HttpClient httpClient, Logger logger, BackoffOptions options) { this.httpClient = httpClient; this.logger = logger; - this.retryAfter = retryAfter; - this.maxRetries = maxRetries; + this.options = options; } - public HttpResponse callApiWithRetries( - String method, URI uri, String[] headers, BodyPublisher body, int timeMultiple) + public HttpResponse callApiWithRetries(HttpRequest request) throws IOException, InterruptedException { - long delayInMilliseconds = retryAfter * 1000L; int nbRetries = 0; + long delayInMilliseconds = 0; while (true) { - this.logger.debug(method.toUpperCase() + " " + uri); - HttpRequest request = - HttpRequest.newBuilder().headers(headers).uri(uri).method(method, body).build(); + String uri = request.uri().toString(); + String reqMethod = request.method(); + this.logger.debug(reqMethod + " " + uri); HttpResponse response = this.httpClient.send(request, HttpResponse.BodyHandlers.ofString()); this.logResponse(response); - nbRetries++; - if (response.statusCode() == 429 && nbRetries <= maxRetries) { + if (response.statusCode() == 429 && nbRetries < this.options.getMaxRetries()) { + nbRetries++; + delayInMilliseconds = + this.options.getRetryAfter() + + (this.options.getRetryAfter() * this.options.getTimeMultiple() * (nbRetries - 1)); Thread.sleep(delayInMilliseconds); - delayInMilliseconds = delayInMilliseconds * timeMultiple; } else { if (response.statusCode() >= 400) { throw new InterruptedException( @@ -67,34 +64,29 @@ public HttpResponse post(URI uri, String[] headers) public HttpResponse post(URI uri, String[] headers, BodyPublisher body) throws IOException, InterruptedException { - this.logger.debug("POST " + uri); - HttpResponse response = this.callApiWithRetries("post", uri, headers, body, 2); - this.logResponse(response); + HttpRequest request = HttpRequest.newBuilder().headers(headers).uri(uri).POST(body).build(); + HttpResponse response = this.callApiWithRetries(request); return response; } public HttpResponse put(URI uri, String[] headers, BodyPublisher body) throws IOException, InterruptedException { - this.logger.debug("PUT " + uri); - HttpResponse response = this.callApiWithRetries("put", uri, headers, body, 2); - this.logResponse(response); + HttpRequest request = HttpRequest.newBuilder().headers(headers).uri(uri).PUT(body).build(); + HttpResponse response = this.callApiWithRetries(request); return response; } public HttpResponse delete(URI uri, String[] headers) throws IOException, InterruptedException { - this.logger.debug("DELETE " + uri); - HttpResponse response = - this.callApiWithRetries("delete", uri, headers, HttpRequest.BodyPublishers.ofString(""), 2); - this.logResponse(response); + HttpRequest request = HttpRequest.newBuilder().headers(headers).uri(uri).DELETE().build(); + HttpResponse response = this.callApiWithRetries(request); return response; } public HttpResponse delete(URI uri, String[] headers, BodyPublisher body) throws IOException, InterruptedException { - this.logger.debug("DELETE " + uri); - HttpResponse response = this.callApiWithRetries("delete", uri, headers, body, 2); - this.logResponse(response); + HttpRequest request = HttpRequest.newBuilder().headers(headers).uri(uri).method("DELETE", body).build(); + HttpResponse response = this.callApiWithRetries(request); return response; } diff --git a/src/main/java/com/coveo/pushapiclient/BackoffOptions.java b/src/main/java/com/coveo/pushapiclient/BackoffOptions.java new file mode 100644 index 00000000..4b263d6a --- /dev/null +++ b/src/main/java/com/coveo/pushapiclient/BackoffOptions.java @@ -0,0 +1,29 @@ +package com.coveo.pushapiclient; + +public class BackoffOptions { + public static final int DEFAULT_RETRY_AFTER = 5000; + public static final int DEFAULT_MAX_RETRIES = 50; + public static final int DEFAULT_TIME_MULTIPLE = 2; + + private final int retryAfter; + private final int maxRetries; + private final int timeMultiple; + + public BackoffOptions(int retryAfter, int maxRetries, int timeMultiple) { + this.retryAfter = DEFAULT_RETRY_AFTER; + this.maxRetries = DEFAULT_MAX_RETRIES; + this.timeMultiple = DEFAULT_TIME_MULTIPLE; + } + + public int getRetryAfter() { + return this.retryAfter; + } + + public int getMaxRetries() { + return this.maxRetries; + } + + public int getTimeMultiple() { + return this.timeMultiple; + } +} diff --git a/src/main/java/com/coveo/pushapiclient/BackoffOptionsBuilder.java b/src/main/java/com/coveo/pushapiclient/BackoffOptionsBuilder.java new file mode 100644 index 00000000..c3ffebdb --- /dev/null +++ b/src/main/java/com/coveo/pushapiclient/BackoffOptionsBuilder.java @@ -0,0 +1,27 @@ +package com.coveo.pushapiclient; + +public class BackoffOptionsBuilder { + + private int retryAfter = BackoffOptions.DEFAULT_RETRY_AFTER; + private int maxRetries = BackoffOptions.DEFAULT_MAX_RETRIES; + private int timeMultiple = BackoffOptions.DEFAULT_TIME_MULTIPLE; + + public BackoffOptionsBuilder withRetryAfter(int retryAfter) { + this.retryAfter = retryAfter; + return this; + } + + public BackoffOptionsBuilder withMaxRetries(int maxRetries) { + this.maxRetries = maxRetries; + return this; + } + + public BackoffOptionsBuilder withTimeMultiple(int timeMultiple) { + this.timeMultiple = timeMultiple; + return this; + } + + public BackoffOptions build() { + return new BackoffOptions(this.retryAfter, this.maxRetries, this.timeMultiple); + } +} diff --git a/src/main/java/com/coveo/pushapiclient/PlatformClient.java b/src/main/java/com/coveo/pushapiclient/PlatformClient.java index eb38341d..26a7af89 100644 --- a/src/main/java/com/coveo/pushapiclient/PlatformClient.java +++ b/src/main/java/com/coveo/pushapiclient/PlatformClient.java @@ -14,9 +14,6 @@ /** PlatformClient handles network requests to the Coveo platform */ public class PlatformClient { - public static final int DEFAULT_RETRY_AFTER = 5000; - public static final int DEFAULT_MAX_RETRIES = 50; - private final String apiKey; private final String organizationId; private final ApiCore api; @@ -35,8 +32,7 @@ public PlatformClient(String apiKey, String organizationId) { apiKey, organizationId, new PlatformUrlBuilder().build(), - DEFAULT_RETRY_AFTER, - DEFAULT_MAX_RETRIES); + new BackoffOptionsBuilder().build()); } /** @@ -48,7 +44,7 @@ public PlatformClient(String apiKey, String organizationId) { * @param organizationId The Coveo Organization identifier. */ public PlatformClient(String apiKey, String organizationId, PlatformUrl platformUrl) { - this(apiKey, organizationId, platformUrl, DEFAULT_RETRY_AFTER, DEFAULT_MAX_RETRIES); + this(apiKey, organizationId, platformUrl, new BackoffOptionsBuilder().build()); } /** @@ -61,8 +57,8 @@ public PlatformClient(String apiKey, String organizationId, PlatformUrl platform * @param retryAfter The amount of time, in milliseconds, to wait between request attempts. * @param maxRetries The maximum number of attempts to make for a request. */ - public PlatformClient(String apiKey, String organizationId, int retryAfter, int maxRetries) { - this(apiKey, organizationId, new PlatformUrlBuilder().build(), retryAfter, maxRetries); + public PlatformClient(String apiKey, String organizationId, BackoffOptions options) { + this(apiKey, organizationId, new PlatformUrlBuilder().build(), options); } /** @@ -77,11 +73,7 @@ public PlatformClient(String apiKey, String organizationId, int retryAfter, int * @param maxRetries The maximum number of attempts to make for a request. */ public PlatformClient( - String apiKey, - String organizationId, - PlatformUrl platformUrl, - int retryAfter, - int maxRetries) { + String apiKey, String organizationId, PlatformUrl platformUrl, BackoffOptions options) { this.apiKey = apiKey; this.organizationId = organizationId; this.api = new ApiCore(); @@ -98,7 +90,7 @@ public PlatformClient( * @param httpClient The HttpClient. */ public PlatformClient(String apiKey, String organizationId, HttpClient httpClient) { - this(apiKey, organizationId, httpClient, DEFAULT_RETRY_AFTER, DEFAULT_MAX_RETRIES); + this(apiKey, organizationId, httpClient, new BackoffOptionsBuilder().build()); } /** @@ -113,10 +105,10 @@ public PlatformClient(String apiKey, String organizationId, HttpClient httpClien * @param maxRetries The maximum number of attempts to make for a request. */ public PlatformClient( - String apiKey, String organizationId, HttpClient httpClient, int retryAfter, int maxRetries) { + String apiKey, String organizationId, HttpClient httpClient, BackoffOptions options) { this.apiKey = apiKey; this.organizationId = organizationId; - this.api = new ApiCore(httpClient, LogManager.getLogger(ApiCore.class), retryAfter, maxRetries); + this.api = new ApiCore(httpClient, LogManager.getLogger(ApiCore.class), options); this.platformUrl = new PlatformUrlBuilder().build(); } diff --git a/src/main/java/com/coveo/pushapiclient/PushSource.java b/src/main/java/com/coveo/pushapiclient/PushSource.java index a19ae72f..eeba4066 100644 --- a/src/main/java/com/coveo/pushapiclient/PushSource.java +++ b/src/main/java/com/coveo/pushapiclient/PushSource.java @@ -143,17 +143,14 @@ public static PushSource fromPlatformUrl( * use the {@link PlatformUrl} when your organization is located in a non-default Coveo * environement and/or region. When not specified, the default platform URL values will be * used: {@link PlatformUrl#DEFAULT_ENVIRONMENT} and {@link PlatformUrl#DEFAULT_REGION} - * @param retryAfter The amount of time, in milliseconds, to wait between request attempts. - * @param maxRetries The maximum number of attempts to make for a request. */ public static PushSource fromPlatformUrl( String apiKey, String organizationId, String sourceId, PlatformUrl platformUrl, - int retryAfter, - int maxRetries) { - return new PushSource(apiKey, organizationId, sourceId, platformUrl, retryAfter, maxRetries); + BackoffOptions options) { + return new PushSource(apiKey, organizationId, sourceId, platformUrl, options); } private PushSource( @@ -168,12 +165,10 @@ private PushSource( String organizationId, String sourceId, PlatformUrl platformUrl, - int retryAfter, - int maxRetries) { + BackoffOptions options) { this.apiKey = apiKey; this.urlExtractor = new ApiUrl(organizationId, sourceId, platformUrl); - this.platformClient = - new PlatformClient(apiKey, organizationId, platformUrl, retryAfter, maxRetries); + this.platformClient = new PlatformClient(apiKey, organizationId, platformUrl, options); } /** From e523534b595f7612bc912db5b55de3e69725fb24 Mon Sep 17 00:00:00 2001 From: David Brooke <38883189+dmbrooke@users.noreply.github.com> Date: Fri, 1 Sep 2023 16:47:26 -0400 Subject: [PATCH 05/12] Further implement options config --- .../java/com/coveo/pushapiclient/ApiCore.java | 11 ++++---- .../com/coveo/pushapiclient/PushService.java | 6 +++- .../java/com/coveo/pushapiclient/Source.java | 28 +++++++++++++++++-- .../coveo/pushapiclient/StreamService.java | 17 ++++++++++- 4 files changed, 52 insertions(+), 10 deletions(-) diff --git a/src/main/java/com/coveo/pushapiclient/ApiCore.java b/src/main/java/com/coveo/pushapiclient/ApiCore.java index 78499255..bbd728ec 100644 --- a/src/main/java/com/coveo/pushapiclient/ApiCore.java +++ b/src/main/java/com/coveo/pushapiclient/ApiCore.java @@ -41,17 +41,15 @@ public HttpResponse callApiWithRetries(HttpRequest request) this.httpClient.send(request, HttpResponse.BodyHandlers.ofString()); this.logResponse(response); - if (response.statusCode() == 429 && nbRetries < this.options.getMaxRetries()) { + if (response != null + && response.statusCode() == 429 + && nbRetries < this.options.getMaxRetries()) { nbRetries++; delayInMilliseconds = this.options.getRetryAfter() + (this.options.getRetryAfter() * this.options.getTimeMultiple() * (nbRetries - 1)); Thread.sleep(delayInMilliseconds); } else { - if (response.statusCode() >= 400) { - throw new InterruptedException( - "HTTP error " + response.statusCode() + " : " + response.body()); - } return response; } } @@ -85,7 +83,8 @@ public HttpResponse delete(URI uri, String[] headers) public HttpResponse delete(URI uri, String[] headers, BodyPublisher body) throws IOException, InterruptedException { - HttpRequest request = HttpRequest.newBuilder().headers(headers).uri(uri).method("DELETE", body).build(); + HttpRequest request = + HttpRequest.newBuilder().headers(headers).uri(uri).method("DELETE", body).build(); HttpResponse response = this.callApiWithRetries(request); return response; } diff --git a/src/main/java/com/coveo/pushapiclient/PushService.java b/src/main/java/com/coveo/pushapiclient/PushService.java index 8691fe83..a7ba3665 100644 --- a/src/main/java/com/coveo/pushapiclient/PushService.java +++ b/src/main/java/com/coveo/pushapiclient/PushService.java @@ -10,13 +10,17 @@ public class PushService { private PushServiceInternal service; public PushService(PushEnabledSource source) { + this(source, new BackoffOptionsBuilder().build()); + } + + public PushService(PushEnabledSource source, BackoffOptions options) { String apiKey = source.getApiKey(); String organizationId = source.getOrganizationId(); PlatformUrl platformUrl = source.getPlatformUrl(); UploadStrategy uploader = this.getUploadStrategy(); DocumentUploadQueue queue = new DocumentUploadQueue(uploader); - this.platformClient = new PlatformClient(apiKey, organizationId, platformUrl); + this.platformClient = new PlatformClient(apiKey, organizationId, platformUrl, options); this.service = new PushServiceInternal(queue); this.source = source; } diff --git a/src/main/java/com/coveo/pushapiclient/Source.java b/src/main/java/com/coveo/pushapiclient/Source.java index a9099ff8..054db017 100644 --- a/src/main/java/com/coveo/pushapiclient/Source.java +++ b/src/main/java/com/coveo/pushapiclient/Source.java @@ -15,7 +15,18 @@ public class Source { * @param organizationId The Coveo Organization identifier. */ public Source(String apiKey, String organizationId) { - this.platformClient = new PlatformClient(apiKey, organizationId); + this(apiKey, organizationId, new BackoffOptionsBuilder().build()); + } + + /** + * @param apiKey An apiKey capable of pushing documents and managing sources in a Coveo + * organization. + * @see Manage API Keys. + * @param organizationId The Coveo Organization identifier. + * @param options The options for exponential backoff. + */ + public Source(String apiKey, String organizationId, BackoffOptions options) { + this.platformClient = new PlatformClient(apiKey, organizationId, options); } /** @@ -26,7 +37,20 @@ public Source(String apiKey, String organizationId) { * @param platformUrl */ public Source(String apiKey, String organizationId, PlatformUrl platformUrl) { - this.platformClient = new PlatformClient(apiKey, organizationId, platformUrl); + this(apiKey, organizationId, platformUrl, new BackoffOptionsBuilder().build()); + } + + /** + * @param apiKey An apiKey capable of pushing documents and managing sources in a Coveo + * organization. + * @see Manage API Keys. + * @param organizationId The Coveo Organization identifier. + * @param platformUrl + * @param options + */ + public Source( + String apiKey, String organizationId, PlatformUrl platformUrl, BackoffOptions options) { + this.platformClient = new PlatformClient(apiKey, organizationId, platformUrl, options); } /** diff --git a/src/main/java/com/coveo/pushapiclient/StreamService.java b/src/main/java/com/coveo/pushapiclient/StreamService.java index 3ba94f5e..ca5da586 100644 --- a/src/main/java/com/coveo/pushapiclient/StreamService.java +++ b/src/main/java/com/coveo/pushapiclient/StreamService.java @@ -25,6 +25,21 @@ public class StreamService { * @param source The source to which you want to send your documents. */ public StreamService(StreamEnabledSource source) { + this(source, new BackoffOptionsBuilder().build()); + } + + /** + * Creates a service to stream your documents to the provided source by interacting with the + * Stream API. + * + *

To perform full document updates, use the + * {@PushService}, since pushing documents with the {@StreamService} is equivalent to triggering a + * full source rebuild. The {@StreamService} can also be used for an initial catalog upload. + * + * @param source The source to which you want to send your documents. + * @param options The options for exponential backoff. + */ + public StreamService(StreamEnabledSource source, BackoffOptions options) { String apiKey = source.getApiKey(); String organizationId = source.getOrganizationId(); PlatformUrl platformUrl = source.getPlatformUrl(); @@ -33,7 +48,7 @@ public StreamService(StreamEnabledSource source) { this.source = source; this.queue = new DocumentUploadQueue(uploader); - this.platformClient = new PlatformClient(apiKey, organizationId, platformUrl); + this.platformClient = new PlatformClient(apiKey, organizationId, platformUrl, options); this.service = new StreamServiceInternal(this.source, this.queue, this.platformClient, logger); } From 4f8f290e7689c97b8d18d7d851b7b83e5ebcdb14 Mon Sep 17 00:00:00 2001 From: David Brooke <38883189+dmbrooke@users.noreply.github.com> Date: Tue, 5 Sep 2023 10:29:19 -0400 Subject: [PATCH 06/12] Backoff options fix + tests --- .../coveo/pushapiclient/BackoffOptions.java | 6 +-- .../BackoffOptionsBuilderTest.java | 42 +++++++++++++++++++ 2 files changed, 45 insertions(+), 3 deletions(-) create mode 100644 src/test/java/com/coveo/pushapiclient/BackoffOptionsBuilderTest.java diff --git a/src/main/java/com/coveo/pushapiclient/BackoffOptions.java b/src/main/java/com/coveo/pushapiclient/BackoffOptions.java index 4b263d6a..0a0c7469 100644 --- a/src/main/java/com/coveo/pushapiclient/BackoffOptions.java +++ b/src/main/java/com/coveo/pushapiclient/BackoffOptions.java @@ -10,9 +10,9 @@ public class BackoffOptions { private final int timeMultiple; public BackoffOptions(int retryAfter, int maxRetries, int timeMultiple) { - this.retryAfter = DEFAULT_RETRY_AFTER; - this.maxRetries = DEFAULT_MAX_RETRIES; - this.timeMultiple = DEFAULT_TIME_MULTIPLE; + this.retryAfter = retryAfter; + this.maxRetries = maxRetries; + this.timeMultiple = timeMultiple; } public int getRetryAfter() { diff --git a/src/test/java/com/coveo/pushapiclient/BackoffOptionsBuilderTest.java b/src/test/java/com/coveo/pushapiclient/BackoffOptionsBuilderTest.java new file mode 100644 index 00000000..453b6651 --- /dev/null +++ b/src/test/java/com/coveo/pushapiclient/BackoffOptionsBuilderTest.java @@ -0,0 +1,42 @@ +package com.coveo.pushapiclient; + +import static org.junit.Assert.*; + +import org.junit.Before; +import org.junit.Test; + +public class BackoffOptionsBuilderTest { + + private BackoffOptionsBuilder backoffOptionsBuilder; + + @Before + public void setup() { + backoffOptionsBuilder = new BackoffOptionsBuilder(); + } + + @Test + public void testWithDefaultValues() { + BackoffOptions backoffOptions = backoffOptionsBuilder.build(); + assertEquals("Should return default retry after time", 5000, backoffOptions.getRetryAfter()); + assertEquals("Should return default max retries", 50, backoffOptions.getMaxRetries()); + assertEquals("Should return default time multiple", 2, backoffOptions.getTimeMultiple()); + } + + @Test + public void testWithNonDefaultRetryAfter() { + BackoffOptions backoffOptions = backoffOptionsBuilder.withRetryAfter(1000).build(); + assertEquals("Should return Europe platform URL", 1000, backoffOptions.getRetryAfter()); + } + + @Test + public void testWithNonDefaultMaxRetries() { + BackoffOptions backoffOptions = backoffOptionsBuilder.withMaxRetries(15).build(); + assertEquals("Should return the staging platform URL", 15, backoffOptions.getMaxRetries()); + } + + @Test + public void testWithNonDefaultTimeMultiple() { + BackoffOptions backoffOptions = backoffOptionsBuilder.withTimeMultiple(3).build(); + assertEquals(3, backoffOptions.getTimeMultiple()); + } +} From d46683c1640e9a59adb3b92fe9fea355eeea6184 Mon Sep 17 00:00:00 2001 From: David Brooke <38883189+dmbrooke@users.noreply.github.com> Date: Tue, 5 Sep 2023 11:37:56 -0400 Subject: [PATCH 07/12] Improved logic, APICore tests --- .../java/com/coveo/pushapiclient/ApiCore.java | 8 +++--- .../com/coveo/pushapiclient/ApiCoreTest.java | 26 +++++++++++++++++++ 2 files changed, 29 insertions(+), 5 deletions(-) diff --git a/src/main/java/com/coveo/pushapiclient/ApiCore.java b/src/main/java/com/coveo/pushapiclient/ApiCore.java index bbd728ec..0d5fbe51 100644 --- a/src/main/java/com/coveo/pushapiclient/ApiCore.java +++ b/src/main/java/com/coveo/pushapiclient/ApiCore.java @@ -31,7 +31,7 @@ public ApiCore(HttpClient httpClient, Logger logger, BackoffOptions options) { public HttpResponse callApiWithRetries(HttpRequest request) throws IOException, InterruptedException { int nbRetries = 0; - long delayInMilliseconds = 0; + long delayInMilliseconds = this.options.getRetryAfter(); while (true) { String uri = request.uri().toString(); @@ -44,11 +44,9 @@ public HttpResponse callApiWithRetries(HttpRequest request) if (response != null && response.statusCode() == 429 && nbRetries < this.options.getMaxRetries()) { + Thread.sleep(delayInMilliseconds); nbRetries++; - delayInMilliseconds = - this.options.getRetryAfter() - + (this.options.getRetryAfter() * this.options.getTimeMultiple() * (nbRetries - 1)); - Thread.sleep(delayInMilliseconds); + delayInMilliseconds *= this.options.getTimeMultiple(); } else { return response; } diff --git a/src/test/java/com/coveo/pushapiclient/ApiCoreTest.java b/src/test/java/com/coveo/pushapiclient/ApiCoreTest.java index c876fbfe..1ffe7c60 100644 --- a/src/test/java/com/coveo/pushapiclient/ApiCoreTest.java +++ b/src/test/java/com/coveo/pushapiclient/ApiCoreTest.java @@ -25,6 +25,7 @@ public class ApiCoreTest { @Mock private HttpClient httpClient; @Mock private HttpRequest httpRequest; @Mock private Logger logger; + @Mock private BackoffOptions backoffOptions; @Mock private HttpResponse httpResponse; @InjectMocks private ApiCore api; @@ -46,12 +47,25 @@ private void mockErrorResponse() { when(httpRequest.method()).thenReturn("DELETE"); } + private void mockThrottledResponse() { + when(httpResponse.statusCode()).thenReturn(429); + when(httpResponse.body()).thenReturn("THROTTLED_REQUEST"); + when(httpRequest.method()).thenReturn("POST"); + } + + private void mockBackoffOptions() { + when(backoffOptions.getMaxRetries()).thenReturn(2); + when(backoffOptions.getRetryAfter()).thenReturn(100); + when(backoffOptions.getTimeMultiple()).thenReturn(2); + } + @Before public void setUp() throws Exception { closeable = MockitoAnnotations.openMocks(this); when(httpClient.send(any(HttpRequest.class), any(BodyHandler.class))).thenReturn(httpResponse); when(httpResponse.request()).thenReturn(httpRequest); + mockBackoffOptions(); } @After @@ -79,4 +93,16 @@ public void testShouldLogResponse() throws IOException, InterruptedException, UR verify(logger, times(1)).error("DELETE status: 412"); verify(logger, times(1)).error("DELETE response: BAD_REQUEST"); } + + @Test + public void testShouldHandleBackoffOptions() + throws IOException, InterruptedException, URISyntaxException { + this.mockThrottledResponse(); + + this.api.post(new URI("https://perdu.com/"), headers); + + verify(logger, times(3)).debug("POST https://perdu.com/"); + verify(logger, times(3)).error("POST status: 429"); + verify(logger, times(3)).error("POST response: THROTTLED_REQUEST"); + } } From af774e8613ab80f47e1e90321be602506be92b36 Mon Sep 17 00:00:00 2001 From: David Brooke <38883189+dmbrooke@users.noreply.github.com> Date: Tue, 5 Sep 2023 11:46:54 -0400 Subject: [PATCH 08/12] Better docs strings, linting --- src/main/java/com/coveo/pushapiclient/ApiCore.java | 2 +- .../java/com/coveo/pushapiclient/PlatformClient.java | 9 +++------ .../java/com/coveo/pushapiclient/PushSource.java | 1 + src/main/java/com/coveo/pushapiclient/Source.java | 12 +++++++++--- .../java/com/coveo/pushapiclient/StreamService.java | 2 +- 5 files changed, 15 insertions(+), 11 deletions(-) diff --git a/src/main/java/com/coveo/pushapiclient/ApiCore.java b/src/main/java/com/coveo/pushapiclient/ApiCore.java index 0d5fbe51..f03a9e11 100644 --- a/src/main/java/com/coveo/pushapiclient/ApiCore.java +++ b/src/main/java/com/coveo/pushapiclient/ApiCore.java @@ -44,7 +44,7 @@ public HttpResponse callApiWithRetries(HttpRequest request) if (response != null && response.statusCode() == 429 && nbRetries < this.options.getMaxRetries()) { - Thread.sleep(delayInMilliseconds); + Thread.sleep(delayInMilliseconds); nbRetries++; delayInMilliseconds *= this.options.getTimeMultiple(); } else { diff --git a/src/main/java/com/coveo/pushapiclient/PlatformClient.java b/src/main/java/com/coveo/pushapiclient/PlatformClient.java index 26a7af89..f11ad128 100644 --- a/src/main/java/com/coveo/pushapiclient/PlatformClient.java +++ b/src/main/java/com/coveo/pushapiclient/PlatformClient.java @@ -54,8 +54,7 @@ public PlatformClient(String apiKey, String organizationId, PlatformUrl platform * organization. * @see Manage API Keys * @param organizationId The Coveo Organization identifier. - * @param retryAfter The amount of time, in milliseconds, to wait between request attempts. - * @param maxRetries The maximum number of attempts to make for a request. + * @param options The configuration options for exponential backoff */ public PlatformClient(String apiKey, String organizationId, BackoffOptions options) { this(apiKey, organizationId, new PlatformUrlBuilder().build(), options); @@ -69,8 +68,7 @@ public PlatformClient(String apiKey, String organizationId, BackoffOptions optio * @see Manage API Keys * @param organizationId The Coveo Organization identifier. * @param platformUrl The PlatformUrl. - * @param retryAfter The amount of time, in milliseconds, to wait between request attempts. - * @param maxRetries The maximum number of attempts to make for a request. + * @param options The configuration options for exponential backoff */ public PlatformClient( String apiKey, String organizationId, PlatformUrl platformUrl, BackoffOptions options) { @@ -101,8 +99,7 @@ public PlatformClient(String apiKey, String organizationId, HttpClient httpClien * @see Manage API Keys * @param organizationId The Coveo Organization identifier. * @param httpClient The HttpClient. - * @param retryAfter The amount of time, in milliseconds, to wait between request attempts. - * @param maxRetries The maximum number of attempts to make for a request. + * @param options The configuration options for exponential backoff */ public PlatformClient( String apiKey, String organizationId, HttpClient httpClient, BackoffOptions options) { diff --git a/src/main/java/com/coveo/pushapiclient/PushSource.java b/src/main/java/com/coveo/pushapiclient/PushSource.java index eeba4066..87d1ab16 100644 --- a/src/main/java/com/coveo/pushapiclient/PushSource.java +++ b/src/main/java/com/coveo/pushapiclient/PushSource.java @@ -143,6 +143,7 @@ public static PushSource fromPlatformUrl( * use the {@link PlatformUrl} when your organization is located in a non-default Coveo * environement and/or region. When not specified, the default platform URL values will be * used: {@link PlatformUrl#DEFAULT_ENVIRONMENT} and {@link PlatformUrl#DEFAULT_REGION} + * * @param options The configuration options for exponential backoff */ public static PushSource fromPlatformUrl( String apiKey, diff --git a/src/main/java/com/coveo/pushapiclient/Source.java b/src/main/java/com/coveo/pushapiclient/Source.java index 054db017..df757643 100644 --- a/src/main/java/com/coveo/pushapiclient/Source.java +++ b/src/main/java/com/coveo/pushapiclient/Source.java @@ -34,7 +34,10 @@ public Source(String apiKey, String organizationId, BackoffOptions options) { * organization. * @see Manage API Keys. * @param organizationId The Coveo Organization identifier. - * @param platformUrl + * @param platformUrl The object containing additional information on the URL endpoint. You can + * use the {@link PlatformUrl} when your organization is located in a non-default Coveo + * environement and/or region. When not specified, the default platform URL values will be + * used: {@link PlatformUrl#DEFAULT_ENVIRONMENT} and {@link PlatformUrl#DEFAULT_REGION} */ public Source(String apiKey, String organizationId, PlatformUrl platformUrl) { this(apiKey, organizationId, platformUrl, new BackoffOptionsBuilder().build()); @@ -45,8 +48,11 @@ public Source(String apiKey, String organizationId, PlatformUrl platformUrl) { * organization. * @see Manage API Keys. * @param organizationId The Coveo Organization identifier. - * @param platformUrl - * @param options + * @param platformUrl The object containing additional information on the URL endpoint. You can + * use the {@link PlatformUrl} when your organization is located in a non-default Coveo + * environement and/or region. When not specified, the default platform URL values will be + * used: {@link PlatformUrl#DEFAULT_ENVIRONMENT} and {@link PlatformUrl#DEFAULT_REGION} + * @param options The configuration options for exponential backoff */ public Source( String apiKey, String organizationId, PlatformUrl platformUrl, BackoffOptions options) { diff --git a/src/main/java/com/coveo/pushapiclient/StreamService.java b/src/main/java/com/coveo/pushapiclient/StreamService.java index ca5da586..263bc16f 100644 --- a/src/main/java/com/coveo/pushapiclient/StreamService.java +++ b/src/main/java/com/coveo/pushapiclient/StreamService.java @@ -37,7 +37,7 @@ public StreamService(StreamEnabledSource source) { * full source rebuild. The {@StreamService} can also be used for an initial catalog upload. * * @param source The source to which you want to send your documents. - * @param options The options for exponential backoff. + * @param options The configuration options for exponential backoff */ public StreamService(StreamEnabledSource source, BackoffOptions options) { String apiKey = source.getApiKey(); From 31b9b304ddbce54fbb74e31e08bc6cb518743144 Mon Sep 17 00:00:00 2001 From: David Brooke <38883189+dmbrooke@users.noreply.github.com> Date: Wed, 6 Sep 2023 15:22:26 -0400 Subject: [PATCH 09/12] Replace custom logic with Resilience4j Retry --- pom.xml | 5 ++ .../java/com/coveo/pushapiclient/ApiCore.java | 46 ++++++++++++------- .../coveo/pushapiclient/BackoffOptions.java | 4 -- .../pushapiclient/BackoffOptionsBuilder.java | 9 ++-- .../com/coveo/pushapiclient/ApiCoreTest.java | 6 +-- 5 files changed, 44 insertions(+), 26 deletions(-) diff --git a/pom.xml b/pom.xml index f0c8d3d7..0b9d5a26 100644 --- a/pom.xml +++ b/pom.xml @@ -167,6 +167,11 @@ commons-codec 1.16.0 + + io.github.resilience4j + resilience4j-retry + 2.0.0 + diff --git a/src/main/java/com/coveo/pushapiclient/ApiCore.java b/src/main/java/com/coveo/pushapiclient/ApiCore.java index f03a9e11..283b3761 100644 --- a/src/main/java/com/coveo/pushapiclient/ApiCore.java +++ b/src/main/java/com/coveo/pushapiclient/ApiCore.java @@ -1,11 +1,15 @@ package com.coveo.pushapiclient; +import io.github.resilience4j.core.IntervalFunction; +import io.github.resilience4j.retry.Retry; +import io.github.resilience4j.retry.RetryConfig; import java.io.IOException; import java.net.URI; import java.net.http.HttpClient; import java.net.http.HttpRequest; import java.net.http.HttpRequest.BodyPublisher; import java.net.http.HttpResponse; +import java.util.function.Function; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -30,26 +34,36 @@ public ApiCore(HttpClient httpClient, Logger logger, BackoffOptions options) { public HttpResponse callApiWithRetries(HttpRequest request) throws IOException, InterruptedException { - int nbRetries = 0; - long delayInMilliseconds = this.options.getRetryAfter(); + IntervalFunction intervalFn = + IntervalFunction.ofExponentialRandomBackoff( + this.options.getRetryAfter(), this.options.getTimeMultiple()); - while (true) { - String uri = request.uri().toString(); - String reqMethod = request.method(); - this.logger.debug(reqMethod + " " + uri); + RetryConfig retryConfig = + RetryConfig.>custom() + .maxAttempts(this.options.getMaxRetries()) + .intervalFunction(intervalFn) + .retryOnResult(response -> response != null && response.statusCode() == 429) + .build(); + + Retry retry = Retry.of("platformRequest", retryConfig); + + Function> retryRequestFn = + Retry.decorateFunction(retry, req -> sendRequest(req)); + + return retryRequestFn.apply(request); + } + + public HttpResponse sendRequest(HttpRequest request) { + String uri = request.uri().toString(); + String reqMethod = request.method(); + this.logger.debug(reqMethod + " " + uri); + try { HttpResponse response = this.httpClient.send(request, HttpResponse.BodyHandlers.ofString()); this.logResponse(response); - - if (response != null - && response.statusCode() == 429 - && nbRetries < this.options.getMaxRetries()) { - Thread.sleep(delayInMilliseconds); - nbRetries++; - delayInMilliseconds *= this.options.getTimeMultiple(); - } else { - return response; - } + return response; + } catch (IOException | InterruptedException e) { + throw new Error(e.getMessage()); } } diff --git a/src/main/java/com/coveo/pushapiclient/BackoffOptions.java b/src/main/java/com/coveo/pushapiclient/BackoffOptions.java index 0a0c7469..ad94adaa 100644 --- a/src/main/java/com/coveo/pushapiclient/BackoffOptions.java +++ b/src/main/java/com/coveo/pushapiclient/BackoffOptions.java @@ -1,10 +1,6 @@ package com.coveo.pushapiclient; public class BackoffOptions { - public static final int DEFAULT_RETRY_AFTER = 5000; - public static final int DEFAULT_MAX_RETRIES = 50; - public static final int DEFAULT_TIME_MULTIPLE = 2; - private final int retryAfter; private final int maxRetries; private final int timeMultiple; diff --git a/src/main/java/com/coveo/pushapiclient/BackoffOptionsBuilder.java b/src/main/java/com/coveo/pushapiclient/BackoffOptionsBuilder.java index c3ffebdb..1f254d39 100644 --- a/src/main/java/com/coveo/pushapiclient/BackoffOptionsBuilder.java +++ b/src/main/java/com/coveo/pushapiclient/BackoffOptionsBuilder.java @@ -1,10 +1,13 @@ package com.coveo.pushapiclient; public class BackoffOptionsBuilder { + public static final int DEFAULT_RETRY_AFTER = 5000; + public static final int DEFAULT_MAX_RETRIES = 50; + public static final int DEFAULT_TIME_MULTIPLE = 2; - private int retryAfter = BackoffOptions.DEFAULT_RETRY_AFTER; - private int maxRetries = BackoffOptions.DEFAULT_MAX_RETRIES; - private int timeMultiple = BackoffOptions.DEFAULT_TIME_MULTIPLE; + private int retryAfter = DEFAULT_RETRY_AFTER; + private int maxRetries = DEFAULT_MAX_RETRIES; + private int timeMultiple = DEFAULT_TIME_MULTIPLE; public BackoffOptionsBuilder withRetryAfter(int retryAfter) { this.retryAfter = retryAfter; diff --git a/src/test/java/com/coveo/pushapiclient/ApiCoreTest.java b/src/test/java/com/coveo/pushapiclient/ApiCoreTest.java index 1ffe7c60..ae60e5fa 100644 --- a/src/test/java/com/coveo/pushapiclient/ApiCoreTest.java +++ b/src/test/java/com/coveo/pushapiclient/ApiCoreTest.java @@ -101,8 +101,8 @@ public void testShouldHandleBackoffOptions() this.api.post(new URI("https://perdu.com/"), headers); - verify(logger, times(3)).debug("POST https://perdu.com/"); - verify(logger, times(3)).error("POST status: 429"); - verify(logger, times(3)).error("POST response: THROTTLED_REQUEST"); + verify(logger, times(2)).debug("POST https://perdu.com/"); + verify(logger, times(2)).error("POST status: 429"); + verify(logger, times(2)).error("POST response: THROTTLED_REQUEST"); } } From cf5d641141fec912dfade784749eb638e709fdea Mon Sep 17 00:00:00 2001 From: David Brooke <38883189+dmbrooke@users.noreply.github.com> Date: Wed, 6 Sep 2023 15:59:57 -0400 Subject: [PATCH 10/12] Move to older version of resilience4j --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 0b9d5a26..42e9db55 100644 --- a/pom.xml +++ b/pom.xml @@ -170,7 +170,7 @@ io.github.resilience4j resilience4j-retry - 2.0.0 + 1.7.0 From 8f9e188835df95caa4ddb8ff9e1a0fbd32bf6241 Mon Sep 17 00:00:00 2001 From: David Brooke <38883189+dmbrooke@users.noreply.github.com> Date: Mon, 11 Sep 2023 15:57:08 -0400 Subject: [PATCH 11/12] Add samples, undo Source constructor changes --- samples/DeleteOneDocument.java | 5 ++- samples/PushOneDocument.java | 3 +- samples/PushOneDocumentWithMetadata.java | 5 ++- .../coveo/pushapiclient/PlatformClient.java | 6 +-- .../com/coveo/pushapiclient/PushSource.java | 2 +- .../java/com/coveo/pushapiclient/Source.java | 38 ++----------------- .../coveo/pushapiclient/StreamService.java | 2 +- 7 files changed, 18 insertions(+), 43 deletions(-) diff --git a/samples/DeleteOneDocument.java b/samples/DeleteOneDocument.java index 5fff10dd..a9c32133 100644 --- a/samples/DeleteOneDocument.java +++ b/samples/DeleteOneDocument.java @@ -1,3 +1,4 @@ +import com.coveo.pushapiclient.PushSource; import com.coveo.pushapiclient.Source; import java.io.IOException; @@ -5,12 +6,12 @@ public class DeleteOneDocument { public static void main(String[] args) { - Source source = new Source("my_api_key", "my_org_id"); + PushSource source = new PushSource("my_api_key", "my_org_id"); String documentId = "https://my.document.uri"; Boolean deleteChildren = true; try { - HttpResponse response = source.deleteDocument("my_source_id", documentId, deleteChildren); + HttpResponse response = source.deleteDocument(documentId, deleteChildren); System.out.println(String.format("Delete document status: %s", response.statusCode())); System.out.println(String.format("Delete document response: %s", response.body())); } catch (IOException e) { diff --git a/samples/PushOneDocument.java b/samples/PushOneDocument.java index 0f44b529..e5c30aac 100644 --- a/samples/PushOneDocument.java +++ b/samples/PushOneDocument.java @@ -1,3 +1,4 @@ +import com.coveo.pushapiclient.BackoffOptionsBuilder; import com.coveo.pushapiclient.DocumentBuilder; import com.coveo.pushapiclient.Source; @@ -6,7 +7,7 @@ public class PushOneDocument { public static void main(String[] args) { - Source source = new Source("my_api_key", "my_org_id"); + PushSource source = new PushSource("my_api_key", "my_org_id", new BackoffOptionsBuilder().withMaxRetries(5).withRetryAfter(10000).build()); DocumentBuilder documentBuilder = new DocumentBuilder("https://my.document.uri", "My document title") .withData("these words will be searchable"); diff --git a/samples/PushOneDocumentWithMetadata.java b/samples/PushOneDocumentWithMetadata.java index d28ea89f..f006d1ce 100644 --- a/samples/PushOneDocumentWithMetadata.java +++ b/samples/PushOneDocumentWithMetadata.java @@ -1,4 +1,7 @@ +import com.coveo.pushapiclient.BackoffOptions; +import com.coveo.pushapiclient.BackoffOptionsBuilder; import com.coveo.pushapiclient.DocumentBuilder; +import com.coveo.pushapiclient.PushSource; import com.coveo.pushapiclient.Source; import java.io.IOException; @@ -7,7 +10,7 @@ public class PushOneDocumentWithMetadata { public static void main(String[] args) { - Source source = new Source("my_api_key", "my_org_id"); + PushSource source = new PushSource("my_api_key", "my_org_id", new BackoffOptionsBuilder().withTimeMultiple(1).build()); DocumentBuilder documentBuilder = new DocumentBuilder("https://my.document.uri", "My document title") .withData("these words will be searchable") .withAuthor("bob") diff --git a/src/main/java/com/coveo/pushapiclient/PlatformClient.java b/src/main/java/com/coveo/pushapiclient/PlatformClient.java index f11ad128..9d2c665b 100644 --- a/src/main/java/com/coveo/pushapiclient/PlatformClient.java +++ b/src/main/java/com/coveo/pushapiclient/PlatformClient.java @@ -54,7 +54,7 @@ public PlatformClient(String apiKey, String organizationId, PlatformUrl platform * organization. * @see Manage API Keys * @param organizationId The Coveo Organization identifier. - * @param options The configuration options for exponential backoff + * @param options The configuration options for exponential backoff. */ public PlatformClient(String apiKey, String organizationId, BackoffOptions options) { this(apiKey, organizationId, new PlatformUrlBuilder().build(), options); @@ -68,7 +68,7 @@ public PlatformClient(String apiKey, String organizationId, BackoffOptions optio * @see Manage API Keys * @param organizationId The Coveo Organization identifier. * @param platformUrl The PlatformUrl. - * @param options The configuration options for exponential backoff + * @param options The configuration options for exponential backoff. */ public PlatformClient( String apiKey, String organizationId, PlatformUrl platformUrl, BackoffOptions options) { @@ -99,7 +99,7 @@ public PlatformClient(String apiKey, String organizationId, HttpClient httpClien * @see Manage API Keys * @param organizationId The Coveo Organization identifier. * @param httpClient The HttpClient. - * @param options The configuration options for exponential backoff + * @param options The configuration options for exponential backoff. */ public PlatformClient( String apiKey, String organizationId, HttpClient httpClient, BackoffOptions options) { diff --git a/src/main/java/com/coveo/pushapiclient/PushSource.java b/src/main/java/com/coveo/pushapiclient/PushSource.java index 87d1ab16..5371598a 100644 --- a/src/main/java/com/coveo/pushapiclient/PushSource.java +++ b/src/main/java/com/coveo/pushapiclient/PushSource.java @@ -143,7 +143,7 @@ public static PushSource fromPlatformUrl( * use the {@link PlatformUrl} when your organization is located in a non-default Coveo * environement and/or region. When not specified, the default platform URL values will be * used: {@link PlatformUrl#DEFAULT_ENVIRONMENT} and {@link PlatformUrl#DEFAULT_REGION} - * * @param options The configuration options for exponential backoff + * @param options The configuration options for exponential backoff. */ public static PushSource fromPlatformUrl( String apiKey, diff --git a/src/main/java/com/coveo/pushapiclient/Source.java b/src/main/java/com/coveo/pushapiclient/Source.java index df757643..c4177e4a 100644 --- a/src/main/java/com/coveo/pushapiclient/Source.java +++ b/src/main/java/com/coveo/pushapiclient/Source.java @@ -15,7 +15,7 @@ public class Source { * @param organizationId The Coveo Organization identifier. */ public Source(String apiKey, String organizationId) { - this(apiKey, organizationId, new BackoffOptionsBuilder().build()); + this.platformClient = new PlatformClient(apiKey, organizationId); } /** @@ -23,40 +23,10 @@ public Source(String apiKey, String organizationId) { * organization. * @see Manage API Keys. * @param organizationId The Coveo Organization identifier. - * @param options The options for exponential backoff. - */ - public Source(String apiKey, String organizationId, BackoffOptions options) { - this.platformClient = new PlatformClient(apiKey, organizationId, options); - } - - /** - * @param apiKey An apiKey capable of pushing documents and managing sources in a Coveo - * organization. - * @see Manage API Keys. - * @param organizationId The Coveo Organization identifier. - * @param platformUrl The object containing additional information on the URL endpoint. You can - * use the {@link PlatformUrl} when your organization is located in a non-default Coveo - * environement and/or region. When not specified, the default platform URL values will be - * used: {@link PlatformUrl#DEFAULT_ENVIRONMENT} and {@link PlatformUrl#DEFAULT_REGION} + * @param platformUrl */ public Source(String apiKey, String organizationId, PlatformUrl platformUrl) { - this(apiKey, organizationId, platformUrl, new BackoffOptionsBuilder().build()); - } - - /** - * @param apiKey An apiKey capable of pushing documents and managing sources in a Coveo - * organization. - * @see Manage API Keys. - * @param organizationId The Coveo Organization identifier. - * @param platformUrl The object containing additional information on the URL endpoint. You can - * use the {@link PlatformUrl} when your organization is located in a non-default Coveo - * environement and/or region. When not specified, the default platform URL values will be - * used: {@link PlatformUrl#DEFAULT_ENVIRONMENT} and {@link PlatformUrl#DEFAULT_REGION} - * @param options The configuration options for exponential backoff - */ - public Source( - String apiKey, String organizationId, PlatformUrl platformUrl, BackoffOptions options) { - this.platformClient = new PlatformClient(apiKey, organizationId, platformUrl, options); + this.platformClient = new PlatformClient(apiKey, organizationId, platformUrl); } /** @@ -302,4 +272,4 @@ public HttpResponse pushBinaryToFileContainer( FileContainer fileContainer, byte[] fileAsBytes) throws IOException, InterruptedException { return this.platformClient.pushBinaryToFileContainer(fileContainer, fileAsBytes); } -} +} \ No newline at end of file diff --git a/src/main/java/com/coveo/pushapiclient/StreamService.java b/src/main/java/com/coveo/pushapiclient/StreamService.java index 263bc16f..41dde7e0 100644 --- a/src/main/java/com/coveo/pushapiclient/StreamService.java +++ b/src/main/java/com/coveo/pushapiclient/StreamService.java @@ -37,7 +37,7 @@ public StreamService(StreamEnabledSource source) { * full source rebuild. The {@StreamService} can also be used for an initial catalog upload. * * @param source The source to which you want to send your documents. - * @param options The configuration options for exponential backoff + * @param options The configuration options for exponential backoff. */ public StreamService(StreamEnabledSource source, BackoffOptions options) { String apiKey = source.getApiKey(); From 2a893cd0b7c22409a99b64ce1156485d74645c0d Mon Sep 17 00:00:00 2001 From: David Brooke <38883189+dmbrooke@users.noreply.github.com> Date: Mon, 11 Sep 2023 16:01:00 -0400 Subject: [PATCH 12/12] Linting --- samples/DeleteOneDocument.java | 1 - samples/PushOneDocument.java | 2 +- samples/PushOneDocumentWithMetadata.java | 1 - src/main/java/com/coveo/pushapiclient/Source.java | 2 +- 4 files changed, 2 insertions(+), 4 deletions(-) diff --git a/samples/DeleteOneDocument.java b/samples/DeleteOneDocument.java index a9c32133..e31946d7 100644 --- a/samples/DeleteOneDocument.java +++ b/samples/DeleteOneDocument.java @@ -1,5 +1,4 @@ import com.coveo.pushapiclient.PushSource; -import com.coveo.pushapiclient.Source; import java.io.IOException; import java.net.http.HttpResponse; diff --git a/samples/PushOneDocument.java b/samples/PushOneDocument.java index e5c30aac..4416e684 100644 --- a/samples/PushOneDocument.java +++ b/samples/PushOneDocument.java @@ -1,6 +1,6 @@ import com.coveo.pushapiclient.BackoffOptionsBuilder; import com.coveo.pushapiclient.DocumentBuilder; -import com.coveo.pushapiclient.Source; +import com.coveo.pushapiclient.PushSource; import java.io.IOException; import java.net.http.HttpResponse; diff --git a/samples/PushOneDocumentWithMetadata.java b/samples/PushOneDocumentWithMetadata.java index f006d1ce..1fb9d7b5 100644 --- a/samples/PushOneDocumentWithMetadata.java +++ b/samples/PushOneDocumentWithMetadata.java @@ -2,7 +2,6 @@ import com.coveo.pushapiclient.BackoffOptionsBuilder; import com.coveo.pushapiclient.DocumentBuilder; import com.coveo.pushapiclient.PushSource; -import com.coveo.pushapiclient.Source; import java.io.IOException; import java.net.http.HttpResponse; diff --git a/src/main/java/com/coveo/pushapiclient/Source.java b/src/main/java/com/coveo/pushapiclient/Source.java index c4177e4a..a9099ff8 100644 --- a/src/main/java/com/coveo/pushapiclient/Source.java +++ b/src/main/java/com/coveo/pushapiclient/Source.java @@ -272,4 +272,4 @@ public HttpResponse pushBinaryToFileContainer( FileContainer fileContainer, byte[] fileAsBytes) throws IOException, InterruptedException { return this.platformClient.pushBinaryToFileContainer(fileContainer, fileAsBytes); } -} \ No newline at end of file +}