From cad7ed7d3a100e8b98bece90b3bd732c23069d53 Mon Sep 17 00:00:00 2001 From: davidradl Date: Tue, 21 Oct 2025 14:35:48 +0100 Subject: [PATCH 1/3] http-175 Resolve conflicts Signed-off-by: davidradl --- CHANGELOG.md | 1 + .../lookup/JavaNetHttpPollingClient.java | 5 ++-- .../table/lookup/RequestFactoryBase.java | 6 ----- .../Slf4JHttpLookupPostRequestCallback.java | 24 +++---------------- .../sink/Slf4jHttpPostRequestCallback.java | 24 ++++++++----------- src/test/resources/simpleLogger.properties | 2 ++ 6 files changed, 18 insertions(+), 44 deletions(-) create mode 100644 src/test/resources/simpleLogger.properties diff --git a/CHANGELOG.md b/CHANGELOG.md index 6ee55f9b..dfa98d04 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ## [Unreleased] - Added http 2 support. +- Amend to not log HTTP request response and header values by default. ## [0.22.0] - 2025-10-03 diff --git a/src/main/java/com/getindata/connectors/http/internal/table/lookup/JavaNetHttpPollingClient.java b/src/main/java/com/getindata/connectors/http/internal/table/lookup/JavaNetHttpPollingClient.java index df2c3992..2ccb5480 100644 --- a/src/main/java/com/getindata/connectors/http/internal/table/lookup/JavaNetHttpPollingClient.java +++ b/src/main/java/com/getindata/connectors/http/internal/table/lookup/JavaNetHttpPollingClient.java @@ -210,11 +210,10 @@ private HttpRowDataWrapper processHttpResponse( boolean isError) throws IOException { this.httpPostRequestCallback.call(response, request, "endpoint", Collections.emptyMap()); - + request.getHttpRequest().uri(); var responseBody = response.body(); - log.debug("Received status code [{}] for RestTableSource request with Server response body [{}] ", - response.statusCode(), responseBody); + log.debug("Received status code [{}] for RestTableSource request", response.statusCode()); if (!isError && (StringUtils.isNullOrWhitespaceOnly(responseBody) || ignoreResponse(response))) { return HttpRowDataWrapper.builder() .data(Collections.emptyList()) diff --git a/src/main/java/com/getindata/connectors/http/internal/table/lookup/RequestFactoryBase.java b/src/main/java/com/getindata/connectors/http/internal/table/lookup/RequestFactoryBase.java index 7b822280..e7e3f797 100644 --- a/src/main/java/com/getindata/connectors/http/internal/table/lookup/RequestFactoryBase.java +++ b/src/main/java/com/getindata/connectors/http/internal/table/lookup/RequestFactoryBase.java @@ -4,7 +4,6 @@ import java.net.http.HttpRequest.Builder; import java.util.Arrays; import java.util.Map; -import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import org.apache.flink.annotation.VisibleForTesting; @@ -60,11 +59,6 @@ public RequestFactoryBase( ); this.headersAndValues = HttpHeaderUtils.toHeaderAndValueArray(headerMap); - - log.debug("RequestFactoryBase headersAndValues: " + - Arrays.stream(headersAndValues) - .map(Object::toString) - .collect(Collectors.joining(","))); this.httpRequestTimeOutSeconds = Integer.parseInt( options.getProperties().getProperty( HttpConnectorConfigConstants.LOOKUP_HTTP_TIMEOUT_SECONDS, diff --git a/src/main/java/com/getindata/connectors/http/internal/table/lookup/Slf4JHttpLookupPostRequestCallback.java b/src/main/java/com/getindata/connectors/http/internal/table/lookup/Slf4JHttpLookupPostRequestCallback.java index a1ae1eae..4eebadca 100644 --- a/src/main/java/com/getindata/connectors/http/internal/table/lookup/Slf4JHttpLookupPostRequestCallback.java +++ b/src/main/java/com/getindata/connectors/http/internal/table/lookup/Slf4JHttpLookupPostRequestCallback.java @@ -2,15 +2,11 @@ import java.net.http.HttpRequest; import java.net.http.HttpResponse; -import java.util.List; import java.util.Map; -import java.util.Map.Entry; -import java.util.StringJoiner; import lombok.extern.slf4j.Slf4j; import com.getindata.connectors.http.HttpPostRequestCallback; -import com.getindata.connectors.http.internal.utils.ConfigUtils; /** * A {@link HttpPostRequestCallback} that logs pairs of request and response as INFO level @@ -31,40 +27,26 @@ public void call( Map headerMap) { HttpRequest httpRequest = requestEntry.getHttpRequest(); - StringJoiner headers = new StringJoiner(";"); - - for (Entry> reqHeaders : httpRequest.headers().map().entrySet()) { - StringJoiner values = new StringJoiner(";"); - for (String value : reqHeaders.getValue()) { - values.add(value); - } - String header = reqHeaders.getKey() + ": [" + values + "]"; - headers.add(header); - } if (response == null) { log.warn("Null Http response for request " + httpRequest.uri().toString()); log.info( "Got response for a request.\n Request:\n URL: {}\n " + - "Method: {}\n Headers: {}\n Params/Body: {}\nResponse: null", + "Method: {}\n Params/Body: {}\nResponse: null", httpRequest.uri().toString(), httpRequest.method(), - headers, requestEntry.getLookupQueryInfo() ); } else { log.info( "Got response for a request.\n Request:\n URL: {}\n " + - "Method: {}\n Headers: {}\n Params/Body: {}\nResponse: {}\n Body: {}", + "Method: {}\n Params/Body: {}\nResponse status code: {}\n", httpRequest.uri().toString(), httpRequest.method(), - headers, requestEntry.getLookupQueryInfo(), - response, - response.body().replaceAll(ConfigUtils.UNIVERSAL_NEW_LINE_REGEXP, "") + response.statusCode() ); } - } } diff --git a/src/main/java/com/getindata/connectors/http/internal/table/sink/Slf4jHttpPostRequestCallback.java b/src/main/java/com/getindata/connectors/http/internal/table/sink/Slf4jHttpPostRequestCallback.java index b5f2049d..65d6a61c 100644 --- a/src/main/java/com/getindata/connectors/http/internal/table/sink/Slf4jHttpPostRequestCallback.java +++ b/src/main/java/com/getindata/connectors/http/internal/table/sink/Slf4jHttpPostRequestCallback.java @@ -1,19 +1,17 @@ package com.getindata.connectors.http.internal.table.sink; import java.net.http.HttpResponse; -import java.nio.charset.StandardCharsets; import java.util.Map; -import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import com.getindata.connectors.http.HttpPostRequestCallback; import com.getindata.connectors.http.internal.sink.httpclient.HttpRequest; -import com.getindata.connectors.http.internal.utils.ConfigUtils; /** * A {@link HttpPostRequestCallback} that logs pairs of request and response as INFO level - * logs using Slf4j. + * logs using Slf4j. As the request/response body or header might contain sensitive information, + * we do not log those values. * *

Serving as a default implementation of {@link HttpPostRequestCallback} for * the {@link HttpDynamicSink}. @@ -28,25 +26,23 @@ public void call( String endpointUrl, Map headerMap) { - String requestBody = requestEntry.getElements().stream() - .map(element -> new String(element, StandardCharsets.UTF_8)) - .collect(Collectors.joining()); + // Uncomment if you want to see the requestBody in the log + //String requestBody = requestEntry.getElements().stream() + // .map(element -> new String(element, StandardCharsets.UTF_8)) + // .collect(Collectors.joining()); if (response == null) { log.info( "Got response for a request.\n Request:\n " + - "Method: {}\n Body: {}\n Response: null", - requestEntry.getMethod(), - requestBody + "Method: {}\n Response: null", + requestEntry.getMethod() ); } else { log.info( "Got response for a request.\n Request:\n " + - "Method: {}\n Body: {}\n Response: {}\n Body: {}", + "Method: {}\n Response status code: {}\n ", requestEntry.method, - requestBody, - response, - response.body().replaceAll(ConfigUtils.UNIVERSAL_NEW_LINE_REGEXP, "") + response.statusCode() ); } } diff --git a/src/test/resources/simpleLogger.properties b/src/test/resources/simpleLogger.properties new file mode 100644 index 00000000..3e7cd429 --- /dev/null +++ b/src/test/resources/simpleLogger.properties @@ -0,0 +1,2 @@ +org.slf4j.simpleLogger.defaultLogLevel=INFO +org.slf4j.simpleLogger.log.com.getindata.connectors.http.internal.table.lookup.RequestAndResponseLogger=DEBUG From 34f56eb07f7dbf1f7579d33e8149ed11c7c1ce4f Mon Sep 17 00:00:00 2001 From: davidradl Date: Tue, 21 Oct 2025 14:34:18 +0100 Subject: [PATCH 2/3] http-175 Address review comments Signed-off-by: davidradl --- CHANGELOG.md | 1 + README.md | 7 +++---- .../internal/table/lookup/JavaNetHttpPollingClient.java | 1 - 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dfa98d04..39d62d82 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ - Added http 2 support. - Amend to not log HTTP request response and header values by default. +- Added http 2 support. ## [0.22.0] - 2025-10-03 diff --git a/README.md b/README.md index ab9e72d8..e4bfe9ef 100644 --- a/README.md +++ b/README.md @@ -428,10 +428,9 @@ that implements interface `HttpPostRequestCallbackFactory` to creat of class `CustomHttpSinkPostRequestCallbackFactory` in `resources/META-INF/services/org.apache.flink.table.factories.Factory` file and then reference identifier `rest-sink-logger` in the HttpSink DDL property field `gid.connector.http.sink.request-callback`. - A default implementation that logs those pairs as *INFO* level logs using Slf4j -([Slf4jHttpPostRequestCallback](src/main/java/com/getindata/connectors/http/internal/table/sink/Slf4jHttpPostRequestCallback.java)) -is provided. - +A default implementation that logs those pairs as *INFO* level logs using Slf4j ([Slf4jHttpPostRequestCallback](src/main/java/com/getindata/connectors/http/internal/table/sink/Slf4jHttpPostRequestCallback.java)) is provided. +If you would like to log more http content (that maybe contain sensitive information), then you can provide a customized version +of this callback; for inspiration on how to customize in this way, look back in the git history of this file. - Http Lookup Source processes responses that it gets from the HTTP endpoint along their respective requests. One can customize the behaviour of the additional stage of processing done by Table Function API by implementing diff --git a/src/main/java/com/getindata/connectors/http/internal/table/lookup/JavaNetHttpPollingClient.java b/src/main/java/com/getindata/connectors/http/internal/table/lookup/JavaNetHttpPollingClient.java index 2ccb5480..81a75674 100644 --- a/src/main/java/com/getindata/connectors/http/internal/table/lookup/JavaNetHttpPollingClient.java +++ b/src/main/java/com/getindata/connectors/http/internal/table/lookup/JavaNetHttpPollingClient.java @@ -210,7 +210,6 @@ private HttpRowDataWrapper processHttpResponse( boolean isError) throws IOException { this.httpPostRequestCallback.call(response, request, "endpoint", Collections.emptyMap()); - request.getHttpRequest().uri(); var responseBody = response.body(); log.debug("Received status code [{}] for RestTableSource request", response.statusCode()); From 2cffa76a1d7484afab9b562f961664f05bc72d87 Mon Sep 17 00:00:00 2001 From: davidradl Date: Tue, 21 Oct 2025 14:37:47 +0100 Subject: [PATCH 3/3] http-175 Fixup change log Signed-off-by: davidradl --- CHANGELOG.md | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 39d62d82..3c96d151 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,6 @@ ## [Unreleased] -- Added http 2 support. - Amend to not log HTTP request response and header values by default. - Added http 2 support.