From ca1de271f3075b381a46ffb7bee8cfb5d8909161 Mon Sep 17 00:00:00 2001 From: Ashutosh Gupta Date: Sun, 24 Apr 2022 00:20:41 +0100 Subject: [PATCH 01/12] HADOOP-18069. okhttp@2.7.5 to 4.9.3 --- hadoop-hdfs-project/hadoop-hdfs-client/pom.xml | 13 ++++++++++++- ...onfRefreshTokenBasedAccessTokenProvider.java | 17 ++++++++--------- .../CredentialBasedAccessTokenProvider.java | 17 ++++++++--------- .../hadoop/hdfs/web/oauth2/OAuth2Constants.java | 2 +- hadoop-tools/hadoop-azure-datalake/pom.xml | 12 ------------ 5 files changed, 29 insertions(+), 32 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml b/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml index c4e65ef811dbf..e126ea053b281 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml +++ b/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml @@ -35,8 +35,19 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd"> - com.squareup.okhttp + com.squareup.okhttp3 okhttp + 4.9.3 + + + org.jetbrains.kotlin + kotlin-stdlib + + + org.jetbrains.kotlin + kotlin-stdlib-common + + org.apache.hadoop diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/oauth2/ConfRefreshTokenBasedAccessTokenProvider.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/oauth2/ConfRefreshTokenBasedAccessTokenProvider.java index 3e3fbfbd91380..82374a0d7556e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/oauth2/ConfRefreshTokenBasedAccessTokenProvider.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/oauth2/ConfRefreshTokenBasedAccessTokenProvider.java @@ -18,10 +18,10 @@ */ package org.apache.hadoop.hdfs.web.oauth2; -import com.squareup.okhttp.OkHttpClient; -import com.squareup.okhttp.Request; -import com.squareup.okhttp.RequestBody; -import com.squareup.okhttp.Response; +import okhttp3.OkHttpClient; +import okhttp3.Request; +import okhttp3.RequestBody; +import okhttp3.Response; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; @@ -103,11 +103,10 @@ public synchronized String getAccessToken() throws IOException { void refresh() throws IOException { try { - OkHttpClient client = new OkHttpClient(); - client.setConnectTimeout(URLConnectionFactory.DEFAULT_SOCKET_TIMEOUT, - TimeUnit.MILLISECONDS); - client.setReadTimeout(URLConnectionFactory.DEFAULT_SOCKET_TIMEOUT, - TimeUnit.MILLISECONDS); + OkHttpClient client = new OkHttpClient.Builder() + .connectTimeout(URLConnectionFactory.DEFAULT_SOCKET_TIMEOUT, TimeUnit.MILLISECONDS) + .readTimeout(URLConnectionFactory.DEFAULT_SOCKET_TIMEOUT, TimeUnit.MILLISECONDS) + .build(); String bodyString = Utils.postBody(GRANT_TYPE, REFRESH_TOKEN, REFRESH_TOKEN, refreshToken, diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/oauth2/CredentialBasedAccessTokenProvider.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/oauth2/CredentialBasedAccessTokenProvider.java index bfd7055990e81..dc6b9f410d4b6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/oauth2/CredentialBasedAccessTokenProvider.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/oauth2/CredentialBasedAccessTokenProvider.java @@ -18,10 +18,10 @@ */ package org.apache.hadoop.hdfs.web.oauth2; -import com.squareup.okhttp.OkHttpClient; -import com.squareup.okhttp.Request; -import com.squareup.okhttp.RequestBody; -import com.squareup.okhttp.Response; +import okhttp3.OkHttpClient; +import okhttp3.Request; +import okhttp3.RequestBody; +import okhttp3.Response; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; @@ -97,11 +97,10 @@ public synchronized String getAccessToken() throws IOException { void refresh() throws IOException { try { - OkHttpClient client = new OkHttpClient(); - client.setConnectTimeout(URLConnectionFactory.DEFAULT_SOCKET_TIMEOUT, - TimeUnit.MILLISECONDS); - client.setReadTimeout(URLConnectionFactory.DEFAULT_SOCKET_TIMEOUT, - TimeUnit.MILLISECONDS); + OkHttpClient client = new OkHttpClient.Builder() + .connectTimeout(URLConnectionFactory.DEFAULT_SOCKET_TIMEOUT, TimeUnit.MILLISECONDS) + .readTimeout(URLConnectionFactory.DEFAULT_SOCKET_TIMEOUT, TimeUnit.MILLISECONDS) + .build(); String bodyString = Utils.postBody(CLIENT_SECRET, getCredential(), GRANT_TYPE, CLIENT_CREDENTIALS, diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/oauth2/OAuth2Constants.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/oauth2/OAuth2Constants.java index 3f091c2945bcc..2f28b65e40e92 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/oauth2/OAuth2Constants.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/oauth2/OAuth2Constants.java @@ -18,7 +18,7 @@ */ package org.apache.hadoop.hdfs.web.oauth2; -import com.squareup.okhttp.MediaType; +import okhttp3.MediaType; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; diff --git a/hadoop-tools/hadoop-azure-datalake/pom.xml b/hadoop-tools/hadoop-azure-datalake/pom.xml index 446e2957b60ed..04cb92916fc1f 100644 --- a/hadoop-tools/hadoop-azure-datalake/pom.xml +++ b/hadoop-tools/hadoop-azure-datalake/pom.xml @@ -118,12 +118,6 @@ hadoop-common provided - - com.squareup.okhttp - okhttp - ${okhttp.version} - test - junit junit @@ -141,12 +135,6 @@ test test-jar - - com.squareup.okhttp - mockwebserver - ${okhttp.version} - test - org.apache.hadoop From 85f912e7409def181ef1ade961ec85773c3e0fe6 Mon Sep 17 00:00:00 2001 From: Ashutosh Gupta Date: Sun, 24 Apr 2022 17:08:53 +0100 Subject: [PATCH 02/12] Updating dependencies --- .../hadoop-hdfs-client/pom.xml | 19 +++++------ hadoop-project/pom.xml | 34 +++++++++++++++++++ 2 files changed, 42 insertions(+), 11 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml b/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml index e126ea053b281..9bb0932d32867 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml +++ b/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml @@ -37,17 +37,14 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd"> com.squareup.okhttp3 okhttp - 4.9.3 - - - org.jetbrains.kotlin - kotlin-stdlib - - - org.jetbrains.kotlin - kotlin-stdlib-common - - + + + org.jetbrains.kotlin + kotlin-stdlib + + + org.jetbrains.kotlin + kotlin-stdlib-common org.apache.hadoop diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index eaa91c2c90001..499ac38153a29 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -136,6 +136,9 @@ 4.0.3 6.2.1.jre7 2.7.5 + 4.9.3 + 1.4.10 + 1.4.10 2.0.6.1 5.2.0 2.2.21 @@ -227,6 +230,37 @@ okhttp ${okhttp.version} + + com.squareup.okhttp3 + okhttp + ${okhttp3.version} + + + org.jetbrains.kotlin + kotlin-stdlib + + + org.jetbrains.kotlin + kotlin-stdlib-common + + + + + org.jetbrains.kotlin + kotlin-stdlib + ${kotlin-stdlib.verion} + + + org.jetbrains + annotations + + + + + org.jetbrains.kotlin + kotlin-stdlib-common + ${kotlin-stdlib-common.version} + com.squareup.okhttp3 mockwebserver From 1653f4f01ddfa084bb193b38e0061a3d71ff530d Mon Sep 17 00:00:00 2001 From: Ashutosh Gupta Date: Sun, 24 Apr 2022 22:04:21 +0100 Subject: [PATCH 03/12] Add null check and successfull responsebody check --- ...ConfRefreshTokenBasedAccessTokenProvider.java | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/oauth2/ConfRefreshTokenBasedAccessTokenProvider.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/oauth2/ConfRefreshTokenBasedAccessTokenProvider.java index 82374a0d7556e..1f7cbba910a8a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/oauth2/ConfRefreshTokenBasedAccessTokenProvider.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/oauth2/ConfRefreshTokenBasedAccessTokenProvider.java @@ -120,18 +120,24 @@ void refresh() throws IOException { .build(); Response responseBody = client.newCall(request).execute(); + if (!responseBody.isSuccessful()) throw new IOException("Unexpected code " + responseBody); + if (responseBody.code() != HttpStatus.SC_OK) { throw new IllegalArgumentException("Received invalid http response: " + responseBody.code() + ", text = " + responseBody.toString()); } - Map response = JsonSerialization.mapReader().readValue( - responseBody.body().string()); + Map response = null; + if (responseBody.body() != null) { + response = JsonSerialization.mapReader().readValue( + responseBody.body().string()); + + String newExpiresIn = response.get(EXPIRES_IN).toString(); + accessTokenTimer.setExpiresIn(newExpiresIn); - String newExpiresIn = response.get(EXPIRES_IN).toString(); - accessTokenTimer.setExpiresIn(newExpiresIn); + accessToken = response.get(ACCESS_TOKEN).toString(); + } - accessToken = response.get(ACCESS_TOKEN).toString(); } catch (Exception e) { throw new IOException("Exception while refreshing access token", e); } From 4d002c431e91b27e84d233bfe09d3468e3482ad3 Mon Sep 17 00:00:00 2001 From: Ashutosh Gupta Date: Mon, 25 Apr 2022 03:38:06 +0100 Subject: [PATCH 04/12] Style check and NPE warning fixing --- ...fRefreshTokenBasedAccessTokenProvider.java | 35 +++++++++---------- 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/oauth2/ConfRefreshTokenBasedAccessTokenProvider.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/oauth2/ConfRefreshTokenBasedAccessTokenProvider.java index 1f7cbba910a8a..88d10b6972d38 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/oauth2/ConfRefreshTokenBasedAccessTokenProvider.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/oauth2/ConfRefreshTokenBasedAccessTokenProvider.java @@ -32,6 +32,7 @@ import java.io.IOException; import java.util.Map; +import java.util.Objects; import java.util.concurrent.TimeUnit; import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.OAUTH_CLIENT_ID_KEY; @@ -103,39 +104,37 @@ public synchronized String getAccessToken() throws IOException { void refresh() throws IOException { try { - OkHttpClient client = new OkHttpClient.Builder() - .connectTimeout(URLConnectionFactory.DEFAULT_SOCKET_TIMEOUT, TimeUnit.MILLISECONDS) + OkHttpClient client = + new OkHttpClient.Builder().connectTimeout(URLConnectionFactory.DEFAULT_SOCKET_TIMEOUT, + TimeUnit.MILLISECONDS) .readTimeout(URLConnectionFactory.DEFAULT_SOCKET_TIMEOUT, TimeUnit.MILLISECONDS) .build(); - String bodyString = Utils.postBody(GRANT_TYPE, REFRESH_TOKEN, - REFRESH_TOKEN, refreshToken, - CLIENT_ID, clientId); + String bodyString = + Utils.postBody(GRANT_TYPE, REFRESH_TOKEN, REFRESH_TOKEN, refreshToken, CLIENT_ID, + clientId); RequestBody body = RequestBody.create(URLENCODED, bodyString); - Request request = new Request.Builder() - .url(refreshURL) - .post(body) - .build(); + Request request = new Request.Builder().url(refreshURL).post(body).build(); Response responseBody = client.newCall(request).execute(); - if (!responseBody.isSuccessful()) throw new IOException("Unexpected code " + responseBody); + if (responseBody != null) { - if (responseBody.code() != HttpStatus.SC_OK) { - throw new IllegalArgumentException("Received invalid http response: " - + responseBody.code() + ", text = " + responseBody.toString()); - } + if (responseBody.code() != HttpStatus.SC_OK) { + throw new IllegalArgumentException( + "Received invalid http response: " + responseBody.code() + ", text = " + + responseBody); + } - Map response = null; - if (responseBody.body() != null) { - response = JsonSerialization.mapReader().readValue( - responseBody.body().string()); + Map response = JsonSerialization.mapReader() + .readValue(Objects.requireNonNull(responseBody.body()).string()); String newExpiresIn = response.get(EXPIRES_IN).toString(); accessTokenTimer.setExpiresIn(newExpiresIn); accessToken = response.get(ACCESS_TOKEN).toString(); + } } catch (Exception e) { From 9d7ef78778661e103c31159a91bc206511e8fcb9 Mon Sep 17 00:00:00 2001 From: Ashutosh Gupta Date: Mon, 25 Apr 2022 15:04:02 +0100 Subject: [PATCH 05/12] formatting code and update LICENSE-binary --- LICENSE-binary | 1 + ...fRefreshTokenBasedAccessTokenProvider.java | 29 ++++++++----------- hadoop-project/pom.xml | 5 ---- 3 files changed, 13 insertions(+), 22 deletions(-) diff --git a/LICENSE-binary b/LICENSE-binary index ef0db1bb6a4d7..e08abcaa6fe19 100644 --- a/LICENSE-binary +++ b/LICENSE-binary @@ -242,6 +242,7 @@ com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava com.microsoft.azure:azure-storage:7.0.0 com.nimbusds:nimbus-jose-jwt:9.8.1 com.squareup.okhttp:okhttp:2.7.5 +com.squareup.okhttp3:okhttp:4.9.3 com.squareup.okio:okio:1.6.0 com.zaxxer:HikariCP:4.0.3 commons-beanutils:commons-beanutils:1.9.3 diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/oauth2/ConfRefreshTokenBasedAccessTokenProvider.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/oauth2/ConfRefreshTokenBasedAccessTokenProvider.java index 88d10b6972d38..9a331d16792e5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/oauth2/ConfRefreshTokenBasedAccessTokenProvider.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/oauth2/ConfRefreshTokenBasedAccessTokenProvider.java @@ -32,7 +32,6 @@ import java.io.IOException; import java.util.Map; -import java.util.Objects; import java.util.concurrent.TimeUnit; import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.OAUTH_CLIENT_ID_KEY; @@ -104,37 +103,33 @@ public synchronized String getAccessToken() throws IOException { void refresh() throws IOException { try { - OkHttpClient client = - new OkHttpClient.Builder().connectTimeout(URLConnectionFactory.DEFAULT_SOCKET_TIMEOUT, - TimeUnit.MILLISECONDS) - .readTimeout(URLConnectionFactory.DEFAULT_SOCKET_TIMEOUT, TimeUnit.MILLISECONDS) - .build(); + OkHttpClient client = new OkHttpClient.Builder().connectTimeout(URLConnectionFactory.DEFAULT_SOCKET_TIMEOUT, + TimeUnit.MILLISECONDS) + .readTimeout(URLConnectionFactory.DEFAULT_SOCKET_TIMEOUT, + TimeUnit.MILLISECONDS) + .build(); - String bodyString = - Utils.postBody(GRANT_TYPE, REFRESH_TOKEN, REFRESH_TOKEN, refreshToken, CLIENT_ID, - clientId); + String bodyString = Utils.postBody(GRANT_TYPE, REFRESH_TOKEN, REFRESH_TOKEN, refreshToken, CLIENT_ID, clientId); RequestBody body = RequestBody.create(URLENCODED, bodyString); Request request = new Request.Builder().url(refreshURL).post(body).build(); - Response responseBody = client.newCall(request).execute(); - - if (responseBody != null) { - + try (Response responseBody = client.newCall(request).execute()) { + if (!responseBody.isSuccessful()) { + throw new IOException("Unexpected code " + responseBody); + } if (responseBody.code() != HttpStatus.SC_OK) { throw new IllegalArgumentException( "Received invalid http response: " + responseBody.code() + ", text = " - + responseBody); + + responseBody.toString()); } - Map response = JsonSerialization.mapReader() - .readValue(Objects.requireNonNull(responseBody.body()).string()); + Map response = JsonSerialization.mapReader().readValue(responseBody.body().string()); String newExpiresIn = response.get(EXPIRES_IN).toString(); accessTokenTimer.setExpiresIn(newExpiresIn); accessToken = response.get(ACCESS_TOKEN).toString(); - } } catch (Exception e) { diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index 499ac38153a29..b88defe3bae1a 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -225,11 +225,6 @@ - - com.squareup.okhttp - okhttp - ${okhttp.version} - com.squareup.okhttp3 okhttp From 944725f465bc1b4b8361e2b35808faec2d8243d0 Mon Sep 17 00:00:00 2001 From: Ashutosh Gupta Date: Mon, 25 Apr 2022 15:17:32 +0100 Subject: [PATCH 06/12] fomatting code --- ...ConfRefreshTokenBasedAccessTokenProvider.java | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/oauth2/ConfRefreshTokenBasedAccessTokenProvider.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/oauth2/ConfRefreshTokenBasedAccessTokenProvider.java index 9a331d16792e5..4b66c5f35e7ac 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/oauth2/ConfRefreshTokenBasedAccessTokenProvider.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/oauth2/ConfRefreshTokenBasedAccessTokenProvider.java @@ -103,13 +103,15 @@ public synchronized String getAccessToken() throws IOException { void refresh() throws IOException { try { - OkHttpClient client = new OkHttpClient.Builder().connectTimeout(URLConnectionFactory.DEFAULT_SOCKET_TIMEOUT, - TimeUnit.MILLISECONDS) - .readTimeout(URLConnectionFactory.DEFAULT_SOCKET_TIMEOUT, - TimeUnit.MILLISECONDS) - .build(); - - String bodyString = Utils.postBody(GRANT_TYPE, REFRESH_TOKEN, REFRESH_TOKEN, refreshToken, CLIENT_ID, clientId); + OkHttpClient client = + new OkHttpClient.Builder().connectTimeout(URLConnectionFactory.DEFAULT_SOCKET_TIMEOUT, + TimeUnit.MILLISECONDS) + .readTimeout(URLConnectionFactory.DEFAULT_SOCKET_TIMEOUT, TimeUnit.MILLISECONDS) + .build(); + + String bodyString = Utils.postBody(GRANT_TYPE, REFRESH_TOKEN, + REFRESH_TOKEN, refreshToken, + CLIENT_ID, clientId); RequestBody body = RequestBody.create(URLENCODED, bodyString); From 5b7212edd4c3dddd148b3558b5daa06fcae56852 Mon Sep 17 00:00:00 2001 From: Ashutosh Gupta Date: Tue, 26 Apr 2022 13:34:30 +0100 Subject: [PATCH 07/12] Uodating com.squareup.okhttp3:mockwebserver to 4.9.3 and use try-with-resources for OkHttpClient --- ...fRefreshTokenBasedAccessTokenProvider.java | 56 +++++++++---------- .../CredentialBasedAccessTokenProvider.java | 35 ++++++------ hadoop-project/pom.xml | 2 +- 3 files changed, 43 insertions(+), 50 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/oauth2/ConfRefreshTokenBasedAccessTokenProvider.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/oauth2/ConfRefreshTokenBasedAccessTokenProvider.java index 4b66c5f35e7ac..c50ac4bf5f5bb 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/oauth2/ConfRefreshTokenBasedAccessTokenProvider.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/oauth2/ConfRefreshTokenBasedAccessTokenProvider.java @@ -102,38 +102,34 @@ public synchronized String getAccessToken() throws IOException { } void refresh() throws IOException { - try { - OkHttpClient client = - new OkHttpClient.Builder().connectTimeout(URLConnectionFactory.DEFAULT_SOCKET_TIMEOUT, - TimeUnit.MILLISECONDS) - .readTimeout(URLConnectionFactory.DEFAULT_SOCKET_TIMEOUT, TimeUnit.MILLISECONDS) - .build(); - - String bodyString = Utils.postBody(GRANT_TYPE, REFRESH_TOKEN, - REFRESH_TOKEN, refreshToken, - CLIENT_ID, clientId); - - RequestBody body = RequestBody.create(URLENCODED, bodyString); - - Request request = new Request.Builder().url(refreshURL).post(body).build(); - try (Response responseBody = client.newCall(request).execute()) { - if (!responseBody.isSuccessful()) { - throw new IOException("Unexpected code " + responseBody); - } - if (responseBody.code() != HttpStatus.SC_OK) { - throw new IllegalArgumentException( - "Received invalid http response: " + responseBody.code() + ", text = " - + responseBody.toString()); - } - - Map response = JsonSerialization.mapReader().readValue(responseBody.body().string()); - - String newExpiresIn = response.get(EXPIRES_IN).toString(); - accessTokenTimer.setExpiresIn(newExpiresIn); - - accessToken = response.get(ACCESS_TOKEN).toString(); + OkHttpClient client = + new OkHttpClient.Builder().connectTimeout(URLConnectionFactory.DEFAULT_SOCKET_TIMEOUT, + TimeUnit.MILLISECONDS) + .readTimeout(URLConnectionFactory.DEFAULT_SOCKET_TIMEOUT, TimeUnit.MILLISECONDS) + .build(); + + String bodyString = + Utils.postBody(GRANT_TYPE, REFRESH_TOKEN, REFRESH_TOKEN, refreshToken, CLIENT_ID, clientId); + + RequestBody body = RequestBody.create(URLENCODED, bodyString); + + Request request = new Request.Builder().url(refreshURL).post(body).build(); + try (Response responseBody = client.newCall(request).execute()) { + if (!responseBody.isSuccessful()) { + throw new IOException("Unexpected code " + responseBody); } + if (responseBody.code() != HttpStatus.SC_OK) { + throw new IllegalArgumentException( + "Received invalid http response: " + responseBody.code() + ", text = " + + responseBody.toString()); + } + + Map response = JsonSerialization.mapReader().readValue(responseBody.body().string()); + + String newExpiresIn = response.get(EXPIRES_IN).toString(); + accessTokenTimer.setExpiresIn(newExpiresIn); + accessToken = response.get(ACCESS_TOKEN).toString(); } catch (Exception e) { throw new IOException("Exception while refreshing access token", e); } diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/oauth2/CredentialBasedAccessTokenProvider.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/oauth2/CredentialBasedAccessTokenProvider.java index dc6b9f410d4b6..2508c66525d51 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/oauth2/CredentialBasedAccessTokenProvider.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/oauth2/CredentialBasedAccessTokenProvider.java @@ -96,24 +96,22 @@ public synchronized String getAccessToken() throws IOException { } void refresh() throws IOException { - try { - OkHttpClient client = new OkHttpClient.Builder() - .connectTimeout(URLConnectionFactory.DEFAULT_SOCKET_TIMEOUT, TimeUnit.MILLISECONDS) - .readTimeout(URLConnectionFactory.DEFAULT_SOCKET_TIMEOUT, TimeUnit.MILLISECONDS) - .build(); - - String bodyString = Utils.postBody(CLIENT_SECRET, getCredential(), - GRANT_TYPE, CLIENT_CREDENTIALS, - CLIENT_ID, clientId); - - RequestBody body = RequestBody.create(URLENCODED, bodyString); - - Request request = new Request.Builder() - .url(refreshURL) - .post(body) - .build(); - Response responseBody = client.newCall(request).execute(); - + OkHttpClient client = new OkHttpClient.Builder() + .connectTimeout(URLConnectionFactory.DEFAULT_SOCKET_TIMEOUT, TimeUnit.MILLISECONDS) + .readTimeout(URLConnectionFactory.DEFAULT_SOCKET_TIMEOUT, TimeUnit.MILLISECONDS) + .build(); + + String bodyString = Utils.postBody(CLIENT_SECRET, getCredential(), + GRANT_TYPE, CLIENT_CREDENTIALS, + CLIENT_ID, clientId); + + RequestBody body = RequestBody.create(URLENCODED, bodyString); + + Request request = new Request.Builder() + .url(refreshURL) + .post(body) + .build(); + try (Response responseBody = client.newCall(request).execute()) { if (responseBody.code() != HttpStatus.SC_OK) { throw new IllegalArgumentException("Received invalid http response: " + responseBody.code() + ", text = " + responseBody.toString()); @@ -126,7 +124,6 @@ void refresh() throws IOException { timer.setExpiresIn(newExpiresIn); accessToken = response.get(ACCESS_TOKEN).toString(); - } catch (Exception e) { throw new IOException("Unable to obtain access token from credential", e); } diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index b88defe3bae1a..82069c094b246 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -259,7 +259,7 @@ com.squareup.okhttp3 mockwebserver - 3.7.0 + 4.9.3 test From cb4a45e8f49cbbd4bfbe87c3da7983487ac56868 Mon Sep 17 00:00:00 2001 From: Ashutosh Gupta Date: Tue, 26 Apr 2022 21:18:18 +0100 Subject: [PATCH 08/12] Handling depreciating create method and adding additional check --- .../oauth2/ConfRefreshTokenBasedAccessTokenProvider.java | 2 +- .../hdfs/web/oauth2/CredentialBasedAccessTokenProvider.java | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/oauth2/ConfRefreshTokenBasedAccessTokenProvider.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/oauth2/ConfRefreshTokenBasedAccessTokenProvider.java index c50ac4bf5f5bb..59c5c59a4b2ca 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/oauth2/ConfRefreshTokenBasedAccessTokenProvider.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/oauth2/ConfRefreshTokenBasedAccessTokenProvider.java @@ -111,7 +111,7 @@ void refresh() throws IOException { String bodyString = Utils.postBody(GRANT_TYPE, REFRESH_TOKEN, REFRESH_TOKEN, refreshToken, CLIENT_ID, clientId); - RequestBody body = RequestBody.create(URLENCODED, bodyString); + RequestBody body = RequestBody.create(bodyString, URLENCODED); Request request = new Request.Builder().url(refreshURL).post(body).build(); try (Response responseBody = client.newCall(request).execute()) { diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/oauth2/CredentialBasedAccessTokenProvider.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/oauth2/CredentialBasedAccessTokenProvider.java index 2508c66525d51..be5d85d73c60d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/oauth2/CredentialBasedAccessTokenProvider.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/oauth2/CredentialBasedAccessTokenProvider.java @@ -105,13 +105,17 @@ void refresh() throws IOException { GRANT_TYPE, CLIENT_CREDENTIALS, CLIENT_ID, clientId); - RequestBody body = RequestBody.create(URLENCODED, bodyString); + RequestBody body = RequestBody.create(bodyString, URLENCODED); Request request = new Request.Builder() .url(refreshURL) .post(body) .build(); try (Response responseBody = client.newCall(request).execute()) { + if (!responseBody.isSuccessful()) { + throw new IOException("Unexpected code " + responseBody); + } + if (responseBody.code() != HttpStatus.SC_OK) { throw new IllegalArgumentException("Received invalid http response: " + responseBody.code() + ", text = " + responseBody.toString()); From 073a442143c213092a76a41e5cf3fc3e7c04cfb6 Mon Sep 17 00:00:00 2001 From: Ashutosh Gupta Date: Fri, 29 Apr 2022 01:11:01 +0100 Subject: [PATCH 09/12] removing com.squareup.okhttp:okhttp:2.7.5 --- LICENSE-binary | 1 - 1 file changed, 1 deletion(-) diff --git a/LICENSE-binary b/LICENSE-binary index e08abcaa6fe19..f6630b53cb62a 100644 --- a/LICENSE-binary +++ b/LICENSE-binary @@ -241,7 +241,6 @@ com.google.guava:guava:27.0-jre com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava com.microsoft.azure:azure-storage:7.0.0 com.nimbusds:nimbus-jose-jwt:9.8.1 -com.squareup.okhttp:okhttp:2.7.5 com.squareup.okhttp3:okhttp:4.9.3 com.squareup.okio:okio:1.6.0 com.zaxxer:HikariCP:4.0.3 From cd1c10bc52133df012aedd241fc223f73ba949a5 Mon Sep 17 00:00:00 2001 From: Ashutosh Gupta Date: Tue, 3 May 2022 12:55:56 +0100 Subject: [PATCH 10/12] Adding bug exception in findbugsExcludeFile.xml and improving naming conventions --- .../dev-support/findbugsExcludeFile.xml | 13 +++++++++ ...fRefreshTokenBasedAccessTokenProvider.java | 27 ++++++++++--------- .../CredentialBasedAccessTokenProvider.java | 27 ++++++++++--------- 3 files changed, 41 insertions(+), 26 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/dev-support/findbugsExcludeFile.xml b/hadoop-hdfs-project/hadoop-hdfs-client/dev-support/findbugsExcludeFile.xml index c96b3a99bd1c4..508388aa4812e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/dev-support/findbugsExcludeFile.xml +++ b/hadoop-hdfs-project/hadoop-hdfs-client/dev-support/findbugsExcludeFile.xml @@ -94,4 +94,17 @@ + + + + + + + + + + + + + diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/oauth2/ConfRefreshTokenBasedAccessTokenProvider.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/oauth2/ConfRefreshTokenBasedAccessTokenProvider.java index 59c5c59a4b2ca..e944e8c1c8d77 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/oauth2/ConfRefreshTokenBasedAccessTokenProvider.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/oauth2/ConfRefreshTokenBasedAccessTokenProvider.java @@ -18,10 +18,15 @@ */ package org.apache.hadoop.hdfs.web.oauth2; +import java.io.IOException; +import java.util.Map; +import java.util.concurrent.TimeUnit; + import okhttp3.OkHttpClient; import okhttp3.Request; import okhttp3.RequestBody; import okhttp3.Response; + import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; @@ -30,10 +35,6 @@ import org.apache.hadoop.util.Timer; import org.apache.http.HttpStatus; -import java.io.IOException; -import java.util.Map; -import java.util.concurrent.TimeUnit; - import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.OAUTH_CLIENT_ID_KEY; import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.OAUTH_REFRESH_URL_KEY; import static org.apache.hadoop.hdfs.web.oauth2.OAuth2Constants.ACCESS_TOKEN; @@ -114,22 +115,22 @@ void refresh() throws IOException { RequestBody body = RequestBody.create(bodyString, URLENCODED); Request request = new Request.Builder().url(refreshURL).post(body).build(); - try (Response responseBody = client.newCall(request).execute()) { - if (!responseBody.isSuccessful()) { - throw new IOException("Unexpected code " + responseBody); + try (Response response = client.newCall(request).execute()) { + if (!response.isSuccessful()) { + throw new IOException("Unexpected code " + response); } - if (responseBody.code() != HttpStatus.SC_OK) { + if (response.code() != HttpStatus.SC_OK) { throw new IllegalArgumentException( - "Received invalid http response: " + responseBody.code() + ", text = " - + responseBody.toString()); + "Received invalid http response: " + response.code() + ", text = " + + response.toString()); } - Map response = JsonSerialization.mapReader().readValue(responseBody.body().string()); + Map responseBody = JsonSerialization.mapReader().readValue(response.body().string()); - String newExpiresIn = response.get(EXPIRES_IN).toString(); + String newExpiresIn = responseBody.get(EXPIRES_IN).toString(); accessTokenTimer.setExpiresIn(newExpiresIn); - accessToken = response.get(ACCESS_TOKEN).toString(); + accessToken = responseBody.get(ACCESS_TOKEN).toString(); } catch (Exception e) { throw new IOException("Exception while refreshing access token", e); } diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/oauth2/CredentialBasedAccessTokenProvider.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/oauth2/CredentialBasedAccessTokenProvider.java index be5d85d73c60d..25ceb8846092b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/oauth2/CredentialBasedAccessTokenProvider.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/oauth2/CredentialBasedAccessTokenProvider.java @@ -18,10 +18,15 @@ */ package org.apache.hadoop.hdfs.web.oauth2; +import java.io.IOException; +import java.util.Map; +import java.util.concurrent.TimeUnit; + import okhttp3.OkHttpClient; import okhttp3.Request; import okhttp3.RequestBody; import okhttp3.Response; + import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; @@ -30,10 +35,6 @@ import org.apache.hadoop.util.Timer; import org.apache.http.HttpStatus; -import java.io.IOException; -import java.util.Map; -import java.util.concurrent.TimeUnit; - import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.OAUTH_CLIENT_ID_KEY; import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.OAUTH_REFRESH_URL_KEY; import static org.apache.hadoop.hdfs.web.oauth2.OAuth2Constants.ACCESS_TOKEN; @@ -111,23 +112,23 @@ void refresh() throws IOException { .url(refreshURL) .post(body) .build(); - try (Response responseBody = client.newCall(request).execute()) { - if (!responseBody.isSuccessful()) { - throw new IOException("Unexpected code " + responseBody); + try (Response response = client.newCall(request).execute()) { + if (!response.isSuccessful()) { + throw new IOException("Unexpected code " + response); } - if (responseBody.code() != HttpStatus.SC_OK) { + if (response.code() != HttpStatus.SC_OK) { throw new IllegalArgumentException("Received invalid http response: " - + responseBody.code() + ", text = " + responseBody.toString()); + + response.code() + ", text = " + response.toString()); } - Map response = JsonSerialization.mapReader().readValue( - responseBody.body().string()); + Map responseBody = JsonSerialization.mapReader().readValue( + response.body().string()); - String newExpiresIn = response.get(EXPIRES_IN).toString(); + String newExpiresIn = responseBody.get(EXPIRES_IN).toString(); timer.setExpiresIn(newExpiresIn); - accessToken = response.get(ACCESS_TOKEN).toString(); + accessToken = responseBody.get(ACCESS_TOKEN).toString(); } catch (Exception e) { throw new IOException("Unable to obtain access token from credential", e); } From 3de6058af30a394576572d19198ed47522834d52 Mon Sep 17 00:00:00 2001 From: Ashutosh Gupta Date: Sun, 8 May 2022 03:08:05 +0100 Subject: [PATCH 11/12] fixed build errors --- hadoop-client-modules/hadoop-client/pom.xml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/hadoop-client-modules/hadoop-client/pom.xml b/hadoop-client-modules/hadoop-client/pom.xml index dced359b286d9..bc5451f2c5ccc 100644 --- a/hadoop-client-modules/hadoop-client/pom.xml +++ b/hadoop-client-modules/hadoop-client/pom.xml @@ -114,6 +114,18 @@ org.eclipse.jetty jetty-server + + org.jetbrains.kotlin + kotlin-stdlib + + + org.jetbrains.kotlin + kotlin-stdlib-common + + + com.squareup.okhttp3 + okhttp + com.sun.jersey jersey-core From a7c89ff37176fd2c5484aeb5c682e79889acecf2 Mon Sep 17 00:00:00 2001 From: Ashutosh Gupta Date: Thu, 19 May 2022 15:39:25 +0100 Subject: [PATCH 12/12] removing okHttpVersion --- hadoop-tools/hadoop-azure-datalake/pom.xml | 1 - 1 file changed, 1 deletion(-) diff --git a/hadoop-tools/hadoop-azure-datalake/pom.xml b/hadoop-tools/hadoop-azure-datalake/pom.xml index 04cb92916fc1f..cec050d2c1be7 100644 --- a/hadoop-tools/hadoop-azure-datalake/pom.xml +++ b/hadoop-tools/hadoop-azure-datalake/pom.xml @@ -29,7 +29,6 @@ jar - ${okhttp.version} 0.9.1 UTF-8 true