From 822879e374eb7daf7f116ce422257c017c6a85af Mon Sep 17 00:00:00 2001 From: Sadanand Shenoy Date: Tue, 6 Aug 2024 11:51:10 +0530 Subject: [PATCH 1/4] HDDS-11257. Ozone write does not work when http proxy is set for the JVM. --- .../java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java | 1 + .../ozone/container/replication/GrpcReplicationClient.java | 3 ++- .../org/apache/hadoop/hdds/scm/ha/InterSCMGrpcClient.java | 3 ++- hadoop-ozone/integration-test/pom.xml | 7 +++++++ .../ozone/freon/FollowerAppendLogEntryGenerator.java | 3 ++- .../hadoop/ozone/freon/LeaderAppendLogEntryGenerator.java | 3 ++- 6 files changed, 16 insertions(+), 4 deletions(-) diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java index 02747f53ca6d..dd2adbae22b3 100644 --- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java +++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java @@ -191,6 +191,7 @@ protected NettyChannelBuilder createChannel(DatanodeDetails dn, int port) NettyChannelBuilder channelBuilder = NettyChannelBuilder.forAddress(dn.getIpAddress(), port).usePlaintext() .maxInboundMessageSize(OzoneConsts.OZONE_SCM_CHUNK_MAX_SIZE) + .proxyDetector(uri-> null) .intercept(new GrpcClientInterceptor()); if (secConfig.isSecurityEnabled() && secConfig.isGrpcTlsEnabled()) { SslContextBuilder sslContextBuilder = GrpcSslContexts.forClient(); diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/GrpcReplicationClient.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/GrpcReplicationClient.java index 6c9cdc3fef10..9c66f5bd8a1a 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/GrpcReplicationClient.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/GrpcReplicationClient.java @@ -75,7 +75,8 @@ public GrpcReplicationClient( NettyChannelBuilder channelBuilder = NettyChannelBuilder.forAddress(host, port) .usePlaintext() - .maxInboundMessageSize(OzoneConsts.OZONE_SCM_CHUNK_MAX_SIZE); + .maxInboundMessageSize(OzoneConsts.OZONE_SCM_CHUNK_MAX_SIZE) + .proxyDetector(uri -> null); if (secConfig.isSecurityEnabled() && secConfig.isGrpcTlsEnabled()) { channelBuilder.useTransportSecurity(); diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/InterSCMGrpcClient.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/InterSCMGrpcClient.java index bc5a58472e30..975760d7c031 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/InterSCMGrpcClient.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/InterSCMGrpcClient.java @@ -68,7 +68,8 @@ public InterSCMGrpcClient(final String host, TimeUnit.MILLISECONDS); NettyChannelBuilder channelBuilder = NettyChannelBuilder.forAddress(host, port).usePlaintext() - .maxInboundMessageSize(OzoneConsts.OZONE_SCM_CHUNK_MAX_SIZE); + .maxInboundMessageSize(OzoneConsts.OZONE_SCM_CHUNK_MAX_SIZE) + .proxyDetector(uri->null); SecurityConfig securityConfig = new SecurityConfig(conf); if (securityConfig.isSecurityEnabled() && securityConfig.isGrpcTlsEnabled()) { diff --git a/hadoop-ozone/integration-test/pom.xml b/hadoop-ozone/integration-test/pom.xml index 7de7b2f936ec..e54e834f9237 100644 --- a/hadoop-ozone/integration-test/pom.xml +++ b/hadoop-ozone/integration-test/pom.xml @@ -257,6 +257,13 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd"> ${basedir}/dev-support/findbugsExcludeFile.xml + + org.apache.maven.plugins + maven-surefire-plugin + + -Dhttps.proxyHost=127.0.0.1 + + diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/FollowerAppendLogEntryGenerator.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/FollowerAppendLogEntryGenerator.java index d76081fd29ea..76ef7888d0a9 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/FollowerAppendLogEntryGenerator.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/FollowerAppendLogEntryGenerator.java @@ -168,7 +168,8 @@ public Void call() throws Exception { .build(); NettyChannelBuilder channelBuilder = - NettyChannelBuilder.forTarget(serverAddress); + NettyChannelBuilder.forTarget(serverAddress) + .proxyDetector(uri -> null); channelBuilder.negotiationType(NegotiationType.PLAINTEXT); ManagedChannel build = channelBuilder.build(); stub = RaftServerProtocolServiceGrpc.newStub(build); diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/LeaderAppendLogEntryGenerator.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/LeaderAppendLogEntryGenerator.java index 40f670260326..84faeae22344 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/LeaderAppendLogEntryGenerator.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/LeaderAppendLogEntryGenerator.java @@ -144,7 +144,8 @@ public Void call() throws Exception { .build(); NettyChannelBuilder channelBuilder = - NettyChannelBuilder.forTarget(serverAddress); + NettyChannelBuilder.forTarget(serverAddress) + .proxyDetector(uri->null); channelBuilder.negotiationType(NegotiationType.PLAINTEXT); ManagedChannel build = channelBuilder.build(); stub = RaftServerProtocolServiceGrpc.newStub(build); From cbee6236bc3feed7cd75f4b30aa6f20260d6cf4e Mon Sep 17 00:00:00 2001 From: Sadanand Shenoy Date: Tue, 6 Aug 2024 12:31:59 +0530 Subject: [PATCH 2/4] remove config for tests --- hadoop-ozone/integration-test/pom.xml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/hadoop-ozone/integration-test/pom.xml b/hadoop-ozone/integration-test/pom.xml index e54e834f9237..7de7b2f936ec 100644 --- a/hadoop-ozone/integration-test/pom.xml +++ b/hadoop-ozone/integration-test/pom.xml @@ -257,13 +257,6 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd"> ${basedir}/dev-support/findbugsExcludeFile.xml - - org.apache.maven.plugins - maven-surefire-plugin - - -Dhttps.proxyHost=127.0.0.1 - - From ffa687b9353e6c47c8d85872ff1c85f0a1548d8f Mon Sep 17 00:00:00 2001 From: Sadanand Shenoy Date: Tue, 6 Aug 2024 12:40:30 +0530 Subject: [PATCH 3/4] fix checkstyl --- .../java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java | 2 +- .../java/org/apache/hadoop/hdds/scm/ha/InterSCMGrpcClient.java | 2 +- .../hadoop/ozone/freon/LeaderAppendLogEntryGenerator.java | 3 +-- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java index dd2adbae22b3..b586c2d1993d 100644 --- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java +++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java @@ -191,7 +191,7 @@ protected NettyChannelBuilder createChannel(DatanodeDetails dn, int port) NettyChannelBuilder channelBuilder = NettyChannelBuilder.forAddress(dn.getIpAddress(), port).usePlaintext() .maxInboundMessageSize(OzoneConsts.OZONE_SCM_CHUNK_MAX_SIZE) - .proxyDetector(uri-> null) + .proxyDetector(uri -> null) .intercept(new GrpcClientInterceptor()); if (secConfig.isSecurityEnabled() && secConfig.isGrpcTlsEnabled()) { SslContextBuilder sslContextBuilder = GrpcSslContexts.forClient(); diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/InterSCMGrpcClient.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/InterSCMGrpcClient.java index 975760d7c031..03be3d868109 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/InterSCMGrpcClient.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/InterSCMGrpcClient.java @@ -69,7 +69,7 @@ public InterSCMGrpcClient(final String host, NettyChannelBuilder channelBuilder = NettyChannelBuilder.forAddress(host, port).usePlaintext() .maxInboundMessageSize(OzoneConsts.OZONE_SCM_CHUNK_MAX_SIZE) - .proxyDetector(uri->null); + .proxyDetector(uri -> null); SecurityConfig securityConfig = new SecurityConfig(conf); if (securityConfig.isSecurityEnabled() && securityConfig.isGrpcTlsEnabled()) { diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/LeaderAppendLogEntryGenerator.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/LeaderAppendLogEntryGenerator.java index 84faeae22344..9038f379186d 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/LeaderAppendLogEntryGenerator.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/LeaderAppendLogEntryGenerator.java @@ -144,8 +144,7 @@ public Void call() throws Exception { .build(); NettyChannelBuilder channelBuilder = - NettyChannelBuilder.forTarget(serverAddress) - .proxyDetector(uri->null); + NettyChannelBuilder.forTarget(serverAddress).proxyDetector(uri -> null); channelBuilder.negotiationType(NegotiationType.PLAINTEXT); ManagedChannel build = channelBuilder.build(); stub = RaftServerProtocolServiceGrpc.newStub(build); From ad8df33e8bd4500b5aa2a8649e7e3a95e17a7f04 Mon Sep 17 00:00:00 2001 From: Sadanand Shenoy Date: Tue, 6 Aug 2024 13:58:01 +0530 Subject: [PATCH 4/4] address comment --- .../org/apache/hadoop/ozone/om/protocolPB/GrpcOmTransport.java | 1 + 1 file changed, 1 insertion(+) diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/GrpcOmTransport.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/GrpcOmTransport.java index 96f7b48665c2..ac2e85da84df 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/GrpcOmTransport.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/GrpcOmTransport.java @@ -145,6 +145,7 @@ public void start() throws IOException { NettyChannelBuilder channelBuilder = NettyChannelBuilder.forAddress(hp.getHost(), hp.getPort()) .usePlaintext() + .proxyDetector(uri -> null) .maxInboundMessageSize(maxSize); if (secConfig.isSecurityEnabled() && secConfig.isGrpcTlsEnabled()) {