From 401ae5a7c9d363d960307b8314ab2146d67594f8 Mon Sep 17 00:00:00 2001 From: slfan1989 Date: Fri, 17 Jun 2022 20:53:20 -0700 Subject: [PATCH 01/18] HADOOP-18302. Remove WhiteBox in hadoop-commmon module. --- .../java/org/apache/hadoop/portmap/TestPortmap.java | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/hadoop-common-project/hadoop-nfs/src/test/java/org/apache/hadoop/portmap/TestPortmap.java b/hadoop-common-project/hadoop-nfs/src/test/java/org/apache/hadoop/portmap/TestPortmap.java index 8ebf9d03c6c302..9f4c599587925a 100644 --- a/hadoop-common-project/hadoop-nfs/src/test/java/org/apache/hadoop/portmap/TestPortmap.java +++ b/hadoop-common-project/hadoop-nfs/src/test/java/org/apache/hadoop/portmap/TestPortmap.java @@ -25,13 +25,14 @@ import java.net.Socket; import java.util.Map; +import org.apache.commons.lang3.reflect.FieldUtils; +import org.apache.hadoop.crypto.key.kms.KMSClientProvider; import org.junit.Assert; import org.apache.hadoop.oncrpc.RpcCall; import org.apache.hadoop.oncrpc.XDR; import org.apache.hadoop.oncrpc.security.CredentialsNone; import org.apache.hadoop.oncrpc.security.VerifierNone; -import org.apache.hadoop.test.Whitebox; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; @@ -76,7 +77,7 @@ public void testIdle() throws InterruptedException, IOException { } @Test(timeout = 10000) - public void testRegistration() throws IOException, InterruptedException { + public void testRegistration() throws IOException, InterruptedException, IllegalAccessException { XDR req = new XDR(); RpcCall.getInstance(++xid, RpcProgramPortmap.PROGRAM, RpcProgramPortmap.VERSION, @@ -101,8 +102,9 @@ public void testRegistration() throws IOException, InterruptedException { Thread.sleep(100); boolean found = false; @SuppressWarnings("unchecked") - Map map = (Map) Whitebox - .getInternalState(pm.getHandler(), "map"); + Map map = (Map) + FieldUtils.getField(RpcProgramPortmap.class, + "map", true).get(pm.getHandler()); for (PortmapMapping m : map.values()) { if (m.getPort() == sent.getPort() From 9c382a2e1dcb62c4f4d90025b0f7bc7e6d25e470 Mon Sep 17 00:00:00 2001 From: slfan1989 Date: Fri, 17 Jun 2022 23:21:17 -0700 Subject: [PATCH 02/18] HADOOP-18302. Remove WhiteBox in hadoop-commmon module. --- .../test/java/org/apache/hadoop/fs/TestLocalFileSystem.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java index 949ea423315270..80c1a7d5e5e324 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.fs; +import org.apache.commons.lang3.reflect.FieldUtils; import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem.Statistics; @@ -24,7 +25,6 @@ import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.test.LambdaTestUtils; -import org.apache.hadoop.test.Whitebox; import org.apache.hadoop.util.StringUtils; import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_DEFAULT; @@ -650,7 +650,9 @@ public void testFileStatusPipeFile() throws Exception { RawLocalFileSystem fs = spy(origFs); Configuration conf = mock(Configuration.class); fs.setConf(conf); - Whitebox.setInternalState(fs, "useDeprecatedFileStatus", false); + + FieldUtils.getField(RawLocalFileSystem.class, + "useDeprecatedFileStatus", true).set(fs,false); Path path = new Path("/foo"); File pipe = mock(File.class); when(pipe.isFile()).thenReturn(false); From d101a551f7798874cf129e9db2c5dc615015b64a Mon Sep 17 00:00:00 2001 From: slfan1989 Date: Sat, 18 Jun 2022 00:05:30 -0700 Subject: [PATCH 03/18] HADOOP-18302. Remove WhiteBox in hadoop-common module. --- .../apache/hadoop/http/TestHttpServer.java | 9 ++++--- .../java/org/apache/hadoop/ipc/TestIPC.java | 8 +++--- .../java/org/apache/hadoop/ipc/TestRPC.java | 26 +++++++++---------- 3 files changed, 22 insertions(+), 21 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServer.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServer.java index b1255d19d90867..823f27dcddcf13 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServer.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServer.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.http; +import org.apache.commons.lang3.reflect.FieldUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration.IntegerRanges; import org.apache.hadoop.fs.CommonConfigurationKeys; @@ -663,8 +664,8 @@ private HttpServer2 checkBindAddress(String host, int port, boolean findPort) HttpServer2 server = createServer(host, port); try { // not bound, ephemeral should return requested port (0 for ephemeral) - List listeners = (List) Whitebox.getInternalState(server, - "listeners"); + List listeners = (List) FieldUtils. + getField(HttpServer2.class,"listeners", true).get(server); ServerConnector listener = (ServerConnector)listeners.get(0); assertEquals(port, listener.getPort()); @@ -740,8 +741,8 @@ public void testBacklogSize() throws Exception Configuration conf = new Configuration(); conf.setInt(HttpServer2.HTTP_SOCKET_BACKLOG_SIZE_KEY, backlogSize); HttpServer2 srv = createServer("test", conf); - List listeners = (List) Whitebox.getInternalState(srv, - "listeners"); + List listeners = (List) FieldUtils. + getField(HttpServer2.class,"listeners", true).get(srv); ServerConnector listener = (ServerConnector)listeners.get(0); assertEquals(backlogSize, listener.getAcceptQueueSize()); } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestIPC.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestIPC.java index 95ff302103d89d..6851a6aff327b6 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestIPC.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestIPC.java @@ -66,6 +66,7 @@ import javax.net.SocketFactory; +import org.apache.commons.lang3.reflect.FieldUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; @@ -91,7 +92,6 @@ import org.apache.hadoop.security.token.SecretManager.InvalidToken; import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.test.LambdaTestUtils; -import org.apache.hadoop.test.Whitebox; import org.apache.hadoop.util.StringUtils; import org.junit.Assert; import org.junit.Assume; @@ -881,9 +881,9 @@ private void checkBlocking(int readers, int readerQ, int callQ) throws Exception // start server final TestServerQueue server = new TestServerQueue(clients, readers, callQ, handlers, conf); - CallQueueManager spy = spy( - (CallQueueManager)Whitebox.getInternalState(server, "callQueue")); - Whitebox.setInternalState(server, "callQueue", spy); + CallQueueManager spy = spy((CallQueueManager) + FieldUtils.getField(Server.class, "callQueue", true).get(server)); + FieldUtils.getField(Server.class, "callQueue", true).set(server, spy); final InetSocketAddress addr = NetUtils.getConnectAddress(server); server.start(); diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPC.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPC.java index 5fc9cb5410b30b..ff5632c482e779 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPC.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPC.java @@ -18,6 +18,7 @@ package org.apache.hadoop.ipc; +import org.apache.commons.lang3.reflect.FieldUtils; import org.apache.hadoop.ipc.metrics.RpcMetrics; import org.apache.hadoop.thirdparty.protobuf.ServiceException; import org.apache.hadoop.HadoopIllegalArgumentException; @@ -48,7 +49,6 @@ import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.test.MetricsAsserts; import org.apache.hadoop.test.MockitoUtil; -import org.apache.hadoop.test.Whitebox; import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -296,7 +296,7 @@ public ProtocolProxy getProxy( throws IOException { T proxy = (T) Proxy.newProxyInstance(protocol.getClassLoader(), new Class[] { protocol }, new StoppedInvocationHandler()); - return new ProtocolProxy(protocol, proxy, false); + return new ProtocolProxy<>(protocol, proxy, false); } @Override @@ -1160,9 +1160,9 @@ public void testClientBackOff() throws Exception { server = setupTestServer(builder); @SuppressWarnings("unchecked") - CallQueueManager spy = spy((CallQueueManager) Whitebox - .getInternalState(server, "callQueue")); - Whitebox.setInternalState(server, "callQueue", spy); + CallQueueManager spy = spy((CallQueueManager) + FieldUtils.getField(Server.class, "callQueue", true).get(server)); + FieldUtils.getField(Server.class, "callQueue", true).set(server, spy); Exception lastException = null; proxy = getClient(addr, conf); @@ -1214,7 +1214,7 @@ public void testClientBackOffByResponseTime() throws Exception { GenericTestUtils.setLogLevel(DecayRpcScheduler.LOG, Level.DEBUG); GenericTestUtils.setLogLevel(RPC.LOG, Level.DEBUG); - final List> res = new ArrayList>(); + final List> res = new ArrayList<>(); final ExecutorService executorService = Executors.newFixedThreadPool(numClients); conf.setInt(CommonConfigurationKeys.IPC_CLIENT_CONNECT_MAX_RETRIES_KEY, 0); @@ -1223,9 +1223,9 @@ public void testClientBackOffByResponseTime() throws Exception { Server server = setupDecayRpcSchedulerandTestServer(ns + "."); @SuppressWarnings("unchecked") - CallQueueManager spy = spy((CallQueueManager) Whitebox - .getInternalState(server, "callQueue")); - Whitebox.setInternalState(server, "callQueue", spy); + CallQueueManager spy = spy((CallQueueManager) + FieldUtils.getField(Server.class, "callQueue", true).get(server)); + FieldUtils.getField(Server.class, "callQueue", true).set(server, spy); Exception lastException = null; proxy = getClient(addr, conf); @@ -1564,11 +1564,11 @@ public RpcStatusProto getRpcStatusProto() { RPC.Builder builder = newServerBuilder(conf) .setQueueSizePerHandler(1).setNumHandlers(1).setVerbose(true); server = setupTestServer(builder); - Whitebox.setInternalState( - server, "rpcRequestClass", FakeRequestClass.class); + FieldUtils.getField(Server.class, "rpcRequestClass", true). + set(server, FakeRequestClass.class); MutableCounterLong authMetric = - (MutableCounterLong)Whitebox.getInternalState( - server.getRpcMetrics(), "rpcAuthorizationSuccesses"); + (MutableCounterLong) FieldUtils.getField(RpcMetrics.class, + "rpcAuthorizationSuccesses", true).get(server.getRpcMetrics()); proxy = getClient(addr, conf); boolean isDisconnected = true; From 5918769a7e49d19ee7dfe587d1fc06afb2d19c40 Mon Sep 17 00:00:00 2001 From: slfan1989 Date: Sat, 18 Jun 2022 05:47:34 -0700 Subject: [PATCH 04/18] HADOOP-18302. Remove WhiteBox in hadoop-common module. --- .../apache/hadoop/http/TestHttpServer.java | 1 - .../metrics2/impl/TestGraphiteMetrics.java | 29 +++++++++++-------- .../metrics2/impl/TestStatsDMetrics.java | 12 ++++---- 3 files changed, 24 insertions(+), 18 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServer.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServer.java index 823f27dcddcf13..e42b1e30ceb77b 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServer.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServer.java @@ -30,7 +30,6 @@ import org.apache.hadoop.security.ShellBasedUnixGroupsMapping; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.authorize.AccessControlList; -import org.apache.hadoop.test.Whitebox; import org.assertj.core.api.Assertions; import org.eclipse.jetty.server.ServerConnector; diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestGraphiteMetrics.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestGraphiteMetrics.java index 743080acd7a5e8..57a9821411ce55 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestGraphiteMetrics.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestGraphiteMetrics.java @@ -18,11 +18,11 @@ package org.apache.hadoop.metrics2.impl; +import org.apache.commons.lang3.reflect.FieldUtils; import org.apache.hadoop.metrics2.AbstractMetric; import org.apache.hadoop.metrics2.MetricsRecord; import org.apache.hadoop.metrics2.MetricsTag; import org.apache.hadoop.metrics2.sink.GraphiteSink; -import org.apache.hadoop.test.Whitebox; import org.junit.Test; import org.mockito.ArgumentCaptor; @@ -58,7 +58,7 @@ private GraphiteSink.Graphite makeGraphite() { } @Test - public void testPutMetrics() { + public void testPutMetrics() throws IllegalAccessException { GraphiteSink sink = new GraphiteSink(); List tags = new ArrayList(); tags.add(new MetricsTag(MsInfo.Context, "all")); @@ -70,7 +70,8 @@ public void testPutMetrics() { ArgumentCaptor argument = ArgumentCaptor.forClass(String.class); final GraphiteSink.Graphite mockGraphite = makeGraphite(); - Whitebox.setInternalState(sink, "graphite", mockGraphite); + FieldUtils.getField(GraphiteSink.class, "graphite", true). + set(sink, mockGraphite); sink.putMetrics(record); try { @@ -89,11 +90,11 @@ public void testPutMetrics() { } @Test - public void testPutMetrics2() { + public void testPutMetrics2() throws IllegalAccessException { GraphiteSink sink = new GraphiteSink(); List tags = new ArrayList(); tags.add(new MetricsTag(MsInfo.Context, "all")); - tags.add(new MetricsTag(MsInfo.Hostname, null)); + tags.add(new MetricsTag(MsInfo.Hostname, null)); Set metrics = new HashSet(); metrics.add(makeMetric("foo1", 1)); metrics.add(makeMetric("foo2", 2)); @@ -102,7 +103,8 @@ public void testPutMetrics2() { ArgumentCaptor argument = ArgumentCaptor.forClass(String.class); final GraphiteSink.Graphite mockGraphite = makeGraphite(); - Whitebox.setInternalState(sink, "graphite", mockGraphite); + FieldUtils.getField(GraphiteSink.class,"graphite",true). + set(sink,mockGraphite); sink.putMetrics(record); try { @@ -124,12 +126,13 @@ public void testPutMetrics2() { * Assert that timestamps are converted correctly, ticket HADOOP-11182 */ @Test - public void testPutMetrics3() { + public void testPutMetrics3() throws IllegalAccessException { // setup GraphiteSink GraphiteSink sink = new GraphiteSink(); final GraphiteSink.Graphite mockGraphite = makeGraphite(); - Whitebox.setInternalState(sink, "graphite", mockGraphite); + FieldUtils.getField(GraphiteSink.class,"graphite",true). + set(sink,mockGraphite); // given two metrics records with timestamps 1000 milliseconds apart. List tags = Collections.emptyList(); @@ -158,7 +161,7 @@ public void testPutMetrics3() { } @Test - public void testFailureAndPutMetrics() throws IOException { + public void testFailureAndPutMetrics() throws IOException, IllegalAccessException { GraphiteSink sink = new GraphiteSink(); List tags = new ArrayList(); tags.add(new MetricsTag(MsInfo.Context, "all")); @@ -169,7 +172,8 @@ public void testFailureAndPutMetrics() throws IOException { MetricsRecord record = new MetricsRecordImpl(MsInfo.Context, (long) 10000, tags, metrics); final GraphiteSink.Graphite mockGraphite = makeGraphite(); - Whitebox.setInternalState(sink, "graphite", mockGraphite); + FieldUtils.getField(GraphiteSink.class, "graphite", true). + set(sink, mockGraphite); // throw exception when first try doThrow(new IOException("IO exception")).when(mockGraphite).write(anyString()); @@ -196,10 +200,11 @@ public void testFailureAndPutMetrics() throws IOException { } @Test - public void testClose(){ + public void testClose() throws IllegalAccessException { GraphiteSink sink = new GraphiteSink(); final GraphiteSink.Graphite mockGraphite = makeGraphite(); - Whitebox.setInternalState(sink, "graphite", mockGraphite); + FieldUtils.getField(GraphiteSink.class,"graphite",true). + set(sink,mockGraphite); try { sink.close(); } catch (IOException ioe) { diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestStatsDMetrics.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestStatsDMetrics.java index 4cf4894ff8352f..5e2818969b7d2c 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestStatsDMetrics.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestStatsDMetrics.java @@ -31,13 +31,13 @@ import java.util.List; import java.util.Set; +import org.apache.commons.lang3.reflect.FieldUtils; import org.apache.hadoop.metrics2.AbstractMetric; import org.apache.hadoop.metrics2.MetricType; import org.apache.hadoop.metrics2.MetricsRecord; import org.apache.hadoop.metrics2.MetricsTag; import org.apache.hadoop.metrics2.sink.StatsDSink; import org.apache.hadoop.metrics2.sink.StatsDSink.StatsD; -import org.apache.hadoop.test.Whitebox; import org.junit.Test; public class TestStatsDMetrics { @@ -52,7 +52,7 @@ private AbstractMetric makeMetric(String name, Number value, } @Test(timeout=3000) - public void testPutMetrics() throws IOException, InterruptedException { + public void testPutMetrics() throws IOException, IllegalAccessException { final StatsDSink sink = new StatsDSink(); List tags = new ArrayList(); tags.add(new MetricsTag(MsInfo.Hostname, "host")); @@ -69,7 +69,8 @@ public void testPutMetrics() throws IOException, InterruptedException { final StatsDSink.StatsD mockStatsD = new StatsD(sock.getLocalAddress().getHostName(), sock.getLocalPort()); - Whitebox.setInternalState(sink, "statsd", mockStatsD); + FieldUtils.getField(StatsDSink.class, "statsd", true). + set(sink, mockStatsD); final DatagramPacket p = new DatagramPacket(new byte[8192], 8192); sink.putMetrics(record); sock.receive(p); @@ -87,7 +88,7 @@ public void testPutMetrics() throws IOException, InterruptedException { } @Test(timeout=3000) - public void testPutMetrics2() throws IOException { + public void testPutMetrics2() throws IOException, IllegalAccessException { StatsDSink sink = new StatsDSink(); List tags = new ArrayList(); tags.add(new MetricsTag(MsInfo.Hostname, null)); @@ -104,7 +105,8 @@ public void testPutMetrics2() throws IOException { final StatsDSink.StatsD mockStatsD = new StatsD(sock.getLocalAddress().getHostName(), sock.getLocalPort()); - Whitebox.setInternalState(sink, "statsd", mockStatsD); + FieldUtils.getField(StatsDSink.class, "statsd", true). + set(sink, mockStatsD); final DatagramPacket p = new DatagramPacket(new byte[8192], 8192); sink.putMetrics(record); sock.receive(p); From e30456b47f2dcfade0c54b9bcf9a05ee01049b67 Mon Sep 17 00:00:00 2001 From: slfan1989 Date: Mon, 20 Jun 2022 16:16:45 -0700 Subject: [PATCH 05/18] HADOOP-18302. Fix CheckStyle. --- .../apache/hadoop/fs/TestLocalFileSystem.java | 2 +- .../apache/hadoop/http/TestHttpServer.java | 4 +- .../metrics2/impl/TestGraphiteMetrics.java | 41 ++++++++----------- 3 files changed, 21 insertions(+), 26 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java index 80c1a7d5e5e324..cd864c4c7240d5 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java @@ -652,7 +652,7 @@ public void testFileStatusPipeFile() throws Exception { fs.setConf(conf); FieldUtils.getField(RawLocalFileSystem.class, - "useDeprecatedFileStatus", true).set(fs,false); + "useDeprecatedFileStatus", true).set(fs, false); Path path = new Path("/foo"); File pipe = mock(File.class); when(pipe.isFile()).thenReturn(false); diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServer.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServer.java index e42b1e30ceb77b..0491290ffafa89 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServer.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServer.java @@ -664,7 +664,7 @@ private HttpServer2 checkBindAddress(String host, int port, boolean findPort) try { // not bound, ephemeral should return requested port (0 for ephemeral) List listeners = (List) FieldUtils. - getField(HttpServer2.class,"listeners", true).get(server); + getField(HttpServer2.class, "listeners", true).get(server); ServerConnector listener = (ServerConnector)listeners.get(0); assertEquals(port, listener.getPort()); @@ -741,7 +741,7 @@ public void testBacklogSize() throws Exception conf.setInt(HttpServer2.HTTP_SOCKET_BACKLOG_SIZE_KEY, backlogSize); HttpServer2 srv = createServer("test", conf); List listeners = (List) FieldUtils. - getField(HttpServer2.class,"listeners", true).get(srv); + getField(HttpServer2.class, "listeners", true).get(srv); ServerConnector listener = (ServerConnector)listeners.get(0); assertEquals(backlogSize, listener.getAcceptQueueSize()); } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestGraphiteMetrics.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestGraphiteMetrics.java index 57a9821411ce55..3fadf48714bd96 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestGraphiteMetrics.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestGraphiteMetrics.java @@ -70,8 +70,7 @@ public void testPutMetrics() throws IllegalAccessException { ArgumentCaptor argument = ArgumentCaptor.forClass(String.class); final GraphiteSink.Graphite mockGraphite = makeGraphite(); - FieldUtils.getField(GraphiteSink.class, "graphite", true). - set(sink, mockGraphite); + FieldUtils.getField(GraphiteSink.class, "graphite", true).set(sink, mockGraphite); sink.putMetrics(record); try { @@ -85,7 +84,7 @@ public void testPutMetrics() throws IllegalAccessException { assertEquals(true, result.equals("null.all.Context.Context=all.Hostname=host.foo1 1.25 10\n" + "null.all.Context.Context=all.Hostname=host.foo2 2.25 10\n") || - result.equals("null.all.Context.Context=all.Hostname=host.foo2 2.25 10\n" + + result.equals("null.all.Context.Context=all.Hostname=host.foo2 2.25 10\n" + "null.all.Context.Context=all.Hostname=host.foo1 1.25 10\n")); } @@ -103,8 +102,7 @@ public void testPutMetrics2() throws IllegalAccessException { ArgumentCaptor argument = ArgumentCaptor.forClass(String.class); final GraphiteSink.Graphite mockGraphite = makeGraphite(); - FieldUtils.getField(GraphiteSink.class,"graphite",true). - set(sink,mockGraphite); + FieldUtils.getField(GraphiteSink.class, "graphite", true).set(sink, mockGraphite); sink.putMetrics(record); try { @@ -131,8 +129,7 @@ public void testPutMetrics3() throws IllegalAccessException { // setup GraphiteSink GraphiteSink sink = new GraphiteSink(); final GraphiteSink.Graphite mockGraphite = makeGraphite(); - FieldUtils.getField(GraphiteSink.class,"graphite",true). - set(sink,mockGraphite); + FieldUtils.getField(GraphiteSink.class,"graphite",true).set(sink,mockGraphite); // given two metrics records with timestamps 1000 milliseconds apart. List tags = Collections.emptyList(); @@ -172,8 +169,7 @@ public void testFailureAndPutMetrics() throws IOException, IllegalAccessExceptio MetricsRecord record = new MetricsRecordImpl(MsInfo.Context, (long) 10000, tags, metrics); final GraphiteSink.Graphite mockGraphite = makeGraphite(); - FieldUtils.getField(GraphiteSink.class, "graphite", true). - set(sink, mockGraphite); + FieldUtils.getField(GraphiteSink.class, "graphite", true).set(sink, mockGraphite); // throw exception when first try doThrow(new IOException("IO exception")).when(mockGraphite).write(anyString()); @@ -201,20 +197,19 @@ public void testFailureAndPutMetrics() throws IOException, IllegalAccessExceptio @Test public void testClose() throws IllegalAccessException { - GraphiteSink sink = new GraphiteSink(); - final GraphiteSink.Graphite mockGraphite = makeGraphite(); - FieldUtils.getField(GraphiteSink.class,"graphite",true). - set(sink,mockGraphite); - try { - sink.close(); - } catch (IOException ioe) { - ioe.printStackTrace(); - } + GraphiteSink sink = new GraphiteSink(); + final GraphiteSink.Graphite mockGraphite = makeGraphite(); + FieldUtils.getField(GraphiteSink.class,"graphite",true).set(sink,mockGraphite); + try { + sink.close(); + } catch (IOException ioe) { + ioe.printStackTrace(); + } - try { - verify(mockGraphite).close(); - } catch (IOException ioe) { - ioe.printStackTrace(); - } + try { + verify(mockGraphite).close(); + } catch (IOException ioe) { + ioe.printStackTrace(); + } } } From 891427d90c3b86a8ce9b9d87ece146c972cef6a7 Mon Sep 17 00:00:00 2001 From: slfan1989 Date: Sun, 10 Jul 2022 05:53:49 -0700 Subject: [PATCH 06/18] HADOOP-18302. Remove WhiteBox in hadoop-common module. --- .../main/java/org/apache/hadoop/fs/RawLocalFileSystem.java | 4 ++++ .../src/main/java/org/apache/hadoop/http/HttpServer2.java | 3 +++ .../src/main/java/org/apache/hadoop/ipc/Server.java | 7 +++++++ .../java/org/apache/hadoop/fs/TestLocalFileSystem.java | 3 +-- .../test/java/org/apache/hadoop/http/TestHttpServer.java | 6 ++---- .../src/test/java/org/apache/hadoop/ipc/TestIPC.java | 5 ++--- 6 files changed, 19 insertions(+), 9 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java index f525c3cba78fe0..03b1850f2485e7 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java @@ -1293,4 +1293,8 @@ public boolean hasPathCapability(final Path path, final String capability) return super.hasPathCapability(path, capability); } } + + public static void setUseDeprecatedFileStatus(boolean useDeprecatedFileStatus) { + RawLocalFileSystem.useDeprecatedFileStatus = useDeprecatedFileStatus; + } } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java index 2928f885982079..af9bcd91ed12a6 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java @@ -1967,4 +1967,7 @@ HttpServer2Metrics getMetrics() { return metrics; } + public List getListeners() { + return listeners; + } } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java index e79612f7a5a0f8..6725c410c9ee2c 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java @@ -4131,4 +4131,11 @@ public synchronized void run() { } } + public CallQueueManager getCallQueue() { + return callQueue; + } + + public void setCallQueue(CallQueueManager callQueue) { + this.callQueue = callQueue; + } } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java index e1a39ea4a5f512..fbfe1170282133 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java @@ -651,8 +651,7 @@ public void testFileStatusPipeFile() throws Exception { Configuration conf = mock(Configuration.class); fs.setConf(conf); - FieldUtils.getField(RawLocalFileSystem.class, - "useDeprecatedFileStatus", true).set(fs, false); + RawLocalFileSystem.setUseDeprecatedFileStatus(false); Path path = new Path("/foo"); File pipe = mock(File.class); when(pipe.isFile()).thenReturn(false); diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServer.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServer.java index 0491290ffafa89..995dcdda21d9d7 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServer.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServer.java @@ -663,8 +663,7 @@ private HttpServer2 checkBindAddress(String host, int port, boolean findPort) HttpServer2 server = createServer(host, port); try { // not bound, ephemeral should return requested port (0 for ephemeral) - List listeners = (List) FieldUtils. - getField(HttpServer2.class, "listeners", true).get(server); + List listeners = server.getListeners(); ServerConnector listener = (ServerConnector)listeners.get(0); assertEquals(port, listener.getPort()); @@ -740,8 +739,7 @@ public void testBacklogSize() throws Exception Configuration conf = new Configuration(); conf.setInt(HttpServer2.HTTP_SOCKET_BACKLOG_SIZE_KEY, backlogSize); HttpServer2 srv = createServer("test", conf); - List listeners = (List) FieldUtils. - getField(HttpServer2.class, "listeners", true).get(srv); + List listeners = srv.getListeners(); ServerConnector listener = (ServerConnector)listeners.get(0); assertEquals(backlogSize, listener.getAcceptQueueSize()); } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestIPC.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestIPC.java index 6851a6aff327b6..72776924cc4391 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestIPC.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestIPC.java @@ -881,9 +881,8 @@ private void checkBlocking(int readers, int readerQ, int callQ) throws Exception // start server final TestServerQueue server = new TestServerQueue(clients, readers, callQ, handlers, conf); - CallQueueManager spy = spy((CallQueueManager) - FieldUtils.getField(Server.class, "callQueue", true).get(server)); - FieldUtils.getField(Server.class, "callQueue", true).set(server, spy); + CallQueueManager spy = spy(server.getCallQueue()); + server.setCallQueue(spy); final InetSocketAddress addr = NetUtils.getConnectAddress(server); server.start(); From 536c1436f2d2aee5cedbdc1a144c7bdcc9ef3fbb Mon Sep 17 00:00:00 2001 From: slfan1989 Date: Sun, 10 Jul 2022 06:01:30 -0700 Subject: [PATCH 07/18] HADOOP-18302. Remove WhiteBox in hadoop-common module. --- .../java/org/apache/hadoop/ipc/Server.java | 4 ++++ .../apache/hadoop/ipc/metrics/RpcMetrics.java | 4 ++++ .../java/org/apache/hadoop/ipc/TestRPC.java | 18 ++++++------------ 3 files changed, 14 insertions(+), 12 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java index 6725c410c9ee2c..0762b3a73b643e 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java @@ -4138,4 +4138,8 @@ public CallQueueManager getCallQueue() { public void setCallQueue(CallQueueManager callQueue) { this.callQueue = callQueue; } + + public void setRpcRequestClass(Class rpcRequestClass) { + this.rpcRequestClass = rpcRequestClass; + } } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/metrics/RpcMetrics.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/metrics/RpcMetrics.java index bf21e3865fa8a2..9459dcf5a95255 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/metrics/RpcMetrics.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/metrics/RpcMetrics.java @@ -364,4 +364,8 @@ public double getDeferredRpcProcessingStdDev() { public MetricsTag getTag(String tagName) { return registry.getTag(tagName); } + + public MutableCounterLong getRpcAuthorizationSuccesses() { + return rpcAuthorizationSuccesses; + } } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPC.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPC.java index b3278ee76a249b..aeeb24ce17354e 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPC.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPC.java @@ -18,7 +18,6 @@ package org.apache.hadoop.ipc; -import org.apache.commons.lang3.reflect.FieldUtils; import org.apache.hadoop.ipc.metrics.RpcMetrics; import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder; @@ -1163,9 +1162,8 @@ public void testClientBackOff() throws Exception { server = setupTestServer(builder); @SuppressWarnings("unchecked") - CallQueueManager spy = spy((CallQueueManager) - FieldUtils.getField(Server.class, "callQueue", true).get(server)); - FieldUtils.getField(Server.class, "callQueue", true).set(server, spy); + CallQueueManager spy = spy(server.getCallQueue()); + server.setCallQueue(spy); Exception lastException = null; proxy = getClient(addr, conf); @@ -1226,9 +1224,8 @@ public void testClientBackOffByResponseTime() throws Exception { Server server = setupDecayRpcSchedulerandTestServer(ns + "."); @SuppressWarnings("unchecked") - CallQueueManager spy = spy((CallQueueManager) - FieldUtils.getField(Server.class, "callQueue", true).get(server)); - FieldUtils.getField(Server.class, "callQueue", true).set(server, spy); + CallQueueManager spy = spy(server.getCallQueue()); + server.setCallQueue(spy); Exception lastException = null; proxy = getClient(addr, conf); @@ -1567,11 +1564,8 @@ public RpcStatusProto getRpcStatusProto() { RPC.Builder builder = newServerBuilder(conf) .setQueueSizePerHandler(1).setNumHandlers(1).setVerbose(true); server = setupTestServer(builder); - FieldUtils.getField(Server.class, "rpcRequestClass", true). - set(server, FakeRequestClass.class); - MutableCounterLong authMetric = - (MutableCounterLong) FieldUtils.getField(RpcMetrics.class, - "rpcAuthorizationSuccesses", true).get(server.getRpcMetrics()); + server.setRpcRequestClass(FakeRequestClass.class); + MutableCounterLong authMetric = server.getRpcMetrics().getRpcAuthorizationSuccesses(); proxy = getClient(addr, conf); boolean isDisconnected = true; From 2625e1df53cc068d8df6f39d76b1586a91706723 Mon Sep 17 00:00:00 2001 From: slfan1989 Date: Sun, 10 Jul 2022 06:07:09 -0700 Subject: [PATCH 08/18] HADOOP-18302. Remove WhiteBox in hadoop-common module. --- .../apache/hadoop/metrics2/sink/GraphiteSink.java | 7 +++++++ .../hadoop/metrics2/impl/TestGraphiteMetrics.java | 13 ++++++------- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/GraphiteSink.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/GraphiteSink.java index ea1bde3a75e031..32284fbf6b662f 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/GraphiteSink.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/GraphiteSink.java @@ -204,4 +204,11 @@ private boolean tooManyConnectionFailures() { } + public Graphite getGraphite() { + return graphite; + } + + public void setGraphite(Graphite graphite) { + this.graphite = graphite; + } } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestGraphiteMetrics.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestGraphiteMetrics.java index 3fadf48714bd96..a59cd17e42dedd 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestGraphiteMetrics.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestGraphiteMetrics.java @@ -18,7 +18,6 @@ package org.apache.hadoop.metrics2.impl; -import org.apache.commons.lang3.reflect.FieldUtils; import org.apache.hadoop.metrics2.AbstractMetric; import org.apache.hadoop.metrics2.MetricsRecord; import org.apache.hadoop.metrics2.MetricsTag; @@ -58,7 +57,7 @@ private GraphiteSink.Graphite makeGraphite() { } @Test - public void testPutMetrics() throws IllegalAccessException { + public void testPutMetrics() { GraphiteSink sink = new GraphiteSink(); List tags = new ArrayList(); tags.add(new MetricsTag(MsInfo.Context, "all")); @@ -70,7 +69,7 @@ public void testPutMetrics() throws IllegalAccessException { ArgumentCaptor argument = ArgumentCaptor.forClass(String.class); final GraphiteSink.Graphite mockGraphite = makeGraphite(); - FieldUtils.getField(GraphiteSink.class, "graphite", true).set(sink, mockGraphite); + sink.setGraphite(mockGraphite); sink.putMetrics(record); try { @@ -102,7 +101,7 @@ public void testPutMetrics2() throws IllegalAccessException { ArgumentCaptor argument = ArgumentCaptor.forClass(String.class); final GraphiteSink.Graphite mockGraphite = makeGraphite(); - FieldUtils.getField(GraphiteSink.class, "graphite", true).set(sink, mockGraphite); + sink.setGraphite(mockGraphite); sink.putMetrics(record); try { @@ -129,7 +128,7 @@ public void testPutMetrics3() throws IllegalAccessException { // setup GraphiteSink GraphiteSink sink = new GraphiteSink(); final GraphiteSink.Graphite mockGraphite = makeGraphite(); - FieldUtils.getField(GraphiteSink.class,"graphite",true).set(sink,mockGraphite); + sink.setGraphite(mockGraphite); // given two metrics records with timestamps 1000 milliseconds apart. List tags = Collections.emptyList(); @@ -169,7 +168,7 @@ public void testFailureAndPutMetrics() throws IOException, IllegalAccessExceptio MetricsRecord record = new MetricsRecordImpl(MsInfo.Context, (long) 10000, tags, metrics); final GraphiteSink.Graphite mockGraphite = makeGraphite(); - FieldUtils.getField(GraphiteSink.class, "graphite", true).set(sink, mockGraphite); + sink.setGraphite(mockGraphite); // throw exception when first try doThrow(new IOException("IO exception")).when(mockGraphite).write(anyString()); @@ -199,7 +198,7 @@ public void testFailureAndPutMetrics() throws IOException, IllegalAccessExceptio public void testClose() throws IllegalAccessException { GraphiteSink sink = new GraphiteSink(); final GraphiteSink.Graphite mockGraphite = makeGraphite(); - FieldUtils.getField(GraphiteSink.class,"graphite",true).set(sink,mockGraphite); + sink.setGraphite(mockGraphite); try { sink.close(); } catch (IOException ioe) { From 2594eb0a2f3a357806152e8d5ef06e7b15acee6e Mon Sep 17 00:00:00 2001 From: slfan1989 Date: Sun, 10 Jul 2022 16:20:57 -0700 Subject: [PATCH 09/18] HADOOP-18302. Fix CheckStyle. --- .../apache/hadoop/fs/TestLocalFileSystem.java | 1 - .../apache/hadoop/http/TestHttpServer.java | 1 - .../java/org/apache/hadoop/ipc/TestIPC.java | 1 - .../metrics2/impl/TestGraphiteMetrics.java | 46 +++++++++---------- .../hadoop/portmap/RpcProgramPortmap.java | 6 ++- .../apache/hadoop/portmap/TestPortmap.java | 7 +-- 6 files changed, 29 insertions(+), 33 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java index fbfe1170282133..38e16221a45183 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java @@ -17,7 +17,6 @@ */ package org.apache.hadoop.fs; -import org.apache.commons.lang3.reflect.FieldUtils; import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem.Statistics; diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServer.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServer.java index 995dcdda21d9d7..dfe6723d6c51ab 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServer.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServer.java @@ -17,7 +17,6 @@ */ package org.apache.hadoop.http; -import org.apache.commons.lang3.reflect.FieldUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration.IntegerRanges; import org.apache.hadoop.fs.CommonConfigurationKeys; diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestIPC.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestIPC.java index 72776924cc4391..77225a19302907 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestIPC.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestIPC.java @@ -66,7 +66,6 @@ import javax.net.SocketFactory; -import org.apache.commons.lang3.reflect.FieldUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestGraphiteMetrics.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestGraphiteMetrics.java index a59cd17e42dedd..ac1b29237a4d78 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestGraphiteMetrics.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestGraphiteMetrics.java @@ -58,33 +58,33 @@ private GraphiteSink.Graphite makeGraphite() { @Test public void testPutMetrics() { - GraphiteSink sink = new GraphiteSink(); - List tags = new ArrayList(); - tags.add(new MetricsTag(MsInfo.Context, "all")); - tags.add(new MetricsTag(MsInfo.Hostname, "host")); - Set metrics = new HashSet(); - metrics.add(makeMetric("foo1", 1.25)); - metrics.add(makeMetric("foo2", 2.25)); - MetricsRecord record = new MetricsRecordImpl(MsInfo.Context, (long) 10000, tags, metrics); + GraphiteSink sink = new GraphiteSink(); + List tags = new ArrayList(); + tags.add(new MetricsTag(MsInfo.Context, "all")); + tags.add(new MetricsTag(MsInfo.Hostname, "host")); + Set metrics = new HashSet(); + metrics.add(makeMetric("foo1", 1.25)); + metrics.add(makeMetric("foo2", 2.25)); + MetricsRecord record = new MetricsRecordImpl(MsInfo.Context, (long) 10000, tags, metrics); - ArgumentCaptor argument = ArgumentCaptor.forClass(String.class); - final GraphiteSink.Graphite mockGraphite = makeGraphite(); - sink.setGraphite(mockGraphite); - sink.putMetrics(record); + ArgumentCaptor argument = ArgumentCaptor.forClass(String.class); + final GraphiteSink.Graphite mockGraphite = makeGraphite(); + sink.setGraphite(mockGraphite); + sink.putMetrics(record); - try { - verify(mockGraphite).write(argument.capture()); - } catch (IOException e) { - e.printStackTrace(); - } + try { + verify(mockGraphite).write(argument.capture()); + } catch (IOException e) { + e.printStackTrace(); + } - String result = argument.getValue(); + String result = argument.getValue(); - assertEquals(true, - result.equals("null.all.Context.Context=all.Hostname=host.foo1 1.25 10\n" + - "null.all.Context.Context=all.Hostname=host.foo2 2.25 10\n") || - result.equals("null.all.Context.Context=all.Hostname=host.foo2 2.25 10\n" + - "null.all.Context.Context=all.Hostname=host.foo1 1.25 10\n")); + assertEquals(true, + result.equals("null.all.Context.Context=all.Hostname=host.foo1 1.25 10\n" + + "null.all.Context.Context=all.Hostname=host.foo2 2.25 10\n") || + result.equals("null.all.Context.Context=all.Hostname=host.foo2 2.25 10\n" + + "null.all.Context.Context=all.Hostname=host.foo1 1.25 10\n")); } @Test diff --git a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/portmap/RpcProgramPortmap.java b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/portmap/RpcProgramPortmap.java index 7b33a644fbe76a..10afaacff04ad8 100644 --- a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/portmap/RpcProgramPortmap.java +++ b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/portmap/RpcProgramPortmap.java @@ -54,7 +54,7 @@ final class RpcProgramPortmap extends IdleStateHandler { private static final Logger LOG = LoggerFactory.getLogger(RpcProgramPortmap.class); - private final ConcurrentHashMap map = new ConcurrentHashMap(); + private final ConcurrentHashMap map = new ConcurrentHashMap<>(); /** ChannelGroup that remembers all active channels for gracefully shutdown. */ private final ChannelGroup allChannels; @@ -208,4 +208,8 @@ public void exceptionCaught(ChannelHandlerContext ctx, Throwable t) { LOG.warn("Encountered ", t); ctx.channel().close(); } + + public ConcurrentHashMap getMap() { + return map; + } } diff --git a/hadoop-common-project/hadoop-nfs/src/test/java/org/apache/hadoop/portmap/TestPortmap.java b/hadoop-common-project/hadoop-nfs/src/test/java/org/apache/hadoop/portmap/TestPortmap.java index 9f4c599587925a..84fa71a269d71a 100644 --- a/hadoop-common-project/hadoop-nfs/src/test/java/org/apache/hadoop/portmap/TestPortmap.java +++ b/hadoop-common-project/hadoop-nfs/src/test/java/org/apache/hadoop/portmap/TestPortmap.java @@ -25,8 +25,6 @@ import java.net.Socket; import java.util.Map; -import org.apache.commons.lang3.reflect.FieldUtils; -import org.apache.hadoop.crypto.key.kms.KMSClientProvider; import org.junit.Assert; import org.apache.hadoop.oncrpc.RpcCall; @@ -101,10 +99,7 @@ public void testRegistration() throws IOException, InterruptedException, Illegal // Give the server a chance to process the request Thread.sleep(100); boolean found = false; - @SuppressWarnings("unchecked") - Map map = (Map) - FieldUtils.getField(RpcProgramPortmap.class, - "map", true).get(pm.getHandler()); + Map map = pm.getHandler().getMap(); for (PortmapMapping m : map.values()) { if (m.getPort() == sent.getPort() From 2eab504ba7989b013c8bda040d67f0c6c4eaa28f Mon Sep 17 00:00:00 2001 From: slfan1989 Date: Fri, 15 Jul 2022 20:30:36 -0700 Subject: [PATCH 10/18] HADOOP-18302. Fix CheckStyle. --- .../hadoop/metrics2/sink/GraphiteSink.java | 276 ++++++++-------- .../metrics2/impl/TestGraphiteMetrics.java | 311 +++++++++--------- 2 files changed, 292 insertions(+), 295 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/GraphiteSink.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/GraphiteSink.java index 32284fbf6b662f..567f68e75bebbe 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/GraphiteSink.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/GraphiteSink.java @@ -37,178 +37,176 @@ import java.nio.charset.StandardCharsets; /** - * A metrics sink that writes to a Graphite server + * A metrics sink that writes to a Graphite server. */ @InterfaceAudience.Public @InterfaceStability.Evolving public class GraphiteSink implements MetricsSink, Closeable { - private static final Logger LOG = - LoggerFactory.getLogger(GraphiteSink.class); - private static final String SERVER_HOST_KEY = "server_host"; - private static final String SERVER_PORT_KEY = "server_port"; - private static final String METRICS_PREFIX = "metrics_prefix"; - private String metricsPrefix = null; - private Graphite graphite = null; - - @Override - public void init(SubsetConfiguration conf) { - // Get Graphite host configurations. - final String serverHost = conf.getString(SERVER_HOST_KEY); - final int serverPort = Integer.parseInt(conf.getString(SERVER_PORT_KEY)); - - // Get Graphite metrics graph prefix. - metricsPrefix = conf.getString(METRICS_PREFIX); - if (metricsPrefix == null) - metricsPrefix = ""; - - graphite = new Graphite(serverHost, serverPort); - graphite.connect(); + private static final Logger LOG = + LoggerFactory.getLogger(GraphiteSink.class); + private static final String SERVER_HOST_KEY = "server_host"; + private static final String SERVER_PORT_KEY = "server_port"; + private static final String METRICS_PREFIX = "metrics_prefix"; + private String metricsPrefix = null; + private Graphite graphite = null; + + @Override + public void init(SubsetConfiguration conf) { + // Get Graphite host configurations. + final String serverHost = conf.getString(SERVER_HOST_KEY); + final int serverPort = Integer.parseInt(conf.getString(SERVER_PORT_KEY)); + + // Get Graphite metrics graph prefix. + metricsPrefix = conf.getString(METRICS_PREFIX); + if (metricsPrefix == null) { + metricsPrefix = ""; } - @Override - public void putMetrics(MetricsRecord record) { - StringBuilder lines = new StringBuilder(); - StringBuilder metricsPathPrefix = new StringBuilder(); - - // Configure the hierarchical place to display the graph. - metricsPathPrefix.append(metricsPrefix).append(".") - .append(record.context()).append(".").append(record.name()); - - for (MetricsTag tag : record.tags()) { - if (tag.value() != null) { - metricsPathPrefix.append(".") - .append(tag.name()) - .append("=") - .append(tag.value()); - } - } - - // The record timestamp is in milliseconds while Graphite expects an epoc time in seconds. - long timestamp = record.timestamp() / 1000L; + graphite = new Graphite(serverHost, serverPort); + graphite.connect(); + } + + @Override + public void putMetrics(MetricsRecord record) { + StringBuilder lines = new StringBuilder(); + StringBuilder metricsPathPrefix = new StringBuilder(); + + // Configure the hierarchical place to display the graph. + metricsPathPrefix.append(metricsPrefix).append(".") + .append(record.context()).append(".").append(record.name()); + + for (MetricsTag tag : record.tags()) { + if (tag.value() != null) { + metricsPathPrefix.append(".") + .append(tag.name()) + .append("=") + .append(tag.value()); + } + } - // Collect datapoints. - for (AbstractMetric metric : record.metrics()) { - lines.append( - metricsPathPrefix.toString() + "." - + metric.name().replace(' ', '.')).append(" ") - .append(metric.value()).append(" ").append(timestamp) - .append("\n"); - } + // The record timestamp is in milliseconds while Graphite expects an epoc time in seconds. + long timestamp = record.timestamp() / 1000L; - try { - graphite.write(lines.toString()); - } catch (Exception e) { - LOG.warn("Error sending metrics to Graphite", e); - try { - graphite.close(); - } catch (Exception e1) { - throw new MetricsException("Error closing connection to Graphite", e1); - } - } + // Collect datapoints. + for (AbstractMetric metric : record.metrics()) { + lines.append(metricsPathPrefix + "." + metric.name().replace(' ', '.')).append(" ") + .append(metric.value()).append(" ").append(timestamp) + .append("\n"); } - @Override - public void flush() { + try { + graphite.write(lines.toString()); + } catch (Exception e) { + LOG.warn("Error sending metrics to Graphite.", e); try { - graphite.flush(); - } catch (Exception e) { - LOG.warn("Error flushing metrics to Graphite", e); - try { - graphite.close(); - } catch (Exception e1) { - throw new MetricsException("Error closing connection to Graphite", e1); - } + graphite.close(); + } catch (Exception e1) { + throw new MetricsException("Error closing connection to Graphite", e1); } } - - @Override - public void close() throws IOException { - graphite.close(); + } + + @Override + public void flush() { + try { + graphite.flush(); + } catch (Exception e) { + LOG.warn("Error flushing metrics to Graphite.", e); + try { + graphite.close(); + } catch (Exception e1) { + throw new MetricsException("Error closing connection to Graphite.", e1); + } } + } - public static class Graphite { - private final static int MAX_CONNECTION_FAILURES = 5; + @Override + public void close() throws IOException { + graphite.close(); + } - private String serverHost; - private int serverPort; - private Writer writer = null; - private Socket socket = null; - private int connectionFailures = 0; + public static class Graphite { + private final static int MAX_CONNECTION_FAILURES = 5; - public Graphite(String serverHost, int serverPort) { - this.serverHost = serverHost; - this.serverPort = serverPort; - } + private String serverHost; + private int serverPort; + private Writer writer = null; + private Socket socket = null; + private int connectionFailures = 0; - public void connect() { - if (isConnected()) { - throw new MetricsException("Already connected to Graphite"); - } - if (tooManyConnectionFailures()) { - // return silently (there was ERROR in logs when we reached limit for the first time) - return; - } - try { + public Graphite(String serverHost, int serverPort) { + this.serverHost = serverHost; + this.serverPort = serverPort; + } + + public void connect() { + if (isConnected()) { + throw new MetricsException("Already connected to Graphite"); + } + if (tooManyConnectionFailures()) { + // return silently (there was ERROR in logs when we reached limit for the first time) + return; + } + try { // Open a connection to Graphite server. - socket = new Socket(serverHost, serverPort); + socket = new Socket(serverHost, serverPort); writer = new OutputStreamWriter(socket.getOutputStream(), StandardCharsets.UTF_8); - } catch (Exception e) { - connectionFailures++; - if (tooManyConnectionFailures()) { - // first time when connection limit reached, report to logs - LOG.error("Too many connection failures, would not try to connect again."); - } - throw new MetricsException("Error creating connection, " - + serverHost + ":" + serverPort, e); + } catch (Exception e) { + connectionFailures++; + if (tooManyConnectionFailures()) { + // first time when connection limit reached, report to logs + LOG.error("Too many connection failures, would not try to connect again."); } + throw new MetricsException("Error creating connection, " + + serverHost + ":" + serverPort, e); } + } - public void write(String msg) throws IOException { - if (!isConnected()) { - connect(); - } - if (isConnected()) { - writer.write(msg); - } + public void write(String msg) throws IOException { + if (!isConnected()) { + connect(); } - - public void flush() throws IOException { - if (isConnected()) { - writer.flush(); - } + if (isConnected()) { + writer.write(msg); } + } - public boolean isConnected() { - return socket != null && socket.isConnected() && !socket.isClosed(); + public void flush() throws IOException { + if (isConnected()) { + writer.flush(); } + } - public void close() throws IOException { - try { - if (writer != null) { - writer.close(); - } - } catch (IOException ex) { - if (socket != null) { - socket.close(); - } - } finally { - socket = null; - writer = null; - } - } + public boolean isConnected() { + return socket != null && socket.isConnected() && !socket.isClosed(); + } - private boolean tooManyConnectionFailures() { - return connectionFailures > MAX_CONNECTION_FAILURES; + public void close() throws IOException { + try { + if (writer != null) { + writer.close(); + } + } catch (IOException ex) { + if (socket != null) { + socket.close(); + } + } finally { + socket = null; + writer = null; } - } - public Graphite getGraphite() { - return graphite; + private boolean tooManyConnectionFailures() { + return connectionFailures > MAX_CONNECTION_FAILURES; } + } - public void setGraphite(Graphite graphite) { - this.graphite = graphite; - } + public Graphite getGraphite() { + return graphite; + } + + public void setGraphite(Graphite graphite) { + this.graphite = graphite; + } } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestGraphiteMetrics.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestGraphiteMetrics.java index ac1b29237a4d78..0ee94705d5b58a 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestGraphiteMetrics.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestGraphiteMetrics.java @@ -43,172 +43,171 @@ public class TestGraphiteMetrics { - private AbstractMetric makeMetric(String name, Number value) { - AbstractMetric metric = mock(AbstractMetric.class); - when(metric.name()).thenReturn(name); - when(metric.value()).thenReturn(value); - return metric; + private AbstractMetric makeMetric(String name, Number value) { + AbstractMetric metric = mock(AbstractMetric.class); + when(metric.name()).thenReturn(name); + when(metric.value()).thenReturn(value); + return metric; + } + + private GraphiteSink.Graphite makeGraphite() { + GraphiteSink.Graphite mockGraphite = mock(GraphiteSink.Graphite.class); + when(mockGraphite.isConnected()).thenReturn(true); + return mockGraphite; + } + + @Test + public void testPutMetrics() { + GraphiteSink sink = new GraphiteSink(); + List tags = new ArrayList<>(); + tags.add(new MetricsTag(MsInfo.Context, "all")); + tags.add(new MetricsTag(MsInfo.Hostname, "host")); + Set metrics = new HashSet<>(); + metrics.add(makeMetric("foo1", 1.25)); + metrics.add(makeMetric("foo2", 2.25)); + MetricsRecord record = new MetricsRecordImpl(MsInfo.Context, 10000, tags, metrics); + + ArgumentCaptor argument = ArgumentCaptor.forClass(String.class); + final GraphiteSink.Graphite mockGraphite = makeGraphite(); + sink.setGraphite(mockGraphite); + sink.putMetrics(record); + + try { + verify(mockGraphite).write(argument.capture()); + } catch (IOException e) { + e.printStackTrace(); } - private GraphiteSink.Graphite makeGraphite() { - GraphiteSink.Graphite mockGraphite = mock(GraphiteSink.Graphite.class); - when(mockGraphite.isConnected()).thenReturn(true); - return mockGraphite; - } + String result = argument.getValue(); - @Test - public void testPutMetrics() { - GraphiteSink sink = new GraphiteSink(); - List tags = new ArrayList(); - tags.add(new MetricsTag(MsInfo.Context, "all")); - tags.add(new MetricsTag(MsInfo.Hostname, "host")); - Set metrics = new HashSet(); - metrics.add(makeMetric("foo1", 1.25)); - metrics.add(makeMetric("foo2", 2.25)); - MetricsRecord record = new MetricsRecordImpl(MsInfo.Context, (long) 10000, tags, metrics); - - ArgumentCaptor argument = ArgumentCaptor.forClass(String.class); - final GraphiteSink.Graphite mockGraphite = makeGraphite(); - sink.setGraphite(mockGraphite); - sink.putMetrics(record); - - try { - verify(mockGraphite).write(argument.capture()); - } catch (IOException e) { - e.printStackTrace(); - } - - String result = argument.getValue(); - - assertEquals(true, - result.equals("null.all.Context.Context=all.Hostname=host.foo1 1.25 10\n" + - "null.all.Context.Context=all.Hostname=host.foo2 2.25 10\n") || - result.equals("null.all.Context.Context=all.Hostname=host.foo2 2.25 10\n" + - "null.all.Context.Context=all.Hostname=host.foo1 1.25 10\n")); + assertEquals(true, + result.equals("null.all.Context.Context=all.Hostname=host.foo1 1.25 10\n" + + "null.all.Context.Context=all.Hostname=host.foo2 2.25 10\n") || + result.equals("null.all.Context.Context=all.Hostname=host.foo2 2.25 10\n" + + "null.all.Context.Context=all.Hostname=host.foo1 1.25 10\n")); } - @Test - public void testPutMetrics2() throws IllegalAccessException { - GraphiteSink sink = new GraphiteSink(); - List tags = new ArrayList(); - tags.add(new MetricsTag(MsInfo.Context, "all")); - tags.add(new MetricsTag(MsInfo.Hostname, null)); - Set metrics = new HashSet(); - metrics.add(makeMetric("foo1", 1)); - metrics.add(makeMetric("foo2", 2)); - MetricsRecord record = new MetricsRecordImpl(MsInfo.Context, (long) 10000, tags, metrics); - - - ArgumentCaptor argument = ArgumentCaptor.forClass(String.class); - final GraphiteSink.Graphite mockGraphite = makeGraphite(); - sink.setGraphite(mockGraphite); - sink.putMetrics(record); - - try { - verify(mockGraphite).write(argument.capture()); - } catch (IOException e) { - e.printStackTrace(); - } - - String result = argument.getValue(); - - assertEquals(true, - result.equals("null.all.Context.Context=all.foo1 1 10\n" + - "null.all.Context.Context=all.foo2 2 10\n") || - result.equals("null.all.Context.Context=all.foo2 2 10\n" + - "null.all.Context.Context=all.foo1 1 10\n")); + @Test + public void testPutMetrics2() throws IllegalAccessException { + GraphiteSink sink = new GraphiteSink(); + List tags = new ArrayList<>(); + tags.add(new MetricsTag(MsInfo.Context, "all")); + tags.add(new MetricsTag(MsInfo.Hostname, null)); + Set metrics = new HashSet<>(); + metrics.add(makeMetric("foo1", 1)); + metrics.add(makeMetric("foo2", 2)); + MetricsRecord record = new MetricsRecordImpl(MsInfo.Context, 10000, tags, metrics); + + ArgumentCaptor argument = ArgumentCaptor.forClass(String.class); + final GraphiteSink.Graphite mockGraphite = makeGraphite(); + sink.setGraphite(mockGraphite); + sink.putMetrics(record); + + try { + verify(mockGraphite).write(argument.capture()); + } catch (IOException e) { + e.printStackTrace(); } - /** - * Assert that timestamps are converted correctly, ticket HADOOP-11182 - */ - @Test - public void testPutMetrics3() throws IllegalAccessException { - - // setup GraphiteSink - GraphiteSink sink = new GraphiteSink(); - final GraphiteSink.Graphite mockGraphite = makeGraphite(); - sink.setGraphite(mockGraphite); - - // given two metrics records with timestamps 1000 milliseconds apart. - List tags = Collections.emptyList(); - Set metrics = new HashSet(); - metrics.add(makeMetric("foo1", 1)); - MetricsRecord record1 = new MetricsRecordImpl(MsInfo.Context, 1000000000000L, tags, metrics); - MetricsRecord record2 = new MetricsRecordImpl(MsInfo.Context, 1000000001000L, tags, metrics); - - sink.putMetrics(record1); - sink.putMetrics(record2); - - sink.flush(); - try { - sink.close(); - } catch(IOException e) { - e.printStackTrace(); - } - - // then the timestamps in the graphite stream should differ by one second. - try { - verify(mockGraphite).write(eq("null.default.Context.foo1 1 1000000000\n")); - verify(mockGraphite).write(eq("null.default.Context.foo1 1 1000000001\n")); - } catch (IOException e) { - e.printStackTrace(); - } + String result = argument.getValue(); + + assertEquals(true, + result.equals("null.all.Context.Context=all.foo1 1 10\n" + + "null.all.Context.Context=all.foo2 2 10\n") || + result.equals("null.all.Context.Context=all.foo2 2 10\n" + + "null.all.Context.Context=all.foo1 1 10\n")); + } + + /** + * Assert that timestamps are converted correctly, ticket HADOOP-11182. + */ + @Test + public void testPutMetrics3() throws IllegalAccessException { + + // setup GraphiteSink + GraphiteSink sink = new GraphiteSink(); + final GraphiteSink.Graphite mockGraphite = makeGraphite(); + sink.setGraphite(mockGraphite); + + // given two metrics records with timestamps 1000 milliseconds apart. + List tags = Collections.emptyList(); + Set metrics = new HashSet<>(); + metrics.add(makeMetric("foo1", 1)); + MetricsRecord record1 = new MetricsRecordImpl(MsInfo.Context, 1000000000000L, tags, metrics); + MetricsRecord record2 = new MetricsRecordImpl(MsInfo.Context, 1000000001000L, tags, metrics); + + sink.putMetrics(record1); + sink.putMetrics(record2); + + sink.flush(); + try { + sink.close(); + } catch(IOException e) { + e.printStackTrace(); } - @Test - public void testFailureAndPutMetrics() throws IOException, IllegalAccessException { - GraphiteSink sink = new GraphiteSink(); - List tags = new ArrayList(); - tags.add(new MetricsTag(MsInfo.Context, "all")); - tags.add(new MetricsTag(MsInfo.Hostname, "host")); - Set metrics = new HashSet(); - metrics.add(makeMetric("foo1", 1.25)); - metrics.add(makeMetric("foo2", 2.25)); - MetricsRecord record = new MetricsRecordImpl(MsInfo.Context, (long) 10000, tags, metrics); - - final GraphiteSink.Graphite mockGraphite = makeGraphite(); - sink.setGraphite(mockGraphite); - - // throw exception when first try - doThrow(new IOException("IO exception")).when(mockGraphite).write(anyString()); - - sink.putMetrics(record); - verify(mockGraphite).write(anyString()); - verify(mockGraphite).close(); - - // reset mock and try again - reset(mockGraphite); - when(mockGraphite.isConnected()).thenReturn(false); - - ArgumentCaptor argument = ArgumentCaptor.forClass(String.class); - sink.putMetrics(record); - - verify(mockGraphite).write(argument.capture()); - String result = argument.getValue(); - - assertEquals(true, - result.equals("null.all.Context.Context=all.Hostname=host.foo1 1.25 10\n" + - "null.all.Context.Context=all.Hostname=host.foo2 2.25 10\n") || - result.equals("null.all.Context.Context=all.Hostname=host.foo2 2.25 10\n" + - "null.all.Context.Context=all.Hostname=host.foo1 1.25 10\n")); + // then the timestamps in the graphite stream should differ by one second. + try { + verify(mockGraphite).write(eq("null.default.Context.foo1 1 1000000000\n")); + verify(mockGraphite).write(eq("null.default.Context.foo1 1 1000000001\n")); + } catch (IOException e) { + e.printStackTrace(); + } + } + + @Test + public void testFailureAndPutMetrics() throws IOException, IllegalAccessException { + GraphiteSink sink = new GraphiteSink(); + List tags = new ArrayList<>(); + tags.add(new MetricsTag(MsInfo.Context, "all")); + tags.add(new MetricsTag(MsInfo.Hostname, "host")); + Set metrics = new HashSet<>(); + metrics.add(makeMetric("foo1", 1.25)); + metrics.add(makeMetric("foo2", 2.25)); + MetricsRecord record = new MetricsRecordImpl(MsInfo.Context, 10000, tags, metrics); + + final GraphiteSink.Graphite mockGraphite = makeGraphite(); + sink.setGraphite(mockGraphite); + + // throw exception when first try + doThrow(new IOException("IO exception")).when(mockGraphite).write(anyString()); + + sink.putMetrics(record); + verify(mockGraphite).write(anyString()); + verify(mockGraphite).close(); + + // reset mock and try again + reset(mockGraphite); + when(mockGraphite.isConnected()).thenReturn(false); + + ArgumentCaptor argument = ArgumentCaptor.forClass(String.class); + sink.putMetrics(record); + + verify(mockGraphite).write(argument.capture()); + String result = argument.getValue(); + + assertEquals(true, + result.equals("null.all.Context.Context=all.Hostname=host.foo1 1.25 10\n" + + "null.all.Context.Context=all.Hostname=host.foo2 2.25 10\n") || + result.equals("null.all.Context.Context=all.Hostname=host.foo2 2.25 10\n" + + "null.all.Context.Context=all.Hostname=host.foo1 1.25 10\n")); + } + + @Test + public void testClose() throws IllegalAccessException { + GraphiteSink sink = new GraphiteSink(); + final GraphiteSink.Graphite mockGraphite = makeGraphite(); + sink.setGraphite(mockGraphite); + try { + sink.close(); + } catch (IOException ioe) { + ioe.printStackTrace(); } - @Test - public void testClose() throws IllegalAccessException { - GraphiteSink sink = new GraphiteSink(); - final GraphiteSink.Graphite mockGraphite = makeGraphite(); - sink.setGraphite(mockGraphite); - try { - sink.close(); - } catch (IOException ioe) { - ioe.printStackTrace(); - } - - try { - verify(mockGraphite).close(); - } catch (IOException ioe) { - ioe.printStackTrace(); - } + try { + verify(mockGraphite).close(); + } catch (IOException ioe) { + ioe.printStackTrace(); } + } } From 89f4e9c2575110910f88bbc7439807761ee5c34c Mon Sep 17 00:00:00 2001 From: slfan1989 Date: Sat, 16 Jul 2022 02:00:09 -0700 Subject: [PATCH 11/18] HADOOP-18302. Remove WhiteBox in hadoop-common module. --- .../java/org/apache/hadoop/fs/RawLocalFileSystem.java | 3 ++- .../main/java/org/apache/hadoop/http/HttpServer2.java | 3 ++- .../src/main/java/org/apache/hadoop/ipc/Server.java | 9 ++++++--- .../java/org/apache/hadoop/ipc/metrics/RpcMetrics.java | 1 + .../apache/hadoop/metrics2/impl/TestGraphiteMetrics.java | 8 ++++---- 5 files changed, 15 insertions(+), 9 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java index 03b1850f2485e7..a009b5242d6c7f 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java @@ -1294,7 +1294,8 @@ public boolean hasPathCapability(final Path path, final String capability) } } - public static void setUseDeprecatedFileStatus(boolean useDeprecatedFileStatus) { + @VisibleForTesting + static void setUseDeprecatedFileStatus(boolean useDeprecatedFileStatus) { RawLocalFileSystem.useDeprecatedFileStatus = useDeprecatedFileStatus; } } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java index af9bcd91ed12a6..cd1516a9f80ed9 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java @@ -1967,7 +1967,8 @@ HttpServer2Metrics getMetrics() { return metrics; } - public List getListeners() { + @VisibleForTesting + List getListeners() { return listeners; } } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java index 0762b3a73b643e..7741af15a61bcb 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java @@ -4131,15 +4131,18 @@ public synchronized void run() { } } - public CallQueueManager getCallQueue() { + @VisibleForTesting + CallQueueManager getCallQueue() { return callQueue; } - public void setCallQueue(CallQueueManager callQueue) { + @VisibleForTesting + void setCallQueue(CallQueueManager callQueue) { this.callQueue = callQueue; } - public void setRpcRequestClass(Class rpcRequestClass) { + @VisibleForTesting + void setRpcRequestClass(Class rpcRequestClass) { this.rpcRequestClass = rpcRequestClass; } } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/metrics/RpcMetrics.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/metrics/RpcMetrics.java index 9459dcf5a95255..f01cd5bcfda946 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/metrics/RpcMetrics.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/metrics/RpcMetrics.java @@ -365,6 +365,7 @@ public MetricsTag getTag(String tagName) { return registry.getTag(tagName); } + @VisibleForTesting public MutableCounterLong getRpcAuthorizationSuccesses() { return rpcAuthorizationSuccesses; } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestGraphiteMetrics.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestGraphiteMetrics.java index 0ee94705d5b58a..221379fda727b5 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestGraphiteMetrics.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestGraphiteMetrics.java @@ -199,15 +199,15 @@ public void testClose() throws IllegalAccessException { final GraphiteSink.Graphite mockGraphite = makeGraphite(); sink.setGraphite(mockGraphite); try { - sink.close(); + sink.close(); } catch (IOException ioe) { - ioe.printStackTrace(); + ioe.printStackTrace(); } try { - verify(mockGraphite).close(); + verify(mockGraphite).close(); } catch (IOException ioe) { - ioe.printStackTrace(); + ioe.printStackTrace(); } } } From bcbafea1f6582356c7932a67061ccff7e9962c01 Mon Sep 17 00:00:00 2001 From: slfan1989 Date: Thu, 18 Aug 2022 20:48:21 +0800 Subject: [PATCH 12/18] YARN-18302. Fix CheckStyle. --- .../src/test/java/org/apache/hadoop/http/TestHttpServer.java | 4 ++-- .../src/test/java/org/apache/hadoop/ipc/TestIPC.java | 1 - .../src/test/java/org/apache/hadoop/ipc/TestRPC.java | 2 -- .../java/org/apache/hadoop/portmap/RpcProgramPortmap.java | 3 ++- 4 files changed, 4 insertions(+), 6 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServer.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServer.java index dfe6723d6c51ab..062033d4d563f3 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServer.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServer.java @@ -662,7 +662,7 @@ private HttpServer2 checkBindAddress(String host, int port, boolean findPort) HttpServer2 server = createServer(host, port); try { // not bound, ephemeral should return requested port (0 for ephemeral) - List listeners = server.getListeners(); + List listeners = server.getListeners(); ServerConnector listener = (ServerConnector)listeners.get(0); assertEquals(port, listener.getPort()); @@ -738,7 +738,7 @@ public void testBacklogSize() throws Exception Configuration conf = new Configuration(); conf.setInt(HttpServer2.HTTP_SOCKET_BACKLOG_SIZE_KEY, backlogSize); HttpServer2 srv = createServer("test", conf); - List listeners = srv.getListeners(); + List listeners = srv.getListeners(); ServerConnector listener = (ServerConnector)listeners.get(0); assertEquals(backlogSize, listener.getAcceptQueueSize()); } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestIPC.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestIPC.java index 77225a19302907..0e5bf6f5f5449d 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestIPC.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestIPC.java @@ -860,7 +860,6 @@ public void testIpcWithReaderQueuing() throws Exception { // goal is to jam a handler with a connection, fill the callq with // connections, in turn jamming the readers - then flood the server and // ensure that the listener blocks when the reader connection queues fill - @SuppressWarnings("unchecked") private void checkBlocking(int readers, int readerQ, int callQ) throws Exception { int handlers = 1; // makes it easier diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPC.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPC.java index 6221bb1a20dcba..49095b6c61a59c 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPC.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPC.java @@ -1217,7 +1217,6 @@ public void testClientBackOff() throws Exception { .setQueueSizePerHandler(1).setNumHandlers(1).setVerbose(true); server = setupTestServer(builder); - @SuppressWarnings("unchecked") CallQueueManager spy = spy(server.getCallQueue()); server.setCallQueue(spy); @@ -1279,7 +1278,6 @@ public void testClientBackOffByResponseTime() throws Exception { final String ns = CommonConfigurationKeys.IPC_NAMESPACE + ".0"; Server server = setupDecayRpcSchedulerandTestServer(ns + "."); - @SuppressWarnings("unchecked") CallQueueManager spy = spy(server.getCallQueue()); server.setCallQueue(spy); diff --git a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/portmap/RpcProgramPortmap.java b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/portmap/RpcProgramPortmap.java index 10afaacff04ad8..a585dbc6b20b09 100644 --- a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/portmap/RpcProgramPortmap.java +++ b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/portmap/RpcProgramPortmap.java @@ -18,6 +18,7 @@ package org.apache.hadoop.portmap; import java.util.concurrent.ConcurrentHashMap; +import java.util.Map; import io.netty.buffer.ByteBuf; import io.netty.buffer.Unpooled; @@ -209,7 +210,7 @@ public void exceptionCaught(ChannelHandlerContext ctx, Throwable t) { ctx.channel().close(); } - public ConcurrentHashMap getMap() { + public Map getMap() { return map; } } From bcaa569abc0a78375f15b9852b5d5da8f73d860b Mon Sep 17 00:00:00 2001 From: slfan1989 Date: Thu, 18 Aug 2022 20:53:38 +0800 Subject: [PATCH 13/18] YARN-18302. Fix CheckStyle. --- .../sink/TestAbstractMetricsRecord.java | 55 ++++++++++++ .../{impl => sink}/TestGraphiteMetrics.java | 19 ++-- .../metrics2/sink/TestMetricsRecordImpl.java | 89 +++++++++++++++++++ 3 files changed, 156 insertions(+), 7 deletions(-) create mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/sink/TestAbstractMetricsRecord.java rename hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/{impl => sink}/TestGraphiteMetrics.java (92%) create mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/sink/TestMetricsRecordImpl.java diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/sink/TestAbstractMetricsRecord.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/sink/TestAbstractMetricsRecord.java new file mode 100644 index 00000000000000..88f6a9892b9447 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/sink/TestAbstractMetricsRecord.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.metrics2.sink; + +import org.apache.hadoop.metrics2.MetricsRecord; +import org.apache.hadoop.thirdparty.com.google.common.base.Objects; +import org.apache.hadoop.thirdparty.com.google.common.collect.Iterables; + +import java.util.StringJoiner; + +abstract class TestAbstractMetricsRecord implements MetricsRecord { + + @Override public boolean equals(Object obj) { + if (obj instanceof MetricsRecord) { + final MetricsRecord other = (MetricsRecord) obj; + return Objects.equal(timestamp(), other.timestamp()) && + Objects.equal(name(), other.name()) && + Objects.equal(description(), other.description()) && + Objects.equal(tags(), other.tags()) && + Iterables.elementsEqual(metrics(), other.metrics()); + } + return false; + } + + // Should make sense most of the time when the record is used as a key + @Override public int hashCode() { + return Objects.hashCode(name(), description(), tags()); + } + + @Override public String toString() { + return new StringJoiner(", ", this.getClass().getSimpleName() + "{", "}") + .add("timestamp=" + timestamp()) + .add("name=" + name()) + .add("description=" + description()) + .add("tags=" + tags()) + .add("metrics=" + Iterables.toString(metrics())) + .toString(); + } +} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestGraphiteMetrics.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/sink/TestGraphiteMetrics.java similarity index 92% rename from hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestGraphiteMetrics.java rename to hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/sink/TestGraphiteMetrics.java index 221379fda727b5..1cf390c27f5225 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestGraphiteMetrics.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/sink/TestGraphiteMetrics.java @@ -16,12 +16,12 @@ * limitations under the License. */ -package org.apache.hadoop.metrics2.impl; +package org.apache.hadoop.metrics2.sink; import org.apache.hadoop.metrics2.AbstractMetric; import org.apache.hadoop.metrics2.MetricsRecord; import org.apache.hadoop.metrics2.MetricsTag; -import org.apache.hadoop.metrics2.sink.GraphiteSink; +import org.apache.hadoop.metrics2.impl.MsInfo; import org.junit.Test; import org.mockito.ArgumentCaptor; @@ -65,7 +65,8 @@ public void testPutMetrics() { Set metrics = new HashSet<>(); metrics.add(makeMetric("foo1", 1.25)); metrics.add(makeMetric("foo2", 2.25)); - MetricsRecord record = new MetricsRecordImpl(MsInfo.Context, 10000, tags, metrics); + MetricsRecord record = + new TestMetricsRecordImpl(MsInfo.Context, 10000, tags, metrics); ArgumentCaptor argument = ArgumentCaptor.forClass(String.class); final GraphiteSink.Graphite mockGraphite = makeGraphite(); @@ -96,7 +97,8 @@ public void testPutMetrics2() throws IllegalAccessException { Set metrics = new HashSet<>(); metrics.add(makeMetric("foo1", 1)); metrics.add(makeMetric("foo2", 2)); - MetricsRecord record = new MetricsRecordImpl(MsInfo.Context, 10000, tags, metrics); + MetricsRecord record = + new TestMetricsRecordImpl(MsInfo.Context, 10000, tags, metrics); ArgumentCaptor argument = ArgumentCaptor.forClass(String.class); final GraphiteSink.Graphite mockGraphite = makeGraphite(); @@ -133,8 +135,10 @@ public void testPutMetrics3() throws IllegalAccessException { List tags = Collections.emptyList(); Set metrics = new HashSet<>(); metrics.add(makeMetric("foo1", 1)); - MetricsRecord record1 = new MetricsRecordImpl(MsInfo.Context, 1000000000000L, tags, metrics); - MetricsRecord record2 = new MetricsRecordImpl(MsInfo.Context, 1000000001000L, tags, metrics); + MetricsRecord record1 = + new TestMetricsRecordImpl(MsInfo.Context, 1000000000000L, tags, metrics); + MetricsRecord record2 = + new TestMetricsRecordImpl(MsInfo.Context, 1000000001000L, tags, metrics); sink.putMetrics(record1); sink.putMetrics(record2); @@ -164,7 +168,8 @@ public void testFailureAndPutMetrics() throws IOException, IllegalAccessExceptio Set metrics = new HashSet<>(); metrics.add(makeMetric("foo1", 1.25)); metrics.add(makeMetric("foo2", 2.25)); - MetricsRecord record = new MetricsRecordImpl(MsInfo.Context, 10000, tags, metrics); + MetricsRecord record = + new TestMetricsRecordImpl(MsInfo.Context, 10000, tags, metrics); final GraphiteSink.Graphite mockGraphite = makeGraphite(); sink.setGraphite(mockGraphite); diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/sink/TestMetricsRecordImpl.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/sink/TestMetricsRecordImpl.java new file mode 100644 index 00000000000000..7905dea55511b2 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/sink/TestMetricsRecordImpl.java @@ -0,0 +1,89 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.metrics2.sink; + +import org.apache.hadoop.metrics2.AbstractMetric; +import org.apache.hadoop.metrics2.MetricsInfo; +import org.apache.hadoop.metrics2.MetricsTag; +import org.apache.hadoop.metrics2.impl.MsInfo; + +import java.util.List; + +import static org.apache.hadoop.metrics2.util.Contracts.checkArg; +import static org.apache.hadoop.util.Preconditions.checkNotNull; + +class TestMetricsRecordImpl extends TestAbstractMetricsRecord { + protected static final String DEFAULT_CONTEXT = "default"; + + private final long timestamp; + private final MetricsInfo info; + private final List tags; + private final Iterable metrics; + + /** + * Construct a metrics record + * @param info {@link MetricsInfo} of the record + * @param timestamp of the record + * @param tags of the record + * @param metrics of the record + */ + public TestMetricsRecordImpl(MetricsInfo info, long timestamp, + List tags, + Iterable metrics) { + this.timestamp = checkArg(timestamp, timestamp > 0, "timestamp"); + this.info = checkNotNull(info, "info"); + this.tags = checkNotNull(tags, "tags"); + this.metrics = checkNotNull(metrics, "metrics"); + } + + @Override public long timestamp() { + return timestamp; + } + + @Override public String name() { + return info.name(); + } + + MetricsInfo info() { + return info; + } + + @Override public String description() { + return info.description(); + } + + @Override public String context() { + // usually the first tag + for (MetricsTag t : tags) { + if (t.info() == MsInfo.Context) { + return t.value(); + } + } + return DEFAULT_CONTEXT; + } + + @Override + public List tags() { + return tags; // already unmodifiable from MetricsRecordBuilderImpl#tags + } + + @Override public Iterable metrics() { + return metrics; + } +} From fa7bbfb1d460b757ec9d50789d7b66c689731ed5 Mon Sep 17 00:00:00 2001 From: slfan1989 Date: Fri, 19 Aug 2022 07:35:42 +0800 Subject: [PATCH 14/18] YARN-18302. Fix CodeStyle. --- .../apache/hadoop/metrics2/sink/GraphiteSink.java | 8 +++----- .../apache/hadoop/metrics2/sink/StatsDSink.java | 3 +++ .../{impl => sink}/TestStatsDMetrics.java | 15 ++++++--------- 3 files changed, 12 insertions(+), 14 deletions(-) rename hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/{impl => sink}/TestStatsDMetrics.java (89%) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/GraphiteSink.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/GraphiteSink.java index 567f68e75bebbe..e07260c99936f2 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/GraphiteSink.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/GraphiteSink.java @@ -21,6 +21,7 @@ import org.apache.commons.configuration2.SubsetConfiguration; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.classification.VisibleForTesting; import org.apache.hadoop.metrics2.AbstractMetric; import org.apache.hadoop.metrics2.MetricsException; import org.apache.hadoop.metrics2.MetricsRecord; @@ -202,11 +203,8 @@ private boolean tooManyConnectionFailures() { } } - public Graphite getGraphite() { - return graphite; - } - - public void setGraphite(Graphite graphite) { + @VisibleForTesting + void setGraphite(Graphite graphite) { this.graphite = graphite; } } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/StatsDSink.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/StatsDSink.java index d1ec47fdecb312..7e566456d4369e 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/StatsDSink.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/StatsDSink.java @@ -215,4 +215,7 @@ public void close() throws IOException { } + void setStatsd(StatsD statsd) { + this.statsd = statsd; + } } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestStatsDMetrics.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/sink/TestStatsDMetrics.java similarity index 89% rename from hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestStatsDMetrics.java rename to hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/sink/TestStatsDMetrics.java index 5e2818969b7d2c..05d9687a1916e9 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestStatsDMetrics.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/sink/TestStatsDMetrics.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.hadoop.metrics2.impl; +package org.apache.hadoop.metrics2.sink; import static org.junit.Assert.assertTrue; import static org.mockito.Mockito.mock; @@ -31,12 +31,11 @@ import java.util.List; import java.util.Set; -import org.apache.commons.lang3.reflect.FieldUtils; import org.apache.hadoop.metrics2.AbstractMetric; import org.apache.hadoop.metrics2.MetricType; import org.apache.hadoop.metrics2.MetricsRecord; import org.apache.hadoop.metrics2.MetricsTag; -import org.apache.hadoop.metrics2.sink.StatsDSink; +import org.apache.hadoop.metrics2.impl.MsInfo; import org.apache.hadoop.metrics2.sink.StatsDSink.StatsD; import org.junit.Test; @@ -62,15 +61,14 @@ public void testPutMetrics() throws IOException, IllegalAccessException { metrics.add(makeMetric("foo1", 1.25, MetricType.COUNTER)); metrics.add(makeMetric("foo2", 2.25, MetricType.GAUGE)); final MetricsRecord record = - new MetricsRecordImpl(MsInfo.Context, (long) 10000, tags, metrics); + new TestMetricsRecordImpl(MsInfo.Context, (long) 10000, tags, metrics); try (DatagramSocket sock = new DatagramSocket()) { sock.setReceiveBufferSize(8192); final StatsDSink.StatsD mockStatsD = new StatsD(sock.getLocalAddress().getHostName(), sock.getLocalPort()); - FieldUtils.getField(StatsDSink.class, "statsd", true). - set(sink, mockStatsD); + sink.setStatsd(mockStatsD); final DatagramPacket p = new DatagramPacket(new byte[8192], 8192); sink.putMetrics(record); sock.receive(p); @@ -98,15 +96,14 @@ public void testPutMetrics2() throws IOException, IllegalAccessException { metrics.add(makeMetric("foo1", 1, MetricType.COUNTER)); metrics.add(makeMetric("foo2", 2, MetricType.GAUGE)); MetricsRecord record = - new MetricsRecordImpl(MsInfo.Context, (long) 10000, tags, metrics); + new TestMetricsRecordImpl(MsInfo.Context, (long) 10000, tags, metrics); try (DatagramSocket sock = new DatagramSocket()) { sock.setReceiveBufferSize(8192); final StatsDSink.StatsD mockStatsD = new StatsD(sock.getLocalAddress().getHostName(), sock.getLocalPort()); - FieldUtils.getField(StatsDSink.class, "statsd", true). - set(sink, mockStatsD); + sink.setStatsd(mockStatsD); final DatagramPacket p = new DatagramPacket(new byte[8192], 8192); sink.putMetrics(record); sock.receive(p); From 839dc70735b3e53c8bc968e2c34ad6a02b067b4e Mon Sep 17 00:00:00 2001 From: slfan1989 Date: Fri, 19 Aug 2022 14:11:06 +0800 Subject: [PATCH 15/18] HADOOP-18302. Fix CheckStyle. --- .../apache/hadoop/metrics2/sink/TestMetricsRecordImpl.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/sink/TestMetricsRecordImpl.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/sink/TestMetricsRecordImpl.java index 7905dea55511b2..e6981c2df58650 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/sink/TestMetricsRecordImpl.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/sink/TestMetricsRecordImpl.java @@ -37,15 +37,14 @@ class TestMetricsRecordImpl extends TestAbstractMetricsRecord { private final Iterable metrics; /** - * Construct a metrics record + * Construct a metrics record. * @param info {@link MetricsInfo} of the record * @param timestamp of the record * @param tags of the record * @param metrics of the record */ - public TestMetricsRecordImpl(MetricsInfo info, long timestamp, - List tags, - Iterable metrics) { + TestMetricsRecordImpl(MetricsInfo info, long timestamp, List tags, + Iterable metrics) { this.timestamp = checkArg(timestamp, timestamp > 0, "timestamp"); this.info = checkNotNull(info, "info"); this.tags = checkNotNull(tags, "tags"); From bff5a07230cbce76363f2290afb2939318a45ea3 Mon Sep 17 00:00:00 2001 From: slfan1989 Date: Tue, 30 Aug 2022 07:42:24 -0700 Subject: [PATCH 16/18] HADOOP-18302. Fix CheckStyle. --- .../metrics2/impl/MetricsRecordImpl.java | 2 +- .../hadoop/metrics2/sink/StatsDSink.java | 3 +- .../metrics2/sink/TestGraphiteMetrics.java | 11 +-- .../metrics2/sink/TestMetricsRecordImpl.java | 88 ------------------- .../metrics2/sink/TestStatsDMetrics.java | 5 +- 5 files changed, 12 insertions(+), 97 deletions(-) delete mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/sink/TestMetricsRecordImpl.java diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsRecordImpl.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsRecordImpl.java index 9ffceaaa0ddda3..40c551dfda56aa 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsRecordImpl.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsRecordImpl.java @@ -27,7 +27,7 @@ import org.apache.hadoop.metrics2.MetricsTag; import static org.apache.hadoop.metrics2.util.Contracts.*; -class MetricsRecordImpl extends AbstractMetricsRecord { +public class MetricsRecordImpl extends AbstractMetricsRecord { protected static final String DEFAULT_CONTEXT = "default"; private final long timestamp; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/StatsDSink.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/StatsDSink.java index 7e566456d4369e..4f41c0b0057ce5 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/StatsDSink.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/StatsDSink.java @@ -28,6 +28,7 @@ import org.apache.commons.configuration2.SubsetConfiguration; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.classification.VisibleForTesting; import org.apache.hadoop.metrics2.AbstractMetric; import org.apache.hadoop.metrics2.MetricType; import org.apache.hadoop.metrics2.MetricsException; @@ -214,7 +215,7 @@ public void close() throws IOException { } } - + @VisibleForTesting void setStatsd(StatsD statsd) { this.statsd = statsd; } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/sink/TestGraphiteMetrics.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/sink/TestGraphiteMetrics.java index 1cf390c27f5225..9ea81c6e4c62e9 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/sink/TestGraphiteMetrics.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/sink/TestGraphiteMetrics.java @@ -21,6 +21,7 @@ import org.apache.hadoop.metrics2.AbstractMetric; import org.apache.hadoop.metrics2.MetricsRecord; import org.apache.hadoop.metrics2.MetricsTag; +import org.apache.hadoop.metrics2.impl.MetricsRecordImpl; import org.apache.hadoop.metrics2.impl.MsInfo; import org.junit.Test; import org.mockito.ArgumentCaptor; @@ -66,7 +67,7 @@ public void testPutMetrics() { metrics.add(makeMetric("foo1", 1.25)); metrics.add(makeMetric("foo2", 2.25)); MetricsRecord record = - new TestMetricsRecordImpl(MsInfo.Context, 10000, tags, metrics); + new MetricsRecordImpl(MsInfo.Context, 10000, tags, metrics); ArgumentCaptor argument = ArgumentCaptor.forClass(String.class); final GraphiteSink.Graphite mockGraphite = makeGraphite(); @@ -98,7 +99,7 @@ public void testPutMetrics2() throws IllegalAccessException { metrics.add(makeMetric("foo1", 1)); metrics.add(makeMetric("foo2", 2)); MetricsRecord record = - new TestMetricsRecordImpl(MsInfo.Context, 10000, tags, metrics); + new MetricsRecordImpl(MsInfo.Context, 10000, tags, metrics); ArgumentCaptor argument = ArgumentCaptor.forClass(String.class); final GraphiteSink.Graphite mockGraphite = makeGraphite(); @@ -136,9 +137,9 @@ public void testPutMetrics3() throws IllegalAccessException { Set metrics = new HashSet<>(); metrics.add(makeMetric("foo1", 1)); MetricsRecord record1 = - new TestMetricsRecordImpl(MsInfo.Context, 1000000000000L, tags, metrics); + new MetricsRecordImpl(MsInfo.Context, 1000000000000L, tags, metrics); MetricsRecord record2 = - new TestMetricsRecordImpl(MsInfo.Context, 1000000001000L, tags, metrics); + new MetricsRecordImpl(MsInfo.Context, 1000000001000L, tags, metrics); sink.putMetrics(record1); sink.putMetrics(record2); @@ -169,7 +170,7 @@ public void testFailureAndPutMetrics() throws IOException, IllegalAccessExceptio metrics.add(makeMetric("foo1", 1.25)); metrics.add(makeMetric("foo2", 2.25)); MetricsRecord record = - new TestMetricsRecordImpl(MsInfo.Context, 10000, tags, metrics); + new MetricsRecordImpl(MsInfo.Context, 10000, tags, metrics); final GraphiteSink.Graphite mockGraphite = makeGraphite(); sink.setGraphite(mockGraphite); diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/sink/TestMetricsRecordImpl.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/sink/TestMetricsRecordImpl.java deleted file mode 100644 index e6981c2df58650..00000000000000 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/sink/TestMetricsRecordImpl.java +++ /dev/null @@ -1,88 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.metrics2.sink; - -import org.apache.hadoop.metrics2.AbstractMetric; -import org.apache.hadoop.metrics2.MetricsInfo; -import org.apache.hadoop.metrics2.MetricsTag; -import org.apache.hadoop.metrics2.impl.MsInfo; - -import java.util.List; - -import static org.apache.hadoop.metrics2.util.Contracts.checkArg; -import static org.apache.hadoop.util.Preconditions.checkNotNull; - -class TestMetricsRecordImpl extends TestAbstractMetricsRecord { - protected static final String DEFAULT_CONTEXT = "default"; - - private final long timestamp; - private final MetricsInfo info; - private final List tags; - private final Iterable metrics; - - /** - * Construct a metrics record. - * @param info {@link MetricsInfo} of the record - * @param timestamp of the record - * @param tags of the record - * @param metrics of the record - */ - TestMetricsRecordImpl(MetricsInfo info, long timestamp, List tags, - Iterable metrics) { - this.timestamp = checkArg(timestamp, timestamp > 0, "timestamp"); - this.info = checkNotNull(info, "info"); - this.tags = checkNotNull(tags, "tags"); - this.metrics = checkNotNull(metrics, "metrics"); - } - - @Override public long timestamp() { - return timestamp; - } - - @Override public String name() { - return info.name(); - } - - MetricsInfo info() { - return info; - } - - @Override public String description() { - return info.description(); - } - - @Override public String context() { - // usually the first tag - for (MetricsTag t : tags) { - if (t.info() == MsInfo.Context) { - return t.value(); - } - } - return DEFAULT_CONTEXT; - } - - @Override - public List tags() { - return tags; // already unmodifiable from MetricsRecordBuilderImpl#tags - } - - @Override public Iterable metrics() { - return metrics; - } -} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/sink/TestStatsDMetrics.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/sink/TestStatsDMetrics.java index 05d9687a1916e9..99a75787ad8413 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/sink/TestStatsDMetrics.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/sink/TestStatsDMetrics.java @@ -35,6 +35,7 @@ import org.apache.hadoop.metrics2.MetricType; import org.apache.hadoop.metrics2.MetricsRecord; import org.apache.hadoop.metrics2.MetricsTag; +import org.apache.hadoop.metrics2.impl.MetricsRecordImpl; import org.apache.hadoop.metrics2.impl.MsInfo; import org.apache.hadoop.metrics2.sink.StatsDSink.StatsD; import org.junit.Test; @@ -61,7 +62,7 @@ public void testPutMetrics() throws IOException, IllegalAccessException { metrics.add(makeMetric("foo1", 1.25, MetricType.COUNTER)); metrics.add(makeMetric("foo2", 2.25, MetricType.GAUGE)); final MetricsRecord record = - new TestMetricsRecordImpl(MsInfo.Context, (long) 10000, tags, metrics); + new MetricsRecordImpl(MsInfo.Context, (long) 10000, tags, metrics); try (DatagramSocket sock = new DatagramSocket()) { sock.setReceiveBufferSize(8192); @@ -96,7 +97,7 @@ public void testPutMetrics2() throws IOException, IllegalAccessException { metrics.add(makeMetric("foo1", 1, MetricType.COUNTER)); metrics.add(makeMetric("foo2", 2, MetricType.GAUGE)); MetricsRecord record = - new TestMetricsRecordImpl(MsInfo.Context, (long) 10000, tags, metrics); + new MetricsRecordImpl(MsInfo.Context, (long) 10000, tags, metrics); try (DatagramSocket sock = new DatagramSocket()) { sock.setReceiveBufferSize(8192); From 27fd62472e5c303a23a7d7928c297e9e3289c5cf Mon Sep 17 00:00:00 2001 From: slfan1989 Date: Tue, 30 Aug 2022 07:45:04 -0700 Subject: [PATCH 17/18] HADOOP-18302. Fix CheckStyle. --- .../sink/TestAbstractMetricsRecord.java | 55 ------------------- 1 file changed, 55 deletions(-) delete mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/sink/TestAbstractMetricsRecord.java diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/sink/TestAbstractMetricsRecord.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/sink/TestAbstractMetricsRecord.java deleted file mode 100644 index 88f6a9892b9447..00000000000000 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/sink/TestAbstractMetricsRecord.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.metrics2.sink; - -import org.apache.hadoop.metrics2.MetricsRecord; -import org.apache.hadoop.thirdparty.com.google.common.base.Objects; -import org.apache.hadoop.thirdparty.com.google.common.collect.Iterables; - -import java.util.StringJoiner; - -abstract class TestAbstractMetricsRecord implements MetricsRecord { - - @Override public boolean equals(Object obj) { - if (obj instanceof MetricsRecord) { - final MetricsRecord other = (MetricsRecord) obj; - return Objects.equal(timestamp(), other.timestamp()) && - Objects.equal(name(), other.name()) && - Objects.equal(description(), other.description()) && - Objects.equal(tags(), other.tags()) && - Iterables.elementsEqual(metrics(), other.metrics()); - } - return false; - } - - // Should make sense most of the time when the record is used as a key - @Override public int hashCode() { - return Objects.hashCode(name(), description(), tags()); - } - - @Override public String toString() { - return new StringJoiner(", ", this.getClass().getSimpleName() + "{", "}") - .add("timestamp=" + timestamp()) - .add("name=" + name()) - .add("description=" + description()) - .add("tags=" + tags()) - .add("metrics=" + Iterables.toString(metrics())) - .toString(); - } -} From 82a3cc9194081cc05d2c929ab6736fac60690e60 Mon Sep 17 00:00:00 2001 From: slfan1989 Date: Sun, 11 Sep 2022 04:22:09 -0700 Subject: [PATCH 18/18] HADOOP-18302. Fix CheckStyle. --- .../java/org/apache/hadoop/metrics2/impl/MetricsRecordImpl.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsRecordImpl.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsRecordImpl.java index 40c551dfda56aa..b11f775a73db3c 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsRecordImpl.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsRecordImpl.java @@ -22,11 +22,13 @@ import static org.apache.hadoop.util.Preconditions.*; +import org.apache.hadoop.classification.VisibleForTesting; import org.apache.hadoop.metrics2.MetricsInfo; import org.apache.hadoop.metrics2.AbstractMetric; import org.apache.hadoop.metrics2.MetricsTag; import static org.apache.hadoop.metrics2.util.Contracts.*; +@VisibleForTesting public class MetricsRecordImpl extends AbstractMetricsRecord { protected static final String DEFAULT_CONTEXT = "default";