From 72a9826f9655552ab82dfd16b6f1d789c32b7da7 Mon Sep 17 00:00:00 2001 From: Kaijie Chen Date: Fri, 13 May 2022 15:51:32 +0800 Subject: [PATCH 01/10] Add listBlock in ContainerProtocolCalls --- .../scm/storage/ContainerProtocolCalls.java | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/storage/ContainerProtocolCalls.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/storage/ContainerProtocolCalls.java index 259787070e56..48bc64f66637 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/storage/ContainerProtocolCalls.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/storage/ContainerProtocolCalls.java @@ -40,6 +40,8 @@ import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.GetSmallFileRequestProto; import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.GetSmallFileResponseProto; import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.KeyValue; +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ListBlockRequestProto; +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ListBlockResponseProto; import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.PutBlockRequestProto; import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.PutSmallFileRequestProto; import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.PutSmallFileResponseProto; @@ -73,6 +75,50 @@ public final class ContainerProtocolCalls { private ContainerProtocolCalls() { } + /** + * Calls the container protocol to list blocks in container. + * + * @param xceiverClient client to perform call + * @param containerID the ID of the container to list block + * @param replicaIndex the index of the replica in pipeline + * @param startLocalID the localID of the first block to get + * @param count max number of blocks to get + * @param token a token for this block (may be null) + * @return container protocol list block response + * @throws IOException if there is an I/O error while performing the call + */ + public static ListBlockResponseProto listBlock(XceiverClientSpi xceiverClient, + long containerID, int replicaIndex, Long startLocalID, int count, + Token token) throws IOException { + + ListBlockRequestProto.Builder listBlockBuilder = + ListBlockRequestProto.newBuilder() + .setCount(count); + + if (startLocalID != null) { + listBlockBuilder.setStartLocalID(startLocalID); + } + + String datanodeID = xceiverClient.getPipeline().getNodesInOrder() + .get(replicaIndex).getUuidString(); + + ContainerCommandRequestProto.Builder builder = + ContainerCommandRequestProto.newBuilder() + .setCmdType(Type.ListBlock) + .setContainerID(containerID) + .setDatanodeUuid(datanodeID) + .setListBlock(listBlockBuilder.build()); + + if (token != null) { + builder.setEncodedToken(token.encodeToUrlString()); + } + + ContainerCommandRequestProto request = builder.build(); + ContainerCommandResponseProto response = + xceiverClient.sendCommand(request, getValidatorList()); + return response.getListBlock(); + } + /** * Calls the container protocol to get a container block. * From 8e85ec009109930e6a7be766456dad66c1d887bd Mon Sep 17 00:00:00 2001 From: Kaijie Chen Date: Mon, 16 May 2022 16:10:32 +0800 Subject: [PATCH 02/10] DatanodeID doen't matter --- .../hadoop/hdds/scm/storage/ContainerProtocolCalls.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/storage/ContainerProtocolCalls.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/storage/ContainerProtocolCalls.java index 48bc64f66637..a10e1cccb10d 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/storage/ContainerProtocolCalls.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/storage/ContainerProtocolCalls.java @@ -80,7 +80,6 @@ private ContainerProtocolCalls() { * * @param xceiverClient client to perform call * @param containerID the ID of the container to list block - * @param replicaIndex the index of the replica in pipeline * @param startLocalID the localID of the first block to get * @param count max number of blocks to get * @param token a token for this block (may be null) @@ -88,7 +87,7 @@ private ContainerProtocolCalls() { * @throws IOException if there is an I/O error while performing the call */ public static ListBlockResponseProto listBlock(XceiverClientSpi xceiverClient, - long containerID, int replicaIndex, Long startLocalID, int count, + long containerID, Long startLocalID, int count, Token token) throws IOException { ListBlockRequestProto.Builder listBlockBuilder = @@ -99,8 +98,9 @@ public static ListBlockResponseProto listBlock(XceiverClientSpi xceiverClient, listBlockBuilder.setStartLocalID(startLocalID); } - String datanodeID = xceiverClient.getPipeline().getNodesInOrder() - .get(replicaIndex).getUuidString(); + // datanodeID doesn't matter for read only requests + String datanodeID = + xceiverClient.getPipeline().getFirstNode().getUuidString(); ContainerCommandRequestProto.Builder builder = ContainerCommandRequestProto.newBuilder() From f4ad80b268dffd4d5fad51b11cff1bb54649226f Mon Sep 17 00:00:00 2001 From: Kaijie Chen Date: Mon, 16 May 2022 18:50:46 +0800 Subject: [PATCH 03/10] Add integration test --- .../TestECReconstructionCommands.java | 246 ++++++++++++++++++ 1 file changed, 246 insertions(+) create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ec/reconstruction/TestECReconstructionCommands.java diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ec/reconstruction/TestECReconstructionCommands.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ec/reconstruction/TestECReconstructionCommands.java new file mode 100644 index 000000000000..7f5a07802c5a --- /dev/null +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ec/reconstruction/TestECReconstructionCommands.java @@ -0,0 +1,246 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.ozone.container.ec.reconstruction; + +import com.google.common.collect.ImmutableList; +import org.apache.commons.lang3.RandomUtils; +import org.apache.hadoop.hdds.client.ECReplicationConfig; +import org.apache.hadoop.hdds.client.ECReplicationConfig.EcCodec; +import org.apache.hadoop.hdds.client.ReplicationConfig; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.protocol.DatanodeDetails; +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.BlockData; +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ListBlockResponseProto; +import org.apache.hadoop.hdds.scm.ScmConfigKeys; +import org.apache.hadoop.hdds.scm.XceiverClientGrpc; +import org.apache.hadoop.hdds.scm.XceiverClientSpi; +import org.apache.hadoop.hdds.scm.container.ContainerID; +import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException; +import org.apache.hadoop.hdds.scm.pipeline.Pipeline; +import org.apache.hadoop.hdds.scm.pipeline.WritableECContainerProvider.WritableECContainerProviderConfig ; +import org.apache.hadoop.hdds.scm.protocolPB.StorageContainerLocationProtocolClientSideTranslatorPB; +import org.apache.hadoop.hdds.scm.server.StorageContainerManager; +import org.apache.hadoop.hdds.scm.storage.ContainerProtocolCalls; +import org.apache.hadoop.ozone.MiniOzoneCluster; +import org.apache.hadoop.ozone.OzoneConfigKeys; +import org.apache.hadoop.ozone.client.ObjectStore; +import org.apache.hadoop.ozone.client.OzoneBucket; +import org.apache.hadoop.ozone.client.OzoneClient; +import org.apache.hadoop.ozone.client.OzoneClientFactory; +import org.apache.hadoop.ozone.client.OzoneVolume; +import org.apache.hadoop.ozone.om.OzoneManager; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.io.OutputStream; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.function.Function; + +public class TestECReconstructionCommands { + + private static MiniOzoneCluster cluster; + private static OzoneManager om; + private static StorageContainerManager scm; + private static OzoneClient client; + private static ObjectStore store; + private static StorageContainerLocationProtocolClientSideTranslatorPB + storageContainerLocationClient; + private static final String SCM_ID = UUID.randomUUID().toString(); + private static final String CLUSTER_ID = UUID.randomUUID().toString(); + private static final int EC_DATA = 10; + private static final int EC_PARITY = 4; + private static final EcCodec EC_CODEC = EcCodec.RS; + private static final int EC_CHUNK_SIZE = 1024; + private static final int NUM_DN = EC_DATA + EC_PARITY; + private static final int[] KEY_SIZES = new int[]{ + EC_CHUNK_SIZE / 2, + EC_CHUNK_SIZE * 3 / 2, + EC_CHUNK_SIZE * EC_DATA + EC_CHUNK_SIZE / 2, + EC_CHUNK_SIZE * EC_DATA + EC_CHUNK_SIZE * 3 / 2, + }; + private static byte[][] values; + private static long containerID; + private static Pipeline pipeline; + private static List datanodeDetails; + private List clients = null; + + @BeforeAll + public static void init() throws Exception { + OzoneConfiguration conf = new OzoneConfiguration(); + conf.setInt(ScmConfigKeys.OZONE_SCM_PIPELINE_OWNER_CONTAINER_COUNT, 1); + conf.setBoolean(OzoneConfigKeys.OZONE_ACL_ENABLED, true); + conf.set(OzoneConfigKeys.OZONE_ACL_AUTHORIZER_CLASS, + OzoneConfigKeys.OZONE_ACL_AUTHORIZER_CLASS_NATIVE); + startCluster(conf); + prepareData(KEY_SIZES); + } + + @AfterAll + public static void stop() throws IOException { + stopCluster(); + } + + private Pipeline createSingleNodePipeline(Pipeline ecPipeline, + DatanodeDetails node, int replicaIndex) { + + Map indicesForSinglePipeline = new HashMap<>(); + indicesForSinglePipeline.put(node, replicaIndex); + + return Pipeline.newBuilder() + .setId(ecPipeline.getId()) + .setReplicationConfig(ecPipeline.getReplicationConfig()) + .setState(ecPipeline.getPipelineState()) + .setNodes(ImmutableList.of(node)) + .setReplicaIndexes(indicesForSinglePipeline) + .build(); + } + + @BeforeEach + public void connectToDatanodes() { + clients = new ArrayList<>(datanodeDetails.size()); + for (int i = 0; i < datanodeDetails.size(); i++) { + clients.add(new XceiverClientGrpc( + createSingleNodePipeline(pipeline, datanodeDetails.get(i), i + 1), + cluster.getConf())); + } + } + + @AfterEach + public void closeClients() { + if (clients == null) { + return; + } + for (XceiverClientSpi client : clients) { + client.close(); + } + clients = null; + } + + private Function chunksInReplicaFunc(int i) { + if (i < EC_DATA) { + return (keySize) -> { + int dataBlocks = (keySize + EC_CHUNK_SIZE - 1) / EC_CHUNK_SIZE; + return (dataBlocks + EC_DATA - 1 - i) / EC_DATA; + }; + } else { + return (keySize) -> (keySize + EC_CHUNK_SIZE * EC_DATA - 1) / + (EC_CHUNK_SIZE * EC_DATA); + } + } + + @Test + public void testListBlock() throws Exception { + for (int i = 0; i < datanodeDetails.size(); i++) { + final int minKeySize = i < EC_DATA ? i * EC_CHUNK_SIZE : 0; + int numExpectedBlocks = (int) Arrays.stream(KEY_SIZES) + .filter(size -> size > minKeySize).count(); + Function expectedChunksFunc = chunksInReplicaFunc(i); + int numExpectedChunks = Arrays.stream(KEY_SIZES) + .map(expectedChunksFunc::apply).sum(); + try { + ListBlockResponseProto response = ContainerProtocolCalls.listBlock( + clients.get(i), containerID, null, numExpectedBlocks + 1, null); + Assertions.assertEquals(numExpectedBlocks, response.getBlockDataCount(), + "blocks count doesn't match on DN " + i); + Assertions.assertEquals(numExpectedChunks, + response.getBlockDataList().stream() + .mapToInt(BlockData::getChunksCount).sum(), + "chunks count doesn't match on DN " + i); + } catch (StorageContainerException e) { + Assertions.assertTrue(e.getMessage().contains("does not exist")); + Assertions.assertEquals(0, numExpectedBlocks); + } + } + } + + public static void startCluster(OzoneConfiguration conf) throws Exception { + + // Set minimum pipeline to 1 to ensure all data is written to + // the same container group + WritableECContainerProviderConfig writableECContainerProviderConfig = + conf.getObject(WritableECContainerProviderConfig.class); + writableECContainerProviderConfig.setMinimumPipelines(1); + conf.setFromObject(writableECContainerProviderConfig); + + cluster = MiniOzoneCluster.newBuilder(conf) + .setNumDatanodes(NUM_DN) + .setScmId(SCM_ID) + .setClusterId(CLUSTER_ID) + .build(); + cluster.waitForClusterToBeReady(); + om = cluster.getOzoneManager(); + scm = cluster.getStorageContainerManager(); + client = OzoneClientFactory.getRpcClient(conf); + store = client.getObjectStore(); + storageContainerLocationClient = + cluster.getStorageContainerLocationClient(); + } + + public static void prepareData(int[] sizes) throws Exception { + final String volumeName = UUID.randomUUID().toString(); + final String bucketName = UUID.randomUUID().toString(); + store.createVolume(volumeName); + OzoneVolume volume = store.getVolume(volumeName); + volume.createBucket(bucketName); + OzoneBucket bucket = volume.getBucket(bucketName); + final ReplicationConfig repConfig = + new ECReplicationConfig(EC_DATA, EC_PARITY, EC_CODEC, EC_CHUNK_SIZE); + values = new byte[KEY_SIZES.length][]; + for (int i = 0; i < sizes.length; i++) { + values[i] = RandomUtils.nextBytes(sizes[i]); + final String keyName = UUID.randomUUID().toString(); + try (OutputStream out = bucket.createKey( + keyName, values[i].length, repConfig, new HashMap<>())) { + out.write(values[i]); + } + } + List containerIDs = + new ArrayList<>(scm.getContainerManager().getContainerIDs()); + Assertions.assertEquals(1, containerIDs.size()); + containerID = containerIDs.get(0).getId(); + List pipelines = scm.getPipelineManager().getPipelines(repConfig); + Assertions.assertEquals(1, pipelines.size()); + pipeline = pipelines.get(0); + datanodeDetails = pipeline.getNodes(); + } + + public static void stopCluster() throws IOException { + if (client != null) { + client.close(); + } + + if (storageContainerLocationClient != null) { + storageContainerLocationClient.close(); + } + + if (cluster != null) { + cluster.shutdown(); + } + } + +} From 8d577f881e40b81c3d94f6d41ddf46a45bab57e1 Mon Sep 17 00:00:00 2001 From: Kaijie Chen Date: Mon, 16 May 2022 18:56:50 +0800 Subject: [PATCH 04/10] Fix checkstyle --- .../ec/reconstruction/TestECReconstructionCommands.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ec/reconstruction/TestECReconstructionCommands.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ec/reconstruction/TestECReconstructionCommands.java index 7f5a07802c5a..b30fa16b2523 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ec/reconstruction/TestECReconstructionCommands.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ec/reconstruction/TestECReconstructionCommands.java @@ -32,7 +32,7 @@ import org.apache.hadoop.hdds.scm.container.ContainerID; import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException; import org.apache.hadoop.hdds.scm.pipeline.Pipeline; -import org.apache.hadoop.hdds.scm.pipeline.WritableECContainerProvider.WritableECContainerProviderConfig ; +import org.apache.hadoop.hdds.scm.pipeline.WritableECContainerProvider.WritableECContainerProviderConfig; import org.apache.hadoop.hdds.scm.protocolPB.StorageContainerLocationProtocolClientSideTranslatorPB; import org.apache.hadoop.hdds.scm.server.StorageContainerManager; import org.apache.hadoop.hdds.scm.storage.ContainerProtocolCalls; @@ -61,6 +61,9 @@ import java.util.UUID; import java.util.function.Function; +/* +Test commands used in EC reconstruction. + */ public class TestECReconstructionCommands { private static MiniOzoneCluster cluster; @@ -135,8 +138,8 @@ public void closeClients() { if (clients == null) { return; } - for (XceiverClientSpi client : clients) { - client.close(); + for (XceiverClientSpi c : clients) { + c.close(); } clients = null; } From 14f3eec9111e52efb97de157617ced4297c02684 Mon Sep 17 00:00:00 2001 From: Kaijie Chen Date: Mon, 16 May 2022 20:36:09 +0800 Subject: [PATCH 05/10] Fix javadoc --- .../ec/reconstruction/TestECReconstructionCommands.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ec/reconstruction/TestECReconstructionCommands.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ec/reconstruction/TestECReconstructionCommands.java index b30fa16b2523..7dd1c4c96ba3 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ec/reconstruction/TestECReconstructionCommands.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ec/reconstruction/TestECReconstructionCommands.java @@ -61,8 +61,8 @@ import java.util.UUID; import java.util.function.Function; -/* -Test commands used in EC reconstruction. +/** + * This class tests commands used in EC reconstruction. */ public class TestECReconstructionCommands { From 87b87a295088f8de5c016ab4b000aeaa459f2dd6 Mon Sep 17 00:00:00 2001 From: Kaijie Chen Date: Tue, 17 May 2022 09:32:29 +0800 Subject: [PATCH 06/10] handle review comments --- ...mmands.java => TestContainerCommands.java} | 42 ++++++++++--------- 1 file changed, 22 insertions(+), 20 deletions(-) rename hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ec/reconstruction/{TestECReconstructionCommands.java => TestContainerCommands.java} (87%) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ec/reconstruction/TestECReconstructionCommands.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ec/reconstruction/TestContainerCommands.java similarity index 87% rename from hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ec/reconstruction/TestECReconstructionCommands.java rename to hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ec/reconstruction/TestContainerCommands.java index 7dd1c4c96ba3..6a4fec3390c2 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ec/reconstruction/TestECReconstructionCommands.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ec/reconstruction/TestContainerCommands.java @@ -15,6 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.hadoop.ozone.container.ec.reconstruction; import com.google.common.collect.ImmutableList; @@ -43,7 +44,6 @@ import org.apache.hadoop.ozone.client.OzoneClient; import org.apache.hadoop.ozone.client.OzoneClientFactory; import org.apache.hadoop.ozone.client.OzoneVolume; -import org.apache.hadoop.ozone.om.OzoneManager; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; @@ -64,10 +64,9 @@ /** * This class tests commands used in EC reconstruction. */ -public class TestECReconstructionCommands { +public class TestContainerCommands { private static MiniOzoneCluster cluster; - private static OzoneManager om; private static StorageContainerManager scm; private static OzoneClient client; private static ObjectStore store; @@ -160,24 +159,28 @@ private Function chunksInReplicaFunc(int i) { public void testListBlock() throws Exception { for (int i = 0; i < datanodeDetails.size(); i++) { final int minKeySize = i < EC_DATA ? i * EC_CHUNK_SIZE : 0; - int numExpectedBlocks = (int) Arrays.stream(KEY_SIZES) - .filter(size -> size > minKeySize).count(); + final int numExpectedBlocks = (int) Arrays.stream(values) + .mapToInt(v -> v.length).filter(s -> s > minKeySize).count(); Function expectedChunksFunc = chunksInReplicaFunc(i); - int numExpectedChunks = Arrays.stream(KEY_SIZES) - .map(expectedChunksFunc::apply).sum(); - try { - ListBlockResponseProto response = ContainerProtocolCalls.listBlock( - clients.get(i), containerID, null, numExpectedBlocks + 1, null); - Assertions.assertEquals(numExpectedBlocks, response.getBlockDataCount(), - "blocks count doesn't match on DN " + i); - Assertions.assertEquals(numExpectedChunks, - response.getBlockDataList().stream() - .mapToInt(BlockData::getChunksCount).sum(), - "chunks count doesn't match on DN " + i); - } catch (StorageContainerException e) { - Assertions.assertTrue(e.getMessage().contains("does not exist")); - Assertions.assertEquals(0, numExpectedBlocks); + final int numExpectedChunks = Arrays.stream(values) + .mapToInt(v -> v.length).map(expectedChunksFunc::apply).sum(); + if (numExpectedBlocks == 0) { + final int j = i; + Throwable t = Assertions.assertThrows(StorageContainerException.class, + () -> ContainerProtocolCalls.listBlock(clients.get(j), + containerID, null, numExpectedBlocks + 1, null)); + Assertions.assertEquals( + "ContainerID " + containerID + " does not exist", t.getMessage()); + continue; } + ListBlockResponseProto response = ContainerProtocolCalls.listBlock( + clients.get(i), containerID, null, numExpectedBlocks + 1, null); + Assertions.assertEquals(numExpectedBlocks, response.getBlockDataCount(), + "blocks count doesn't match on DN " + i); + Assertions.assertEquals(numExpectedChunks, + response.getBlockDataList().stream() + .mapToInt(BlockData::getChunksCount).sum(), + "chunks count doesn't match on DN " + i); } } @@ -196,7 +199,6 @@ public static void startCluster(OzoneConfiguration conf) throws Exception { .setClusterId(CLUSTER_ID) .build(); cluster.waitForClusterToBeReady(); - om = cluster.getOzoneManager(); scm = cluster.getStorageContainerManager(); client = OzoneClientFactory.getRpcClient(conf); store = client.getObjectStore(); From 0f4e83a8024d19c7442669c3829fd254870b51b1 Mon Sep 17 00:00:00 2001 From: Kaijie Chen Date: Tue, 17 May 2022 09:49:15 +0800 Subject: [PATCH 07/10] Randomize key size --- .../reconstruction/TestContainerCommands.java | 29 +++++++++++-------- 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ec/reconstruction/TestContainerCommands.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ec/reconstruction/TestContainerCommands.java index 6a4fec3390c2..dd72eefbc9ab 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ec/reconstruction/TestContainerCommands.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ec/reconstruction/TestContainerCommands.java @@ -78,12 +78,17 @@ public class TestContainerCommands { private static final int EC_PARITY = 4; private static final EcCodec EC_CODEC = EcCodec.RS; private static final int EC_CHUNK_SIZE = 1024; + private static final int STRIPE_DATA_SIZE = EC_DATA * EC_CHUNK_SIZE; private static final int NUM_DN = EC_DATA + EC_PARITY; - private static final int[] KEY_SIZES = new int[]{ - EC_CHUNK_SIZE / 2, - EC_CHUNK_SIZE * 3 / 2, - EC_CHUNK_SIZE * EC_DATA + EC_CHUNK_SIZE / 2, - EC_CHUNK_SIZE * EC_DATA + EC_CHUNK_SIZE * 3 / 2, + + // Each key size will be in range [min, max), min inclusive, max exclusive + private static final int[][] KEY_SIZE_RANGES = new int[][]{ + {1, EC_CHUNK_SIZE}, + {EC_CHUNK_SIZE, EC_CHUNK_SIZE + 1}, + {EC_CHUNK_SIZE + 1, STRIPE_DATA_SIZE}, + {STRIPE_DATA_SIZE, STRIPE_DATA_SIZE + 1}, + {STRIPE_DATA_SIZE + 1, STRIPE_DATA_SIZE + EC_CHUNK_SIZE}, + {STRIPE_DATA_SIZE + EC_CHUNK_SIZE, STRIPE_DATA_SIZE * 2}, }; private static byte[][] values; private static long containerID; @@ -99,7 +104,7 @@ public static void init() throws Exception { conf.set(OzoneConfigKeys.OZONE_ACL_AUTHORIZER_CLASS, OzoneConfigKeys.OZONE_ACL_AUTHORIZER_CLASS_NATIVE); startCluster(conf); - prepareData(KEY_SIZES); + prepareData(KEY_SIZE_RANGES); } @AfterAll @@ -150,8 +155,7 @@ private Function chunksInReplicaFunc(int i) { return (dataBlocks + EC_DATA - 1 - i) / EC_DATA; }; } else { - return (keySize) -> (keySize + EC_CHUNK_SIZE * EC_DATA - 1) / - (EC_CHUNK_SIZE * EC_DATA); + return (keySize) -> (keySize + STRIPE_DATA_SIZE - 1) / STRIPE_DATA_SIZE; } } @@ -206,7 +210,7 @@ public static void startCluster(OzoneConfiguration conf) throws Exception { cluster.getStorageContainerLocationClient(); } - public static void prepareData(int[] sizes) throws Exception { + public static void prepareData(int[][] ranges) throws Exception { final String volumeName = UUID.randomUUID().toString(); final String bucketName = UUID.randomUUID().toString(); store.createVolume(volumeName); @@ -215,9 +219,10 @@ public static void prepareData(int[] sizes) throws Exception { OzoneBucket bucket = volume.getBucket(bucketName); final ReplicationConfig repConfig = new ECReplicationConfig(EC_DATA, EC_PARITY, EC_CODEC, EC_CHUNK_SIZE); - values = new byte[KEY_SIZES.length][]; - for (int i = 0; i < sizes.length; i++) { - values[i] = RandomUtils.nextBytes(sizes[i]); + values = new byte[ranges.length][]; + for (int i = 0; i < ranges.length; i++) { + int keySize = RandomUtils.nextInt(ranges[i][0], ranges[i][1]); + values[i] = RandomUtils.nextBytes(keySize); final String keyName = UUID.randomUUID().toString(); try (OutputStream out = bucket.createKey( keyName, values[i].length, repConfig, new HashMap<>())) { From 2343920c3eaffba658ce488869c5f86ea5c625ea Mon Sep 17 00:00:00 2001 From: Kaijie Chen Date: Tue, 17 May 2022 09:50:22 +0800 Subject: [PATCH 08/10] Switch to EC/rs-3-2-1024 for faster test --- .../container/ec/reconstruction/TestContainerCommands.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ec/reconstruction/TestContainerCommands.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ec/reconstruction/TestContainerCommands.java index dd72eefbc9ab..49ffae28e0d9 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ec/reconstruction/TestContainerCommands.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ec/reconstruction/TestContainerCommands.java @@ -74,8 +74,8 @@ public class TestContainerCommands { storageContainerLocationClient; private static final String SCM_ID = UUID.randomUUID().toString(); private static final String CLUSTER_ID = UUID.randomUUID().toString(); - private static final int EC_DATA = 10; - private static final int EC_PARITY = 4; + private static final int EC_DATA = 3; + private static final int EC_PARITY = 2; private static final EcCodec EC_CODEC = EcCodec.RS; private static final int EC_CHUNK_SIZE = 1024; private static final int STRIPE_DATA_SIZE = EC_DATA * EC_CHUNK_SIZE; From 636cd3706c136ddf035ebcbba6329845ef4197d3 Mon Sep 17 00:00:00 2001 From: Kaijie Chen Date: Wed, 18 May 2022 10:18:02 +0800 Subject: [PATCH 09/10] Move TestContainerCommands to hdds.scm.storage --- .../scm/storage}/TestContainerCommands.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) rename hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/{ozone/container/ec/reconstruction => hdds/scm/storage}/TestContainerCommands.java (98%) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ec/reconstruction/TestContainerCommands.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/storage/TestContainerCommands.java similarity index 98% rename from hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ec/reconstruction/TestContainerCommands.java rename to hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/storage/TestContainerCommands.java index 49ffae28e0d9..3d56a5d5db52 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ec/reconstruction/TestContainerCommands.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/storage/TestContainerCommands.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.hadoop.ozone.container.ec.reconstruction; +package org.apache.hadoop.hdds.scm.storage; import com.google.common.collect.ImmutableList; import org.apache.commons.lang3.RandomUtils; @@ -36,7 +36,6 @@ import org.apache.hadoop.hdds.scm.pipeline.WritableECContainerProvider.WritableECContainerProviderConfig; import org.apache.hadoop.hdds.scm.protocolPB.StorageContainerLocationProtocolClientSideTranslatorPB; import org.apache.hadoop.hdds.scm.server.StorageContainerManager; -import org.apache.hadoop.hdds.scm.storage.ContainerProtocolCalls; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.OzoneConfigKeys; import org.apache.hadoop.ozone.client.ObjectStore; From 9f8e808428faacfbf0ddce448df0fd29439baff9 Mon Sep 17 00:00:00 2001 From: Kaijie Chen Date: Wed, 18 May 2022 10:20:20 +0800 Subject: [PATCH 10/10] rename to TestContainerCommandsEC --- ...estContainerCommands.java => TestContainerCommandsEC.java} | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) rename hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/storage/{TestContainerCommands.java => TestContainerCommandsEC.java} (99%) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/storage/TestContainerCommands.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/storage/TestContainerCommandsEC.java similarity index 99% rename from hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/storage/TestContainerCommands.java rename to hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/storage/TestContainerCommandsEC.java index 3d56a5d5db52..2ab9d241c8ef 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/storage/TestContainerCommands.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/storage/TestContainerCommandsEC.java @@ -61,9 +61,9 @@ import java.util.function.Function; /** - * This class tests commands used in EC reconstruction. + * This class tests container commands on EC containers. */ -public class TestContainerCommands { +public class TestContainerCommandsEC { private static MiniOzoneCluster cluster; private static StorageContainerManager scm;