diff --git a/hadoop-hdds/docs/content/feature/SCM-HA.md b/hadoop-hdds/docs/content/feature/SCM-HA.md index 4201b13de28..3f509e4ca6b 100644 --- a/hadoop-hdds/docs/content/feature/SCM-HA.md +++ b/hadoop-hdds/docs/content/feature/SCM-HA.md @@ -211,13 +211,13 @@ You can also create data and double check with `ozone debug` tool if all the con ```shell bin/ozone freon randomkeys --numOfVolumes=1 --numOfBuckets=1 --numOfKeys=10000 --keySize=524288 --replicationType=RATIS --numOfThreads=8 --factor=THREE --bufferSize=1048576 - - -// use debug ldb to check scm db on all the machines -bin/ozone debug ldb --db=/tmp/metadata/scm.db/ ls - - -bin/ozone debug ldb --db=/tmp/metadata/scm.db/ scan --with-keys --column_family=containers + + +# use debug ldb to check scm.db on all the machines +bin/ozone debug ldb --db=/tmp/metadata/scm.db ls + + +bin/ozone debug ldb --db=/tmp/metadata/scm.db scan --column-family=containers ``` ## Migrating from existing SCM @@ -226,4 +226,4 @@ SCM HA can be turned on on any Ozone cluster. First enable Ratis (`ozone.scm.rat Start the cluster and test if it works well. -If everything is fine, you can extend the cluster configuration with multiple nodes, restart SCM node, and initialize the additional nodes with `scm --bootstrap` command. \ No newline at end of file +If everything is fine, you can extend the cluster configuration with multiple nodes, restart SCM node, and initialize the additional nodes with `scm --bootstrap` command. diff --git a/hadoop-hdds/docs/content/feature/SCM-HA.zh.md b/hadoop-hdds/docs/content/feature/SCM-HA.zh.md index 9875337e644..a5382735b7a 100644 --- a/hadoop-hdds/docs/content/feature/SCM-HA.zh.md +++ b/hadoop-hdds/docs/content/feature/SCM-HA.zh.md @@ -207,13 +207,13 @@ layoutVersion=0 ```shell bin/ozone freon randomkeys --numOfVolumes=1 --numOfBuckets=1 --numOfKeys=10000 --keySize=524288 --replicationType=RATIS --numOfThreads=8 --factor=THREE --bufferSize=1048576 - - -// use debug ldb to check scm db on all the machines -bin/ozone debug ldb --db=/tmp/metadata/scm.db/ ls - - -bin/ozone debug ldb --db=/tmp/metadata/scm.db/ scan --with-keys --column_family=containers + + +# 使用 debug ldb 工具逐一检查各机上的 scm.db +bin/ozone debug ldb --db=/tmp/metadata/scm.db ls + + +bin/ozone debug ldb --db=/tmp/metadata/scm.db scan --column-family=containers ``` ## 从现有的SCM迁移 diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestLDBCli.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestLDBCli.java index 04f1139a7e9..338ce144c7d 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestLDBCli.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestLDBCli.java @@ -16,8 +16,10 @@ */ package org.apache.hadoop.ozone.om; - -import org.apache.commons.io.FileUtils; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.gson.Gson; +import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.hdds.StringUtils; import org.apache.hadoop.hdds.client.BlockID; import org.apache.hadoop.hdds.conf.OzoneConfiguration; @@ -36,295 +38,296 @@ import org.apache.hadoop.ozone.debug.RDBParser; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.request.OMRequestTestUtils; -import org.apache.ozone.test.GenericTestUtils; -import org.junit.After; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.Assert; -import org.junit.rules.TemporaryFolder; +import org.jetbrains.annotations.NotNull; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Named; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import picocli.CommandLine; -import java.io.BufferedReader; -import java.io.ByteArrayOutputStream; import java.io.File; -import java.io.FileInputStream; -import java.io.InputStreamReader; -import java.io.PrintStream; -import java.time.LocalDateTime; -import java.util.List; +import java.io.IOException; +import java.io.PrintWriter; +import java.io.StringWriter; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.NavigableMap; +import java.util.TreeMap; +import java.util.stream.Stream; import static java.nio.charset.StandardCharsets.UTF_8; - /** - * This class tests the Debug LDB CLI that reads from rocks db file. + * This class tests `ozone debug ldb` CLI that reads from a RocksDB directory. */ public class TestLDBCli { + private static final String KEY_TABLE = "keyTable"; + private static final String BLOCK_DATA = "block_data"; + private static final ObjectMapper MAPPER = new ObjectMapper(); private OzoneConfiguration conf; + private DBStore dbStore; + @TempDir + private File tempDir; + private StringWriter stdout, stderr; + private PrintWriter pstdout, pstderr; + private CommandLine cmd; + private NavigableMap> dbMap; + private String keySeparatorSchemaV3 = + new OzoneConfiguration().getObject(DatanodeConfiguration.class) + .getContainerSchemaV3KeySeparator(); + + @BeforeEach + public void setup() throws IOException { + conf = new OzoneConfiguration(); + stdout = new StringWriter(); + pstdout = new PrintWriter(stdout); + stderr = new StringWriter(); + pstderr = new PrintWriter(stderr); - private RDBParser rdbParser; - private DBScanner dbScanner; - private DBStore dbStore = null; - private List keyNames; - private static final String DEFAULT_ENCODING = UTF_8.name(); - - @Rule - public TemporaryFolder folder = new TemporaryFolder(); + cmd = new CommandLine(new RDBParser()) + .addSubcommand(new DBScanner()) + .setOut(pstdout) + .setErr(pstderr); - @Before - public void setup() throws Exception { - conf = new OzoneConfiguration(); - rdbParser = new RDBParser(); - dbScanner = new DBScanner(); - keyNames = new ArrayList<>(); + dbMap = new TreeMap<>(); } - @After - public void shutdown() throws Exception { + @AfterEach + public void shutdown() throws IOException { if (dbStore != null) { dbStore.close(); } - System.setOut(System.out); - // Restore the static fields in DBScanner - DBScanner.setContainerId(-1); - DBScanner.setDnDBSchemaVersion("V2"); - DBScanner.setWithKey(false); + pstderr.close(); + stderr.close(); + pstdout.close(); + stdout.close(); } - @Test - public void testOMDB() throws Exception { - File newFolder = folder.newFolder(); - if (!newFolder.exists()) { - Assert.assertTrue(newFolder.mkdirs()); - } - // Dummy om.db with only keyTable - dbStore = DBStoreBuilder.newBuilder(conf) - .setName("om.db") - .setPath(newFolder.toPath()) - .addTable("keyTable") - .build(); - // insert 5 keys - for (int i = 0; i < 5; i++) { - OmKeyInfo value = OMRequestTestUtils.createOmKeyInfo("sampleVol", - "sampleBuck", "key" + (i + 1), HddsProtos.ReplicationType.STAND_ALONE, - HddsProtos.ReplicationFactor.ONE); - String key = "key" + (i + 1); - Table keyTable = dbStore.getTable("keyTable"); - byte[] arr = value - .getProtobuf(ClientVersion.CURRENT_VERSION).toByteArray(); - keyTable.put(key.getBytes(UTF_8), arr); - } - rdbParser.setDbPath(dbStore.getDbLocation().getAbsolutePath()); - dbScanner.setParent(rdbParser); - DBScanner.setLimit(100); - Assert.assertEquals(5, getKeyNames(dbScanner).size()); - Assert.assertTrue(getKeyNames(dbScanner).contains("key1")); - Assert.assertTrue(getKeyNames(dbScanner).contains("key5")); - Assert.assertFalse(getKeyNames(dbScanner).contains("key6")); - - final ByteArrayOutputStream outputStreamCaptor = - new ByteArrayOutputStream(); - System.setOut(new PrintStream(outputStreamCaptor, false, DEFAULT_ENCODING)); - DBScanner.setShowCount(true); - dbScanner.call(); - Assert.assertEquals("5", - outputStreamCaptor.toString(DEFAULT_ENCODING).trim()); - System.setOut(System.out); - DBScanner.setShowCount(false); - - - DBScanner.setLimit(1); - Assert.assertEquals(1, getKeyNames(dbScanner).size()); - - // Testing the startKey function - DBScanner.setLimit(-1); - DBScanner.setStartKey("key3"); - Assert.assertEquals(3, getKeyNames(dbScanner).size()); + /** + * Defines ldb tool test cases. + */ + private static Stream scanTestCases() { + return Stream.of( + Arguments.of( + Named.of(KEY_TABLE, Pair.of(KEY_TABLE, false)), + Named.of("Default", Pair.of(0, "")), + Named.of("No extra args", Collections.emptyList()), + Named.of("Expect key1-key5", Pair.of("key1", "key6")) + ), + Arguments.of( + Named.of(KEY_TABLE, Pair.of(KEY_TABLE, false)), + Named.of("Length", Pair.of(0, "")), + Named.of("Limit 1", Arrays.asList("--length", "1")), + Named.of("Expect key1 only", Pair.of("key1", "key2")) + ), + Arguments.of( + Named.of(KEY_TABLE, Pair.of(KEY_TABLE, false)), + Named.of("InvalidLength", Pair.of(1, "IllegalArgumentException")), + Named.of("Limit 0", Arrays.asList("--length", "0")), + Named.of("Expect empty result", null) + ), + Arguments.of( + Named.of(KEY_TABLE, Pair.of(KEY_TABLE, false)), + Named.of("UnlimitedLength", Pair.of(0, "")), + Named.of("Limit -1", Arrays.asList("--length", "-1")), + Named.of("Expect key1-key5", Pair.of("key1", "key6")) + ), + Arguments.of( + Named.of(BLOCK_DATA + " V3", Pair.of(BLOCK_DATA, true)), + Named.of("Default", Pair.of(0, "")), + Named.of("V3", Arrays.asList("--dn-schema", "V3")), + Named.of("Expect '1|1'-'2|4'", Pair.of("1|1", "3|5")) + ), + Arguments.of( + Named.of(BLOCK_DATA + " V3", Pair.of(BLOCK_DATA, true)), + Named.of("Specify", Pair.of(0, "")), + Named.of("V3", Collections.emptyList()), + Named.of("Expect '1|1'-'2|4'", Pair.of("1|1", "3|5")) + ), + Arguments.of( + Named.of(BLOCK_DATA + " V3", Pair.of(BLOCK_DATA, true)), + Named.of("ContainerID 1", Pair.of(0, "")), + Named.of("V3 + cid 1", Arrays.asList( + "--dn-schema", "V3", + "--container-id", "1", + "--length", "2")), + Named.of("Expect '1|1' and '1|2'", Pair.of("1|1", "2|3")) + ), + Arguments.of( + Named.of(BLOCK_DATA + " V3", Pair.of(BLOCK_DATA, true)), + Named.of("ContainerID 2", Pair.of(0, "")), + Named.of("V3 + cid 2", Arrays.asList( + "--dn-schema", "V3", + "--container-id", "2", + "--length", "2")), + Named.of("Expect '2|3' and '2|4'", Pair.of("2|3", "3|5")) + ), + Arguments.of( + Named.of(BLOCK_DATA + " V3", Pair.of(BLOCK_DATA, true)), + Named.of("ContainerID 2 + Length", Pair.of(0, "")), + Named.of("V3 + cid 2 + limit 1", Arrays.asList( + "--dn-schema", "V3", + "--container-id", "2", + "--length", "1")), + Named.of("Expect '2|3' only", Pair.of("2|3", "2|4")) + ), + Arguments.of( + Named.of(BLOCK_DATA + " V2", Pair.of(BLOCK_DATA, false)), + Named.of("Erroneously parse V2 table as V3", + Pair.of(1, "Error: Invalid")), + Named.of("Default to V3", Collections.emptyList()), + Named.of("Expect exception", null) + ), + Arguments.of( + Named.of(BLOCK_DATA + " V2", Pair.of(BLOCK_DATA, false)), + Named.of("Explicit V2", Pair.of(0, "")), + Named.of("V2", Arrays.asList("--dn-schema", "V2")), + Named.of("Expect 1-4", Pair.of("1", "5")) + ) + ); + } - DBScanner.setLimit(0); - try { - getKeyNames(dbScanner); - Assert.fail("IllegalArgumentException is expected"); - } catch (IllegalArgumentException e) { - //ignore + @ParameterizedTest + @MethodSource("scanTestCases") + void testLDBScan( + @NotNull Pair tableAndOption, + @NotNull Pair expectedExitCodeStderrPair, + List scanArgs, + Pair dbMapRange) throws IOException { + + final String tableName = tableAndOption.getLeft(); + final Boolean schemaV3 = tableAndOption.getRight(); + // Prepare dummy table. Populate dbMap that contains expected results + prepareTable(tableName, schemaV3); + + // Prepare scan args + List completeScanArgs = new ArrayList<>(); + completeScanArgs.addAll(Arrays.asList( + "--db", dbStore.getDbLocation().getAbsolutePath(), + "scan", + "--column-family", tableName)); + completeScanArgs.addAll(scanArgs); + + int exitCode = cmd.execute(completeScanArgs.toArray(new String[0])); + // Check exit code. Print stderr if not expected + int expectedExitCode = expectedExitCodeStderrPair.getLeft(); + Assertions.assertEquals(expectedExitCode, exitCode, stderr.toString()); + + // Construct expected result map given test param input + Map> expectedMap; + if (dbMapRange != null) { + expectedMap = dbMap.subMap(dbMapRange.getLeft(), dbMapRange.getRight()); + } else { + expectedMap = new TreeMap<>(); } - // If set with -1, check if it dumps entire table data. - DBScanner.setLimit(-1); - DBScanner.setStartKey(null); - Assert.assertEquals(5, getKeyNames(dbScanner).size()); - - // Test dump to file. - File tempFile = folder.newFolder(); - String outFile = tempFile.getAbsolutePath() + "keyTable" - + LocalDateTime.now(); - BufferedReader bufferedReader = null; - try { - DBScanner.setLimit(-1); - DBScanner.setFileName(outFile); - keyNames = getKeyNames(dbScanner); - Assert.assertEquals(5, keyNames.size()); - Assert.assertTrue(new File(outFile).exists()); + if (exitCode == 0) { + // Verify stdout on success + assertContents(expectedMap, stdout.toString()); + } - bufferedReader = new BufferedReader( - new InputStreamReader(new FileInputStream(outFile), UTF_8)); + // Check stderr + final String stderrShouldContain = expectedExitCodeStderrPair.getRight(); + Assertions.assertTrue(stderr.toString().contains(stderrShouldContain)); + } - String readLine; - int count = 0; + /** + * Converts String input to a Map and compares to the given Map input. + * @param expected expected result Map + * @param actualStr String input + */ + private void assertContents(Map expected, String actualStr) + throws IOException { + // Parse actual output (String) into Map + Map> actualMap = MAPPER.readValue( + actualStr, new TypeReference>>() { }); + + Assertions.assertEquals(expected, actualMap); + } - while ((readLine = bufferedReader.readLine()) != null) { - for (String keyName : keyNames) { - if (readLine.contains(keyName)) { - count++; - break; + /** + * Prepare the table for testing. + * Also populate dbMap that contains all possibly expected results. + * @param tableName table name + * @param schemaV3 set to true for SchemaV3. applicable to block_data table + */ + private void prepareTable(String tableName, boolean schemaV3) + throws IOException { + + switch (tableName) { + case KEY_TABLE: + // Dummy om.db with only keyTable + dbStore = DBStoreBuilder.newBuilder(conf).setName("om.db") + .setPath(tempDir.toPath()).addTable(KEY_TABLE).build(); + + Table keyTable = dbStore.getTable(KEY_TABLE); + // Insert 5 keys + for (int i = 1; i <= 5; i++) { + String key = "key" + i; + OmKeyInfo value = OMRequestTestUtils.createOmKeyInfo("vol1", "buck1", + key, HddsProtos.ReplicationType.STAND_ALONE, + HddsProtos.ReplicationFactor.ONE); + keyTable.put(key.getBytes(UTF_8), + value.getProtobuf(ClientVersion.CURRENT_VERSION).toByteArray()); + + // Populate map + dbMap.put(key, toMap(value)); + } + break; + + case BLOCK_DATA: + conf.setBoolean(DatanodeConfiguration.CONTAINER_SCHEMA_V3_ENABLED, + schemaV3); + dbStore = BlockUtils.getUncachedDatanodeStore( + tempDir.getAbsolutePath() + "/" + OzoneConsts.CONTAINER_DB_NAME, + schemaV3 ? OzoneConsts.SCHEMA_V3 : OzoneConsts.SCHEMA_V2, + conf, false).getStore(); + + Table blockTable = dbStore.getTable(BLOCK_DATA); + // Insert 2 containers with 2 blocks each + final int containerCount = 2; + final int blockCount = 2; + int blockId = 1; + for (int cid = 1; cid <= containerCount; cid++) { + for (int blockIdx = 1; blockIdx <= blockCount; blockIdx++, blockId++) { + byte[] dbKey; + String mapKey; + BlockData blockData = new BlockData(new BlockID(cid, blockId)); + if (schemaV3) { + String dbKeyStr = DatanodeSchemaThreeDBDefinition + .getContainerKeyPrefix(cid) + blockId; + dbKey = FixedLengthStringUtils.string2Bytes(dbKeyStr); + // Schema V3 ldb scan output key is "containerId: blockId" + mapKey = cid + keySeparatorSchemaV3 + blockId; + } else { + String dbKeyStr = String.valueOf(blockId); + dbKey = StringUtils.string2Bytes(dbKeyStr); + // Schema V2 ldb scan output key is "blockId" + mapKey = dbKeyStr; } + blockTable.put(dbKey, blockData.getProtoBufMessage().toByteArray()); + dbMap.put(mapKey, toMap(blockData)); } } + break; - // As keyName will be in the file twice for each key. - // Once in keyName and second time in fileName. - - // Sample key data. - // { - // .. - // .. - // "keyName": "key5", - // "fileName": "key5", - // .. - // .. - // } - - Assert.assertEquals("File does not have all keys", - keyNames.size() * 2, count); - } finally { - if (bufferedReader != null) { - bufferedReader.close(); - } - if (new File(outFile).exists()) { - FileUtils.deleteQuietly(new File(outFile)); - } - } - } - - private List getKeyNames(DBScanner scanner) - throws Exception { - keyNames.clear(); - scanner.setTableName("keyTable"); - scanner.call(); - Assert.assertFalse(scanner.getScannedObjects().isEmpty()); - for (Object o : scanner.getScannedObjects()) { - OmKeyInfo keyInfo = (OmKeyInfo)o; - keyNames.add(keyInfo.getKeyName()); + default: + throw new IllegalArgumentException("Unsupported table: " + tableName); } - return keyNames; } - @Test - public void testDNDBSchemaV3() throws Exception { - File newFolder = folder.newFolder(); - if (!newFolder.exists()) { - Assert.assertTrue(newFolder.mkdirs()); - } - - conf.setBoolean(DatanodeConfiguration.CONTAINER_SCHEMA_V3_ENABLED, true); - dbStore = BlockUtils.getUncachedDatanodeStore(newFolder.getAbsolutePath() + - "/" + OzoneConsts.CONTAINER_DB_NAME, OzoneConsts.SCHEMA_V3, conf, - false).getStore(); - - // insert 2 containers, each with 2 blocks - final int containerCount = 2; - final int blockCount = 2; - int blockId = 1; - Table blockTable = dbStore.getTable("block_data"); - for (int i = 1; i <= containerCount; i++) { - for (int j = 1; j <= blockCount; j++, blockId++) { - String key = - DatanodeSchemaThreeDBDefinition.getContainerKeyPrefix(i) + blockId; - BlockData blockData = new BlockData(new BlockID(i, blockId)); - blockTable.put(FixedLengthStringUtils.string2Bytes(key), - blockData.getProtoBufMessage().toByteArray()); - } - } - - rdbParser.setDbPath(dbStore.getDbLocation().getAbsolutePath()); - dbScanner.setParent(rdbParser); - dbScanner.setTableName("block_data"); - DBScanner.setDnDBSchemaVersion("V3"); - DBScanner.setWithKey(true); - - // Scan all container - DBScanner.setLimit(-1); - try (GenericTestUtils.SystemOutCapturer capture = - new GenericTestUtils.SystemOutCapturer()) { - dbScanner.call(); - // Assert that output has info for container 2 block 4 - Assert.assertTrue(capture.getOutput().contains("2: 4")); - // Assert that output has info for container 1 block 1 - Assert.assertTrue(capture.getOutput().contains("1: 1")); - } - - // Scan container 1 - DBScanner.setContainerId(1); - DBScanner.setLimit(2); - try (GenericTestUtils.SystemOutCapturer capture = - new GenericTestUtils.SystemOutCapturer()) { - dbScanner.call(); - // Assert that output doesn't have info for container 2 block 4 - Assert.assertFalse(capture.getOutput().contains("2: 4")); - // Assert that output has info for container 1 block 1 - Assert.assertTrue(capture.getOutput().contains("1: 1")); - } - - // Scan container 2 - DBScanner.setContainerId(2); - DBScanner.setLimit(2); - try (GenericTestUtils.SystemOutCapturer capture = - new GenericTestUtils.SystemOutCapturer()) { - dbScanner.call(); - // Assert that output has info for container 2 block 4 - Assert.assertTrue(capture.getOutput().contains("2: 4")); - // Assert that output doesn't have info for container 1 block 1 - Assert.assertFalse(capture.getOutput().contains("1: 1")); - } + private static Map toMap(Object obj) throws IOException { + // Have to use the same serializer (Gson) as DBScanner does. + // JsonUtils (ObjectMapper) parses object differently. + String json = new Gson().toJson(obj); + return MAPPER.readValue(json, new TypeReference>() { }); } - @Test - public void testDNDBSchemaV2() throws Exception { - File newFolder = folder.newFolder(); - if (!newFolder.exists()) { - Assert.assertTrue(newFolder.mkdirs()); - } - - conf.setBoolean(DatanodeConfiguration.CONTAINER_SCHEMA_V3_ENABLED, false); - dbStore = BlockUtils.getUncachedDatanodeStore(newFolder.getAbsolutePath() + - "/" + OzoneConsts.CONTAINER_DB_NAME, OzoneConsts.SCHEMA_V2, conf, - false).getStore(); - - // insert 1 containers with 2 blocks - final long cid = 1; - final int blockCount = 2; - int blockId = 1; - Table blockTable = dbStore.getTable("block_data"); - for (int j = 1; j <= blockCount; j++, blockId++) { - String key = String.valueOf(blockId); - BlockData blockData = new BlockData(new BlockID(cid, blockId)); - blockTable.put(StringUtils.string2Bytes(key), - blockData.getProtoBufMessage().toByteArray()); - } - - rdbParser.setDbPath(dbStore.getDbLocation().getAbsolutePath()); - dbScanner.setParent(rdbParser); - dbScanner.setTableName("block_data"); - DBScanner.setDnDBSchemaVersion("V2"); - DBScanner.setWithKey(true); - - // Scan all container - try (GenericTestUtils.SystemOutCapturer capture = - new GenericTestUtils.SystemOutCapturer()) { - dbScanner.call(); - // Assert that output has info for block 2 - Assert.assertTrue(capture.getOutput().contains("2")); - } - } } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/OMRequestTestUtils.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/OMRequestTestUtils.java index 2dbe1881805..6f976a12629 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/OMRequestTestUtils.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/OMRequestTestUtils.java @@ -447,6 +447,7 @@ public static OmKeyInfo createOmKeyInfo(String volumeName, String bucketName, .setVolumeName(volumeName) .setBucketName(bucketName) .setKeyName(keyName) + .setFileName(OzoneFSUtils.getFileName(keyName)) .setOmKeyLocationInfos(Collections.singletonList( new OmKeyLocationInfoGroup(version, new ArrayList<>()))) .setCreationTime(creationTime) diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/DBScanner.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/DBScanner.java index ec8f1a8a540..fb0fd57f75c 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/DBScanner.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/DBScanner.java @@ -34,6 +34,7 @@ import org.apache.hadoop.hdds.utils.db.managed.ManagedRocksIterator; import org.apache.hadoop.hdds.utils.db.managed.ManagedSlice; import org.apache.hadoop.ozone.OzoneConsts; +import org.apache.hadoop.ozone.container.common.statemachine.DatanodeConfiguration; import org.apache.hadoop.ozone.container.metadata.DatanodeSchemaThreeDBDefinition; import org.kohsuke.MetaInfServices; import org.rocksdb.ColumnFamilyDescriptor; @@ -43,86 +44,122 @@ import org.slf4j.LoggerFactory; import picocli.CommandLine; -import java.io.FileOutputStream; import java.io.IOException; -import java.io.OutputStreamWriter; import java.io.PrintWriter; -import java.io.Writer; -import java.nio.charset.StandardCharsets; import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.UUID; import java.util.concurrent.Callable; +import static java.nio.charset.StandardCharsets.UTF_8; + /** * Parser for scm.db, om.db or container db file. */ @CommandLine.Command( - name = "scan", - description = "Parse specified metadataTable" + name = "scan", + description = "Parse specified metadataTable" ) @MetaInfServices(SubcommandWithParent.class) public class DBScanner implements Callable, SubcommandWithParent { - public static final Logger LOG = - LoggerFactory.getLogger(DBScanner.class); + public static final Logger LOG = LoggerFactory.getLogger(DBScanner.class); + private static final String SCHEMA_V3 = "V3"; + + @CommandLine.Spec + private CommandLine.Model.CommandSpec spec; + + @CommandLine.ParentCommand + private RDBParser parent; - @CommandLine.Option(names = {"--column_family", "--column-family"}, + @CommandLine.Option(names = {"--column_family", "--column-family", "--cf"}, required = true, description = "Table name") private String tableName; @CommandLine.Option(names = {"--with-keys"}, - description = "List Key -> Value instead of just Value.", - defaultValue = "false", - showDefaultValue = CommandLine.Help.Visibility.ALWAYS) - private static boolean withKey; + description = "Print a JSON object of key->value pairs (default)" + + " instead of a JSON array of only values.", + defaultValue = "true") + private boolean withKey; - @CommandLine.Option(names = {"--length", "-l"}, - description = "Maximum number of items to list.") - private static int limit = -1; + @CommandLine.Option(names = {"--length", "--limit", "-l"}, + description = "Maximum number of items to list.", + defaultValue = "-1") + private long limit; @CommandLine.Option(names = {"--out", "-o"}, description = "File to dump table scan data") - private static String fileName; + private String fileName; - @CommandLine.Option(names = {"--startkey", "-sk"}, + @CommandLine.Option(names = {"--startkey", "--sk", "-s"}, description = "Key from which to iterate the DB") - private static String startKey; + private String startKey; - @CommandLine.Option(names = {"--dnSchema", "-d", "--dn-schema"}, - description = "Datanode DB Schema Version : V1/V2/V3", + @CommandLine.Option(names = {"--dnSchema", "--dn-schema", "-d"}, + description = "Datanode DB Schema Version: V1/V2/V3", defaultValue = "V3") - private static String dnDBSchemaVersion; + private String dnDBSchemaVersion; - @CommandLine.Option(names = {"--container-id", "-cid"}, - description = "Container ID when datanode DB Schema is V3", + @CommandLine.Option(names = {"--container-id", "--cid"}, + description = "Container ID. Applicable if datanode DB Schema is V3", defaultValue = "-1") - private static long containerId; + private long containerId; - @CommandLine.Option(names = { "--show-count", - "-count" }, description = "Get estimated key count for a" - + " given column family in the db", + @CommandLine.Option(names = { "--show-count", "--count" }, + description = "Get estimated key count for the given DB column family", defaultValue = "false", showDefaultValue = CommandLine.Help.Visibility.ALWAYS) - private static boolean showCount; + private boolean showCount; + private String keySeparatorSchemaV3 = + new OzoneConfiguration().getObject(DatanodeConfiguration.class) + .getContainerSchemaV3KeySeparator(); - @CommandLine.ParentCommand - private RDBParser parent; + @Override + public Void call() throws Exception { - private HashMap columnFamilyMap; + List cfDescList = + RocksDBUtils.getColumnFamilyDescriptors(parent.getDbPath()); + final List cfHandleList = new ArrayList<>(); + + final boolean schemaV3 = dnDBSchemaVersion != null && + dnDBSchemaVersion.equalsIgnoreCase(SCHEMA_V3) && + parent.getDbPath().contains(OzoneConsts.CONTAINER_DB_NAME); + + boolean success; + try (ManagedRocksDB db = ManagedRocksDB.openReadOnly( + parent.getDbPath(), cfDescList, cfHandleList)) { + success = printTable(cfHandleList, db, parent.getDbPath(), schemaV3); + } + + if (!success) { + // Trick to set exit code to 1 on error. + // TODO: Properly set exit code hopefully by refactoring GenericCli + throw new Exception( + "Exit code is non-zero. Check the error message above"); + } - private List scannedObjects; + return null; + } + + private PrintWriter err() { + return spec.commandLine().getErr(); + } + + private PrintWriter out() { + return spec.commandLine().getOut(); + } - public static byte[] getValueObject( - DBColumnFamilyDefinition dbColumnFamilyDefinition) throws IOException { + public byte[] getValueObject( + DBColumnFamilyDefinition dbColumnFamilyDefinition) { Class keyType = dbColumnFamilyDefinition.getKeyType(); if (keyType.equals(String.class)) { - return startKey.getBytes(StandardCharsets.UTF_8); + return startKey.getBytes(UTF_8); } else if (keyType.equals(ContainerID.class)) { return new ContainerID(Long.parseLong(startKey)).getBytes(); } else if (keyType.equals(Long.class)) { @@ -136,210 +173,192 @@ public static byte[] getValueObject( } } - private static List displayTable(ManagedRocksIterator iterator, - DBColumnFamilyDefinition dbColumnFamilyDefinition, boolean schemaV3) - throws IOException { - List outputs = new ArrayList<>(); + private boolean displayTable(ManagedRocksIterator iterator, + DBColumnFamilyDefinition dbColumnFamilyDef, + boolean schemaV3) + throws IOException { - if (startKey != null) { - iterator.get().seek(getValueObject(dbColumnFamilyDefinition)); + if (fileName == null) { + // Print to stdout + return displayTable(iterator, dbColumnFamilyDef, out(), schemaV3); } - Writer fileWriter = null; - PrintWriter printWriter = null; - try { - if (fileName != null) { - fileWriter = new OutputStreamWriter( - new FileOutputStream(fileName), StandardCharsets.UTF_8); - printWriter = new PrintWriter(fileWriter); - } - - while (iterator.get().isValid()) { - StringBuilder result = new StringBuilder(); - if (withKey) { - Object key = dbColumnFamilyDefinition.getKeyCodec() - .fromPersistedFormat(iterator.get().key()); - Gson gson = new GsonBuilder().setPrettyPrinting().create(); - if (schemaV3) { - int index = - DatanodeSchemaThreeDBDefinition.getContainerKeyPrefixLength(); - String cid = key.toString().substring(0, index); - String blockId = key.toString().substring(index); - result.append(gson.toJson(Longs.fromByteArray( - FixedLengthStringUtils.string2Bytes(cid)) + ": " + blockId)); - } else { - result.append(gson.toJson(key)); - } - result.append(" -> "); - } - Object o = dbColumnFamilyDefinition.getValueCodec() - .fromPersistedFormat(iterator.get().value()); - outputs.add(o); - Gson gson = new GsonBuilder().setPrettyPrinting().create(); - result.append(gson.toJson(o)); - if (fileName != null) { - printWriter.println(result); - } else { - System.out.println(result.toString()); - } - limit--; - iterator.get().next(); - if (limit == 0) { - break; - } - } - } finally { - if (printWriter != null) { - printWriter.close(); - } - if (fileWriter != null) { - fileWriter.close(); - } + // Write to file output + try (PrintWriter out = new PrintWriter(fileName, UTF_8.name())) { + return displayTable(iterator, dbColumnFamilyDef, out, schemaV3); } - return outputs; } - public void setTableName(String tableName) { - this.tableName = tableName; - } + private boolean displayTable(ManagedRocksIterator iterator, + DBColumnFamilyDefinition dbColumnFamilyDef, + PrintWriter out, + boolean schemaV3) + throws IOException { - public RDBParser getParent() { - return parent; - } + if (startKey != null) { + iterator.get().seek(getValueObject(dbColumnFamilyDef)); + } - public void setParent(RDBParser parent) { - this.parent = parent; - } + if (withKey) { + // Start JSON object (map) + out.print("{ "); + } else { + // Start JSON array + out.print("[ "); + } - public static void setLimit(int limit) { - DBScanner.limit = limit; - } + // Count number of keys printed so far + long count = 0; + while (withinLimit(count) && iterator.get().isValid()) { + StringBuilder sb = new StringBuilder(); + if (withKey) { + Object key = dbColumnFamilyDef.getKeyCodec() + .fromPersistedFormat(iterator.get().key()); + Gson gson = new GsonBuilder().setPrettyPrinting().create(); + if (schemaV3) { + int index = + DatanodeSchemaThreeDBDefinition.getContainerKeyPrefixLength(); + String keyStr = key.toString(); + if (index > keyStr.length()) { + err().println("Error: Invalid SchemaV3 table key length. " + + "Is this a V2 table? Try again with --dn-schema=V2"); + return false; + } + String cid = keyStr.substring(0, index); + String blockId = keyStr.substring(index); + sb.append(gson.toJson(Longs.fromByteArray( + FixedLengthStringUtils.string2Bytes(cid)) + + keySeparatorSchemaV3 + + blockId)); + } else { + sb.append(gson.toJson(key)); + } + sb.append(": "); + } - public List getScannedObjects() { - return scannedObjects; - } + Gson gson = new GsonBuilder().setPrettyPrinting().create(); + Object o = dbColumnFamilyDef.getValueCodec() + .fromPersistedFormat(iterator.get().value()); + sb.append(gson.toJson(o)); - public static void setFileName(String name) { - DBScanner.fileName = name; - } + iterator.get().next(); + ++count; + if (withinLimit(count) && iterator.get().isValid()) { + // If this is not the last entry, append comma + sb.append(", "); + } - public static void setContainerId(long id) { - DBScanner.containerId = id; - } + out.print(sb); + } - public static void setDnDBSchemaVersion(String version) { - DBScanner.dnDBSchemaVersion = version; - } + if (withKey) { + // End JSON object + out.println(" }"); + } else { + // End JSON array + out.println(" ]"); + } - public static void setWithKey(boolean withKey) { - DBScanner.withKey = withKey; + return true; } - public static void setShowCount(boolean showCount) { - DBScanner.showCount = showCount; + private boolean withinLimit(long i) { + return limit == -1L || i < limit; } - private static ColumnFamilyHandle getColumnFamilyHandle( - byte[] name, List columnFamilyHandles) { + private ColumnFamilyHandle getColumnFamilyHandle( + byte[] name, List columnFamilyHandles) { return columnFamilyHandles - .stream() - .filter( - handle -> { - try { - return Arrays.equals(handle.getName(), name); - } catch (Exception ex) { - throw new RuntimeException(ex); - } - }) - .findAny() - .orElse(null); - } - - private void constructColumnFamilyMap(DBDefinition dbDefinition) { - if (dbDefinition == null) { - System.out.println("Incorrect Db Path"); - return; - } - this.columnFamilyMap = new HashMap<>(); - DBColumnFamilyDefinition[] columnFamilyDefinitions = dbDefinition - .getColumnFamilies(); - for (DBColumnFamilyDefinition definition:columnFamilyDefinitions) { - LOG.info("Added definition for table: {}", definition.getTableName()); - this.columnFamilyMap.put(definition.getTableName(), definition); - } - } - - @Override - public Void call() throws Exception { - List cfs = - RocksDBUtils.getColumnFamilyDescriptors(parent.getDbPath()); - - final List columnFamilyHandleList = - new ArrayList<>(); - ManagedRocksDB rocksDB = ManagedRocksDB.openReadOnly(parent.getDbPath(), - cfs, columnFamilyHandleList); - final boolean schemaV3 = DBScanner.dnDBSchemaVersion != null && - DBScanner.dnDBSchemaVersion.equals("V3") && - parent.getDbPath().contains(OzoneConsts.CONTAINER_DB_NAME); - this.printAppropriateTable(columnFamilyHandleList, - rocksDB, parent.getDbPath(), schemaV3); - return null; + .stream() + .filter( + handle -> { + try { + return Arrays.equals(handle.getName(), name); + } catch (Exception ex) { + throw new RuntimeException(ex); + } + }) + .findAny() + .orElse(null); } - private void printAppropriateTable( - List columnFamilyHandleList, - ManagedRocksDB rocksDB, String dbPath, boolean schemaV3) + /** + * Main table printing logic. + * User-provided args are not in the arg list. Those are instance variables + * parsed by picocli. + */ + private boolean printTable(List columnFamilyHandleList, + ManagedRocksDB rocksDB, + String dbPath, + boolean schemaV3) throws IOException, RocksDBException { + if (limit < 1 && limit != -1) { throw new IllegalArgumentException( - "List length should be a positive number. Only allowed negative" + - " number is -1 which is to dump entire table"); + "List length should be a positive number. Only allowed negative" + + " number is -1 which is to dump entire table"); } dbPath = removeTrailingSlashIfNeeded(dbPath); DBDefinitionFactory.setDnDBSchemaVersion(dnDBSchemaVersion); - this.constructColumnFamilyMap(DBDefinitionFactory. - getDefinition(Paths.get(dbPath), new OzoneConfiguration())); - if (this.columnFamilyMap != null) { - if (!this.columnFamilyMap.containsKey(tableName)) { - System.out.print("Table with name:" + tableName + " does not exist"); + DBDefinition dbDefinition = DBDefinitionFactory.getDefinition( + Paths.get(dbPath), new OzoneConfiguration()); + if (dbDefinition == null) { + err().println("Error: Incorrect DB Path"); + return false; + } + + Map columnFamilyMap = new HashMap<>(); + for (DBColumnFamilyDefinition cfDef : dbDefinition.getColumnFamilies()) { + LOG.info("Found table: {}", cfDef.getTableName()); + columnFamilyMap.put(cfDef.getTableName(), cfDef); + } + if (!columnFamilyMap.containsKey(tableName)) { + err().print("Error: Table with name '" + tableName + "' not found"); + return false; + } + + DBColumnFamilyDefinition columnFamilyDefinition = + columnFamilyMap.get(tableName); + ColumnFamilyHandle columnFamilyHandle = getColumnFamilyHandle( + columnFamilyDefinition.getTableName().getBytes(UTF_8), + columnFamilyHandleList); + if (columnFamilyHandle == null) { + throw new IllegalStateException("columnFamilyHandle is null"); + } + + if (showCount) { + // Only prints estimates key count + long keyCount = rocksDB.get() + .getLongProperty(columnFamilyHandle, RocksDatabase.ESTIMATE_NUM_KEYS); + out().println(keyCount); + return true; + } + + ManagedRocksIterator iterator = null; + try { + if (containerId > 0L && schemaV3) { + // Handle SchemaV3 DN DB + ManagedReadOptions readOptions = new ManagedReadOptions(); + readOptions.setIterateUpperBound(new ManagedSlice( + FixedLengthStringUtils.string2Bytes( + DatanodeSchemaThreeDBDefinition.getContainerKeyPrefix( + containerId + 1L)))); + iterator = new ManagedRocksIterator( + rocksDB.get().newIterator(columnFamilyHandle, readOptions)); + iterator.get().seek(FixedLengthStringUtils.string2Bytes( + DatanodeSchemaThreeDBDefinition.getContainerKeyPrefix( + containerId))); } else { - DBColumnFamilyDefinition columnFamilyDefinition = - this.columnFamilyMap.get(tableName); - ColumnFamilyHandle columnFamilyHandle = getColumnFamilyHandle( - columnFamilyDefinition.getTableName() - .getBytes(StandardCharsets.UTF_8), - columnFamilyHandleList); - if (columnFamilyHandle == null) { - throw new IllegalArgumentException("columnFamilyHandle is null"); - } - if (showCount) { - long keyCount = rocksDB.get().getLongProperty(columnFamilyHandle, - RocksDatabase.ESTIMATE_NUM_KEYS); - System.out.println(keyCount); - return; - } - ManagedRocksIterator iterator; - if (containerId > 0 && schemaV3) { - ManagedReadOptions readOptions = new ManagedReadOptions(); - readOptions.setIterateUpperBound(new ManagedSlice( - FixedLengthStringUtils.string2Bytes( - DatanodeSchemaThreeDBDefinition.getContainerKeyPrefix( - containerId + 1)))); - iterator = new ManagedRocksIterator( - rocksDB.get().newIterator(columnFamilyHandle, readOptions)); - iterator.get().seek(FixedLengthStringUtils.string2Bytes( - DatanodeSchemaThreeDBDefinition.getContainerKeyPrefix( - containerId))); - } else { - iterator = new ManagedRocksIterator( - rocksDB.get().newIterator(columnFamilyHandle)); - iterator.get().seekToFirst(); - } - scannedObjects = displayTable(iterator, - columnFamilyDefinition, schemaV3); + iterator = new ManagedRocksIterator( + rocksDB.get().newIterator(columnFamilyHandle)); + iterator.get().seekToFirst(); + } + + return displayTable(iterator, columnFamilyDefinition, schemaV3); + } finally { + if (iterator != null) { + iterator.close(); } - } else { - System.out.println("Incorrect db Path"); } } @@ -354,9 +373,4 @@ private String removeTrailingSlashIfNeeded(String dbPath) { public Class getParentType() { return RDBParser.class; } - - public static void setStartKey(String startKey) { - DBScanner.startKey = startKey; - } } -