diff --git a/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/codec/package-info.java b/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/codec/package-info.java deleted file mode 100644 index 463b80c01c80..000000000000 --- a/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/codec/package-info.java +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * Utility classes to encode/decode DTO objects to/from byte array. - */ -package org.apache.hadoop.ozone.om.codec; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java index 8a7b446c013c..2053049573cc 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java @@ -136,99 +136,6 @@ public class OmMetadataManagerImpl implements OMMetadataManager, private static final Logger LOG = LoggerFactory.getLogger(OmMetadataManagerImpl.class); - /** - * OM RocksDB Structure . - *

- * OM DB stores metadata as KV pairs iThis n different column families. - *

- * OM DB Schema: - * - *

-   * {@code
-   * Common Tables:
-   * |----------------------------------------------------------------------|
-   * |  Column Family     |        VALUE                                    |
-   * |----------------------------------------------------------------------|
-   * | userTable          |     /user->UserVolumeInfo                       |
-   * |----------------------------------------------------------------------|
-   * | volumeTable        |     /volume->VolumeInfo                         |
-   * |----------------------------------------------------------------------|
-   * | bucketTable        |     /volume/bucket-> BucketInfo                 |
-   * |----------------------------------------------------------------------|
-   * | s3SecretTable      | s3g_access_key_id -> s3Secret                   |
-   * |----------------------------------------------------------------------|
-   * | dTokenTable        | OzoneTokenID -> renew_time                      |
-   * |----------------------------------------------------------------------|
-   * | prefixInfoTable    | prefix -> PrefixInfo                            |
-   * |----------------------------------------------------------------------|
-   * | multipartInfoTable | /volumeName/bucketName/keyName/uploadId ->...   |
-   * |----------------------------------------------------------------------|
-   * | transactionInfoTable| #TRANSACTIONINFO -> OMTransactionInfo          |
-   * |----------------------------------------------------------------------|
-   * }
-   * 
- *
-   * {@code
-   * Multi-Tenant Tables:
-   * |----------------------------------------------------------------------|
-   * | tenantStateTable          | tenantId -> OmDBTenantState              |
-   * |----------------------------------------------------------------------|
-   * | tenantAccessIdTable       | accessId -> OmDBAccessIdInfo             |
-   * |----------------------------------------------------------------------|
-   * | principalToAccessIdsTable | userPrincipal -> OmDBUserPrincipalInfo   |
-   * |----------------------------------------------------------------------|
-   * }
-   * 
- *
-   * {@code
-   * Simple Tables:
-   * |----------------------------------------------------------------------|
-   * |  Column Family     |        VALUE                                    |
-   * |----------------------------------------------------------------------|
-   * | keyTable           | /volumeName/bucketName/keyName->KeyInfo         |
-   * |----------------------------------------------------------------------|
-   * | deletedTable       | /volumeName/bucketName/keyName->RepeatedKeyInfo |
-   * |----------------------------------------------------------------------|
-   * | openKey            | /volumeName/bucketName/keyName/id->KeyInfo      |
-   * |----------------------------------------------------------------------|
-   * }
-   * 
- *
-   * {@code
-   * Prefix Tables:
-   * |----------------------------------------------------------------------|
-   * |  Column Family   |        VALUE                                      |
-   * |----------------------------------------------------------------------|
-   * |  directoryTable  | /volumeId/bucketId/parentId/dirName -> DirInfo    |
-   * |----------------------------------------------------------------------|
-   * |  fileTable       | /volumeId/bucketId/parentId/fileName -> KeyInfo   |
-   * |----------------------------------------------------------------------|
-   * |  openFileTable   | /volumeId/bucketId/parentId/fileName/id -> KeyInfo|
-   * |----------------------------------------------------------------------|
-   * |  deletedDirTable | /volumeId/bucketId/parentId/dirName/objectId ->   |
-   * |                  |                                      KeyInfo      |
-   * |----------------------------------------------------------------------|
-   * }
-   * 
- *
-   * {@code
-   * Snapshot Tables:
-   * |-------------------------------------------------------------------------|
-   * |  Column Family        |        VALUE                                    |
-   * |-------------------------------------------------------------------------|
-   * | snapshotInfoTable     | /volume/bucket/snapshotName -> SnapshotInfo     |
-   * |-------------------------------------------------------------------------|
-   * | snapshotRenamedTable  | /volumeName/bucketName/objectID -> One of:      |
-   * |                       |  1. /volumeId/bucketId/parentId/dirName         |
-   * |                       |  2. /volumeId/bucketId/parentId/fileName        |
-   * |                       |  3. /volumeName/bucketName/keyName              |
-   * |-------------------------------------------------------------------------|
-   * | compactionLogTable    | dbTrxId-compactionTime -> compactionLogEntry    |
-   * |-------------------------------------------------------------------------|
-   * }
-   * 
- */ - public static final String USER_TABLE = "userTable"; public static final String VOLUME_TABLE = "volumeTable"; public static final String BUCKET_TABLE = "bucketTable"; @@ -566,48 +473,48 @@ protected void initializeOmTables(CacheType cacheType, throws IOException { final TableInitializer initializer = new TableInitializer(addCacheMetrics); - userTable = initializer.get(OMDBDefinition.USER_TABLE); + userTable = initializer.get(OMDBDefinition.USER_TABLE_DEF); - volumeTable = initializer.get(OMDBDefinition.VOLUME_TABLE, cacheType); - bucketTable = initializer.get(OMDBDefinition.BUCKET_TABLE, cacheType); - keyTable = initializer.get(OMDBDefinition.KEY_TABLE); + volumeTable = initializer.get(OMDBDefinition.VOLUME_TABLE_DEF, cacheType); + bucketTable = initializer.get(OMDBDefinition.BUCKET_TABLE_DEF, cacheType); + keyTable = initializer.get(OMDBDefinition.KEY_TABLE_DEF); - openKeyTable = initializer.get(OMDBDefinition.OPEN_KEY_TABLE); - multipartInfoTable = initializer.get(OMDBDefinition.MULTIPART_INFO_TABLE); - deletedTable = initializer.get(OMDBDefinition.DELETED_TABLE); + openKeyTable = initializer.get(OMDBDefinition.OPEN_KEY_TABLE_DEF); + multipartInfoTable = initializer.get(OMDBDefinition.MULTIPART_INFO_TABLE_DEF); + deletedTable = initializer.get(OMDBDefinition.DELETED_TABLE_DEF); - dirTable = initializer.get(OMDBDefinition.DIRECTORY_TABLE); - fileTable = initializer.get(OMDBDefinition.FILE_TABLE); - openFileTable = initializer.get(OMDBDefinition.OPEN_FILE_TABLE); - deletedDirTable = initializer.get(OMDBDefinition.DELETED_DIR_TABLE); + dirTable = initializer.get(OMDBDefinition.DIRECTORY_TABLE_DEF); + fileTable = initializer.get(OMDBDefinition.FILE_TABLE_DEF); + openFileTable = initializer.get(OMDBDefinition.OPEN_FILE_TABLE_DEF); + deletedDirTable = initializer.get(OMDBDefinition.DELETED_DIR_TABLE_DEF); - dTokenTable = initializer.get(OMDBDefinition.DELEGATION_TOKEN_TABLE); - s3SecretTable = initializer.get(OMDBDefinition.S3_SECRET_TABLE); - prefixTable = initializer.get(OMDBDefinition.PREFIX_TABLE); + dTokenTable = initializer.get(OMDBDefinition.DELEGATION_TOKEN_TABLE_DEF); + s3SecretTable = initializer.get(OMDBDefinition.S3_SECRET_TABLE_DEF); + prefixTable = initializer.get(OMDBDefinition.PREFIX_TABLE_DEF); - transactionInfoTable = initializer.get(OMDBDefinition.TRANSACTION_INFO_TABLE); + transactionInfoTable = initializer.get(OMDBDefinition.TRANSACTION_INFO_TABLE_DEF); - metaTable = initializer.get(OMDBDefinition.META_TABLE); + metaTable = initializer.get(OMDBDefinition.META_TABLE_DEF); // accessId -> OmDBAccessIdInfo (tenantId, secret, Kerberos principal) - tenantAccessIdTable = initializer.get(OMDBDefinition.TENANT_ACCESS_ID_TABLE); + tenantAccessIdTable = initializer.get(OMDBDefinition.TENANT_ACCESS_ID_TABLE_DEF); // User principal -> OmDBUserPrincipalInfo (a list of accessIds) - principalToAccessIdsTable = initializer.get(OMDBDefinition.PRINCIPAL_TO_ACCESS_IDS_TABLE); + principalToAccessIdsTable = initializer.get(OMDBDefinition.PRINCIPAL_TO_ACCESS_IDS_TABLE_DEF); // tenant name -> tenant (tenant states) - tenantStateTable = initializer.get(OMDBDefinition.TENANT_STATE_TABLE); + tenantStateTable = initializer.get(OMDBDefinition.TENANT_STATE_TABLE_DEF); // TODO: [SNAPSHOT] Consider FULL_CACHE for snapshotInfoTable since // exclusiveSize in SnapshotInfo can be frequently updated. // path -> snapshotInfo (snapshot info for snapshot) - snapshotInfoTable = initializer.get(OMDBDefinition.SNAPSHOT_INFO_TABLE); + snapshotInfoTable = initializer.get(OMDBDefinition.SNAPSHOT_INFO_TABLE_DEF); // volumeName/bucketName/objectID -> renamedKey or renamedDir - snapshotRenamedTable = initializer.get(OMDBDefinition.SNAPSHOT_RENAMED_TABLE); + snapshotRenamedTable = initializer.get(OMDBDefinition.SNAPSHOT_RENAMED_TABLE_DEF); // TODO: [SNAPSHOT] Initialize table lock for snapshotRenamedTable. - compactionLogTable = initializer.get(OMDBDefinition.COMPACTION_LOG_TABLE); + compactionLogTable = initializer.get(OMDBDefinition.COMPACTION_LOG_TABLE_DEF); } /** diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/codec/OMDBDefinition.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/codec/OMDBDefinition.java index b0a6f79b9489..6d053e1e5e05 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/codec/OMDBDefinition.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/codec/OMDBDefinition.java @@ -26,7 +26,6 @@ import org.apache.hadoop.hdds.utils.db.StringCodec; import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.om.OMConfigKeys; -import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmDBAccessIdInfo; import org.apache.hadoop.ozone.om.helpers.OmDBTenantState; @@ -45,166 +44,255 @@ import org.apache.ozone.compaction.log.CompactionLogEntry; /** - * Class defines the structure and types of the om.db. + * OM database definitions. + *
+ * {@code
+ * User, Token and Secret Tables:
+ * |------------------------------------------------------------------------|
+ * |        Column Family |                 Mapping                         |
+ * |------------------------------------------------------------------------|
+ * |            userTable |             /user :- UserVolumeInfo             |
+ * |          dTokenTable |      OzoneTokenID :- renew_time                 |
+ * |        s3SecretTable | s3g_access_key_id :- s3Secret                   |
+ * |------------------------------------------------------------------------|
+ * }
+ * 
+ * + *
+ * {@code
+ * Volume, Bucket, Prefix and Transaction Tables:
+ * |------------------------------------------------------------------------|
+ * |        Column Family |                 Mapping                         |
+ * |------------------------------------------------------------------------|
+ * |          volumeTable |           /volume :- VolumeInfo                 |
+ * |          bucketTable |    /volume/bucket :- BucketInfo                 |
+ * |------------------------------------------------------------------------|
+ * |          prefixTable |            prefix :- PrefixInfo                 |
+ * |------------------------------------------------------------------------|
+ * | transactionInfoTable |  #TRANSACTIONINFO :- OMTransactionInfo          |
+ * |            metaTable |       metaDataKey :- metaDataValue              |
+ * |------------------------------------------------------------------------|
+ * }
+ * 
+ * + *
+ * {@code
+ * Object Store (OBS) Tables:
+ * |-----------------------------------------------------------------------|
+ * |        Column Family |                           Mapping              |
+ * |-----------------------------------------------------------------------|
+ * |             keyTable | /volume/bucket/key          :- KeyInfo         |
+ * |         deletedTable | /volume/bucket/key          :- RepeatedKeyInfo |
+ * |         openKeyTable | /volume/bucket/key/id       :- KeyInfo         |
+ * |   multipartInfoTable | /volume/bucket/key/uploadId :- parts           |
+ * |-----------------------------------------------------------------------|
+ * }
+ * 
+ * Note that "volume", "bucket" and "key" in OBS tables are names. + * + *
+ * {@code
+ * File System Optimized (FSO) Tables:
+ * |-----------------------------------------------------------------------------------|
+ * |          Column Family |                                            Mapping       |
+ * |-----------------------------------------------------------------------------------|
+ * |              fileTable | /volumeId/bucketId/parentId/fileName         :- KeyInfo  |
+ * |          openFileTable | /volumeId/bucketId/parentId/fileName/id      :- KeyInfo  |
+ * |         directoryTable | /volumeId/bucketId/parentId/dirName          :- DirInfo  |
+ * |  deletedDirectoryTable | /volumeId/bucketId/parentId/dirName/objectId :- KeyInfo  |
+ * |-----------------------------------------------------------------------------------|
+ * }
+ * 
+ * + *
+ * {@code
+ * S3 Multi-Tenant Tables:
+ * |----------------------------------------------------------------------|
+ * |             Column Family |             Mapping                      |
+ * |----------------------------------------------------------------------|
+ * |          tenantStateTable |      tenantId :- OmDBTenantState         |
+ * |       tenantAccessIdTable |      accessId :- OmDBAccessIdInfo        |
+ * | principalToAccessIdsTable | userPrincipal :- OmDBUserPrincipalInfo   |
+ * |----------------------------------------------------------------------|
+ * }
+ * 
+ * + *
+ * {@code
+ * Snapshot Tables:
+ * |----------------------------------------------------------------------------------|
+ * |        Column Family |                                   Mapping                 |
+ * |----------------------------------------------------------------------------------|
+ * |    snapshotInfoTable | /volumeName/bucketName/snapshotName :- SnapshotInfo       |
+ * | snapshotRenamedTable | /volumeName/bucketName/objectID     :- renameFrom         |
+ * |   compactionLogTable | dbTrxId-compactionTime              :- compactionLogEntry |
+ * |----------------------------------------------------------------------------------|
+ * }
+ * 
+ * Note that renameFrom is one of the following: + * 1. /volumeId/bucketId/parentId/dirName + * 2. /volumeId/bucketId/parentId/fileName + * 3. /volumeName/bucketName/keyName */ public final class OMDBDefinition extends DBDefinition.WithMap { - public static final DBColumnFamilyDefinition - DELETED_TABLE = - new DBColumnFamilyDefinition<>( - OmMetadataManagerImpl.DELETED_TABLE, - StringCodec.get(), - RepeatedOmKeyInfo.getCodec(true)); - - public static final DBColumnFamilyDefinition - USER_TABLE = new DBColumnFamilyDefinition<>( - OmMetadataManagerImpl.USER_TABLE, + //--------------------------------------------------------------------------- + // User, Token and Secret Tables: + public static final String USER_TABLE = "userTable"; + /** userTable: /user :- UserVolumeInfo. */ + public static final DBColumnFamilyDefinition USER_TABLE_DEF + = new DBColumnFamilyDefinition<>(USER_TABLE, StringCodec.get(), Proto2Codec.get(PersistedUserVolumeInfo.getDefaultInstance())); - public static final DBColumnFamilyDefinition - VOLUME_TABLE = - new DBColumnFamilyDefinition<>( - OmMetadataManagerImpl.VOLUME_TABLE, - StringCodec.get(), - OmVolumeArgs.getCodec()); - - public static final DBColumnFamilyDefinition - OPEN_KEY_TABLE = - new DBColumnFamilyDefinition<>( - OmMetadataManagerImpl.OPEN_KEY_TABLE, - StringCodec.get(), - OmKeyInfo.getCodec(true)); - - public static final DBColumnFamilyDefinition - KEY_TABLE = - new DBColumnFamilyDefinition<>( - OmMetadataManagerImpl.KEY_TABLE, - StringCodec.get(), - OmKeyInfo.getCodec(true)); - - public static final DBColumnFamilyDefinition - BUCKET_TABLE = - new DBColumnFamilyDefinition<>( - OmMetadataManagerImpl.BUCKET_TABLE, - StringCodec.get(), - OmBucketInfo.getCodec()); - - public static final DBColumnFamilyDefinition - MULTIPART_INFO_TABLE = - new DBColumnFamilyDefinition<>( - OmMetadataManagerImpl.MULTIPARTINFO_TABLE, - StringCodec.get(), - OmMultipartKeyInfo.getCodec()); - - public static final DBColumnFamilyDefinition - PREFIX_TABLE = - new DBColumnFamilyDefinition<>( - OmMetadataManagerImpl.PREFIX_TABLE, - StringCodec.get(), - OmPrefixInfo.getCodec()); - - public static final DBColumnFamilyDefinition - DELEGATION_TOKEN_TABLE = - new DBColumnFamilyDefinition<>( - OmMetadataManagerImpl.DELEGATION_TOKEN_TABLE, - TokenIdentifierCodec.get(), - LongCodec.get()); - - public static final DBColumnFamilyDefinition - S3_SECRET_TABLE = - new DBColumnFamilyDefinition<>( - OmMetadataManagerImpl.S3_SECRET_TABLE, - StringCodec.get(), - S3SecretValue.getCodec()); - - public static final DBColumnFamilyDefinition - TRANSACTION_INFO_TABLE = - new DBColumnFamilyDefinition<>( - OmMetadataManagerImpl.TRANSACTION_INFO_TABLE, - StringCodec.get(), - TransactionInfo.getCodec()); - - public static final DBColumnFamilyDefinition - DIRECTORY_TABLE = - new DBColumnFamilyDefinition<>( - OmMetadataManagerImpl.DIRECTORY_TABLE, - StringCodec.get(), - OmDirectoryInfo.getCodec()); - - public static final DBColumnFamilyDefinition - FILE_TABLE = - new DBColumnFamilyDefinition<>( - OmMetadataManagerImpl.FILE_TABLE, - StringCodec.get(), - OmKeyInfo.getCodec(true)); - - public static final DBColumnFamilyDefinition - OPEN_FILE_TABLE = - new DBColumnFamilyDefinition<>( - OmMetadataManagerImpl.OPEN_FILE_TABLE, - StringCodec.get(), - OmKeyInfo.getCodec(true)); - - public static final DBColumnFamilyDefinition - DELETED_DIR_TABLE = - new DBColumnFamilyDefinition<>(OmMetadataManagerImpl.DELETED_DIR_TABLE, + public static final String DELEGATION_TOKEN_TABLE = "dTokenTable"; + /** dTokenTable: OzoneTokenID :- renew_time. */ + public static final DBColumnFamilyDefinition DELEGATION_TOKEN_TABLE_DEF + = new DBColumnFamilyDefinition<>(DELEGATION_TOKEN_TABLE, + TokenIdentifierCodec.get(), + LongCodec.get()); + + public static final String S3_SECRET_TABLE = "s3SecretTable"; + /** s3SecretTable: s3g_access_key_id :- s3Secret. */ + public static final DBColumnFamilyDefinition S3_SECRET_TABLE_DEF + = new DBColumnFamilyDefinition<>(S3_SECRET_TABLE, StringCodec.get(), - OmKeyInfo.getCodec(true)); + S3SecretValue.getCodec()); + + //--------------------------------------------------------------------------- + // Volume, Bucket, Prefix and Transaction Tables: + public static final String VOLUME_TABLE = "volumeTable"; + /** volumeTable: /volume :- VolumeInfo. */ + public static final DBColumnFamilyDefinition VOLUME_TABLE_DEF + = new DBColumnFamilyDefinition<>(VOLUME_TABLE, + StringCodec.get(), + OmVolumeArgs.getCodec()); + + public static final String BUCKET_TABLE = "bucketTable"; + /** bucketTable: /volume/bucket :- BucketInfo. */ + public static final DBColumnFamilyDefinition BUCKET_TABLE_DEF + = new DBColumnFamilyDefinition<>(BUCKET_TABLE, + StringCodec.get(), + OmBucketInfo.getCodec()); - public static final DBColumnFamilyDefinition - META_TABLE = new DBColumnFamilyDefinition<>( - OmMetadataManagerImpl.META_TABLE, + public static final String PREFIX_TABLE = "prefixTable"; + /** prefixTable: prefix :- PrefixInfo. */ + public static final DBColumnFamilyDefinition PREFIX_TABLE_DEF + = new DBColumnFamilyDefinition<>(PREFIX_TABLE, + StringCodec.get(), + OmPrefixInfo.getCodec()); + + public static final String TRANSACTION_INFO_TABLE = "transactionInfoTable"; + /** transactionInfoTable: #TRANSACTIONINFO :- OMTransactionInfo. */ + public static final DBColumnFamilyDefinition TRANSACTION_INFO_TABLE_DEF + = new DBColumnFamilyDefinition<>(TRANSACTION_INFO_TABLE, + StringCodec.get(), + TransactionInfo.getCodec()); + + public static final String META_TABLE = "metaTable"; + /** metaTable: metaDataKey :- metaDataValue. */ + public static final DBColumnFamilyDefinition META_TABLE_DEF + = new DBColumnFamilyDefinition<>(META_TABLE, StringCodec.get(), StringCodec.get()); - // Tables for multi-tenancy - - public static final DBColumnFamilyDefinition - TENANT_ACCESS_ID_TABLE = - new DBColumnFamilyDefinition<>( - OmMetadataManagerImpl.TENANT_ACCESS_ID_TABLE, - // accessId - StringCodec.get(), - // tenantId, secret, principal - OmDBAccessIdInfo.getCodec()); - - public static final DBColumnFamilyDefinition - PRINCIPAL_TO_ACCESS_IDS_TABLE = - new DBColumnFamilyDefinition<>( - OmMetadataManagerImpl.PRINCIPAL_TO_ACCESS_IDS_TABLE, - // User principal - StringCodec.get(), - // List of accessIds - OmDBUserPrincipalInfo.getCodec()); - - public static final DBColumnFamilyDefinition - TENANT_STATE_TABLE = - new DBColumnFamilyDefinition<>( - OmMetadataManagerImpl.TENANT_STATE_TABLE, - // tenantId (tenant name) - StringCodec.get(), - OmDBTenantState.getCodec()); - - // End tables for S3 multi-tenancy - - public static final DBColumnFamilyDefinition - SNAPSHOT_INFO_TABLE = - new DBColumnFamilyDefinition<>( - OmMetadataManagerImpl.SNAPSHOT_INFO_TABLE, - // snapshot path + //--------------------------------------------------------------------------- + // Object Store (OBS) Tables: + public static final String KEY_TABLE = "keyTable"; + /** keyTable: /volume/bucket/key :- KeyInfo. */ + public static final DBColumnFamilyDefinition KEY_TABLE_DEF + = new DBColumnFamilyDefinition<>(KEY_TABLE, StringCodec.get(), - SnapshotInfo.getCodec()); + OmKeyInfo.getCodec(true)); - public static final DBColumnFamilyDefinition - COMPACTION_LOG_TABLE = - new DBColumnFamilyDefinition<>( - OmMetadataManagerImpl.COMPACTION_LOG_TABLE, + public static final String DELETED_TABLE = "deletedTable"; + /** deletedTable: /volume/bucket/key :- RepeatedKeyInfo. */ + public static final DBColumnFamilyDefinition DELETED_TABLE_DEF + = new DBColumnFamilyDefinition<>(DELETED_TABLE, StringCodec.get(), - CompactionLogEntry.getCodec()); + RepeatedOmKeyInfo.getCodec(true)); + + public static final String OPEN_KEY_TABLE = "openKeyTable"; + /** openKeyTable: /volume/bucket/key/id :- KeyInfo. */ + public static final DBColumnFamilyDefinition OPEN_KEY_TABLE_DEF + = new DBColumnFamilyDefinition<>(OPEN_KEY_TABLE, + StringCodec.get(), + OmKeyInfo.getCodec(true)); + public static final String MULTIPART_INFO_TABLE = "multipartInfoTable"; + /** multipartInfoTable: /volume/bucket/key/uploadId :- parts. */ + public static final DBColumnFamilyDefinition MULTIPART_INFO_TABLE_DEF + = new DBColumnFamilyDefinition<>(MULTIPART_INFO_TABLE, + StringCodec.get(), + OmMultipartKeyInfo.getCodec()); + + //--------------------------------------------------------------------------- + // File System Optimized (FSO) Tables: + public static final String FILE_TABLE = "fileTable"; + /** fileTable: /volumeId/bucketId/parentId/fileName :- KeyInfo. */ + public static final DBColumnFamilyDefinition FILE_TABLE_DEF + = new DBColumnFamilyDefinition<>(FILE_TABLE, + StringCodec.get(), + OmKeyInfo.getCodec(true)); + + public static final String OPEN_FILE_TABLE = "openFileTable"; + /** openFileTable: /volumeId/bucketId/parentId/fileName/id :- KeyInfo. */ + public static final DBColumnFamilyDefinition OPEN_FILE_TABLE_DEF + = new DBColumnFamilyDefinition<>(OPEN_FILE_TABLE, + StringCodec.get(), + OmKeyInfo.getCodec(true)); + + public static final String DIRECTORY_TABLE = "directoryTable"; + /** directoryTable: /volumeId/bucketId/parentId/dirName :- DirInfo. */ + public static final DBColumnFamilyDefinition DIRECTORY_TABLE_DEF + = new DBColumnFamilyDefinition<>(DIRECTORY_TABLE, + StringCodec.get(), + OmDirectoryInfo.getCodec()); + + public static final String DELETED_DIR_TABLE = "deletedDirectoryTable"; + /** deletedDirectoryTable: /volumeId/bucketId/parentId/dirName/objectId :- KeyInfo. */ + public static final DBColumnFamilyDefinition DELETED_DIR_TABLE_DEF + = new DBColumnFamilyDefinition<>(DELETED_DIR_TABLE, + StringCodec.get(), + OmKeyInfo.getCodec(true)); + + //--------------------------------------------------------------------------- + // S3 Multi-Tenancy Tables + public static final String TENANT_STATE_TABLE = "tenantStateTable"; + /** tenantStateTable: tenantId :- OmDBTenantState. */ + public static final DBColumnFamilyDefinition TENANT_STATE_TABLE_DEF + = new DBColumnFamilyDefinition<>(TENANT_STATE_TABLE, + StringCodec.get(), // tenantId (tenant name) + OmDBTenantState.getCodec()); + + public static final String TENANT_ACCESS_ID_TABLE = "tenantAccessIdTable"; + /** tenantAccessIdTable: accessId :- OmDBAccessIdInfo. */ + public static final DBColumnFamilyDefinition TENANT_ACCESS_ID_TABLE_DEF + = new DBColumnFamilyDefinition<>(TENANT_ACCESS_ID_TABLE, + StringCodec.get(), // accessId + OmDBAccessIdInfo.getCodec()); // tenantId, secret, principal + + public static final String PRINCIPAL_TO_ACCESS_IDS_TABLE = "principalToAccessIdsTable"; + /** principalToAccessIdsTable: userPrincipal :- OmDBUserPrincipalInfo. */ + public static final DBColumnFamilyDefinition PRINCIPAL_TO_ACCESS_IDS_TABLE_DEF + = new DBColumnFamilyDefinition<>(PRINCIPAL_TO_ACCESS_IDS_TABLE, + StringCodec.get(), // User principal + OmDBUserPrincipalInfo.getCodec()); // List of accessIds + + //--------------------------------------------------------------------------- + // Snapshot Tables + public static final String SNAPSHOT_INFO_TABLE = "snapshotInfoTable"; + /** snapshotInfoTable: /volume/bucket/snapshotName :- SnapshotInfo. */ + public static final DBColumnFamilyDefinition SNAPSHOT_INFO_TABLE_DEF + = new DBColumnFamilyDefinition<>(SNAPSHOT_INFO_TABLE, + StringCodec.get(), // snapshot path + SnapshotInfo.getCodec()); + + public static final String SNAPSHOT_RENAMED_TABLE = "snapshotRenamedTable"; /** - * SnapshotRenamedTable that complements the keyTable (or fileTable) + * snapshotRenamedTable: /volumeName/bucketName/objectID :- renameFrom. + *

+ * This table complements the keyTable (or fileTable) * and dirTable entries of the immediately previous snapshot in the * same snapshot scope (bucket or volume). *

@@ -213,39 +301,43 @@ public final class OMDBDefinition extends DBDefinition.WithMap { * renamedKey or renamedDir is present in the previous snapshot's keyTable * (or fileTable). */ - public static final DBColumnFamilyDefinition - SNAPSHOT_RENAMED_TABLE = - new DBColumnFamilyDefinition<>( - OmMetadataManagerImpl.SNAPSHOT_RENAMED_TABLE, - // /volumeName/bucketName/objectID + public static final DBColumnFamilyDefinition SNAPSHOT_RENAMED_TABLE_DEF + = new DBColumnFamilyDefinition<>(SNAPSHOT_RENAMED_TABLE, StringCodec.get(), - // path to key in prev snapshot's key(file)/dir Table. - StringCodec.get()); + StringCodec.get()); // path to key in prev snapshot's key(file)/dir Table. + + public static final String COMPACTION_LOG_TABLE = "compactionLogTable"; + /** compactionLogTable: dbTrxId-compactionTime :- compactionLogEntry. */ + public static final DBColumnFamilyDefinition COMPACTION_LOG_TABLE_DEF + = new DBColumnFamilyDefinition<>(COMPACTION_LOG_TABLE, + StringCodec.get(), + CompactionLogEntry.getCodec()); - private static final Map> - COLUMN_FAMILIES = DBColumnFamilyDefinition.newUnmodifiableMap( - BUCKET_TABLE, - DELETED_DIR_TABLE, - DELETED_TABLE, - DIRECTORY_TABLE, - DELEGATION_TOKEN_TABLE, - FILE_TABLE, - KEY_TABLE, - META_TABLE, - MULTIPART_INFO_TABLE, - OPEN_FILE_TABLE, - OPEN_KEY_TABLE, - PREFIX_TABLE, - PRINCIPAL_TO_ACCESS_IDS_TABLE, - S3_SECRET_TABLE, - SNAPSHOT_INFO_TABLE, - SNAPSHOT_RENAMED_TABLE, - COMPACTION_LOG_TABLE, - TENANT_ACCESS_ID_TABLE, - TENANT_STATE_TABLE, - TRANSACTION_INFO_TABLE, - USER_TABLE, - VOLUME_TABLE); + //--------------------------------------------------------------------------- + private static final Map> COLUMN_FAMILIES + = DBColumnFamilyDefinition.newUnmodifiableMap( + BUCKET_TABLE_DEF, + DELETED_DIR_TABLE_DEF, + DELETED_TABLE_DEF, + DIRECTORY_TABLE_DEF, + DELEGATION_TOKEN_TABLE_DEF, + FILE_TABLE_DEF, + KEY_TABLE_DEF, + META_TABLE_DEF, + MULTIPART_INFO_TABLE_DEF, + OPEN_FILE_TABLE_DEF, + OPEN_KEY_TABLE_DEF, + PREFIX_TABLE_DEF, + PRINCIPAL_TO_ACCESS_IDS_TABLE_DEF, + S3_SECRET_TABLE_DEF, + SNAPSHOT_INFO_TABLE_DEF, + SNAPSHOT_RENAMED_TABLE_DEF, + COMPACTION_LOG_TABLE_DEF, + TENANT_ACCESS_ID_TABLE_DEF, + TENANT_STATE_TABLE_DEF, + TRANSACTION_INFO_TABLE_DEF, + USER_TABLE_DEF, + VOLUME_TABLE_DEF); private static final OMDBDefinition INSTANCE = new OMDBDefinition(); diff --git a/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/codec/TokenIdentifierCodec.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/codec/TokenIdentifierCodec.java similarity index 100% rename from hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/codec/TokenIdentifierCodec.java rename to hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/codec/TokenIdentifierCodec.java diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerDoubleBuffer.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerDoubleBuffer.java index bf7dd2df7b0f..04515dcd728a 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerDoubleBuffer.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerDoubleBuffer.java @@ -18,7 +18,6 @@ package org.apache.hadoop.ozone.om.ratis; import static org.apache.hadoop.ozone.OzoneConsts.TRANSACTION_INFO_KEY; -import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.S3_SECRET_TABLE; import com.google.common.annotations.VisibleForTesting; import java.io.IOException; @@ -473,7 +472,7 @@ private void cleanupCache(Map> cleanupEpochs) { Collections.sort(epochs); omMetadataManager.getTable(tableName).cleanupCache(epochs); // Check if the table is S3SecretTable, if yes, then clear the cache. - if (tableName.equals(S3_SECRET_TABLE.getName())) { + if (tableName.equals(OMDBDefinition.S3_SECRET_TABLE_DEF.getName())) { s3SecretManager.clearS3Cache(epochs); } }); diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/containergenerator/GeneratorOm.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/containergenerator/GeneratorOm.java index 39f2b3b3e79f..fe3752e4edfc 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/containergenerator/GeneratorOm.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/containergenerator/GeneratorOm.java @@ -100,7 +100,7 @@ public Void call() throws Exception { // initialization: create one bucket and volume in OM. writeOmBucketVolume(); - omKeyTable = OMDBDefinition.KEY_TABLE.getTable(omDb); + omKeyTable = OMDBDefinition.KEY_TABLE_DEF.getTable(omDb); timer = getMetrics().timer("om-generator"); runTests(this::writeOmKeys); @@ -129,7 +129,7 @@ public void writeOmKeys(long index) throws Exception { private void writeOmBucketVolume() throws IOException { - final Table volTable = OMDBDefinition.VOLUME_TABLE.getTable(omDb); + final Table volTable = OMDBDefinition.VOLUME_TABLE_DEF.getTable(omDb); String admin = getUserId(); String owner = getUserId(); @@ -151,7 +151,7 @@ private void writeOmBucketVolume() throws IOException { volTable.put("/" + volumeName, omVolumeArgs); - final Table userTable = OMDBDefinition.USER_TABLE.getTable(omDb); + final Table userTable = OMDBDefinition.USER_TABLE_DEF.getTable(omDb); PersistedUserVolumeInfo currentUserVolumeInfo = userTable.get(getUserId()); @@ -172,7 +172,7 @@ private void writeOmBucketVolume() throws IOException { userTable.put(getUserId(), currentUserVolumeInfo); - final Table bucketTable = OMDBDefinition.BUCKET_TABLE.getTable(omDb); + final Table bucketTable = OMDBDefinition.BUCKET_TABLE_DEF.getTable(omDb); OmBucketInfo omBucketInfo = new OmBucketInfo.Builder() .setBucketName(bucketName) diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/repair/TransactionInfoRepair.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/repair/TransactionInfoRepair.java index 0bc42748e25e..2c547959c1db 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/repair/TransactionInfoRepair.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/repair/TransactionInfoRepair.java @@ -120,7 +120,7 @@ protected Component serviceToBeOffline() { public static DBColumnFamilyDefinition getColumnFamily(Component component) { switch (component) { case OM: - return OMDBDefinition.TRANSACTION_INFO_TABLE; + return OMDBDefinition.TRANSACTION_INFO_TABLE_DEF; case SCM: return SCMDBDefinition.TRANSACTIONINFO; default: diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/repair/om/FSORepairTool.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/repair/om/FSORepairTool.java index f85ec7099a92..bf819bff6254 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/repair/om/FSORepairTool.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/repair/om/FSORepairTool.java @@ -141,13 +141,13 @@ private class Impl { this.unreferencedStats = new ReportStatistics(0, 0, 0); this.store = getStoreFromPath(omDBPath); - this.volumeTable = OMDBDefinition.VOLUME_TABLE.getTable(store); - this.bucketTable = OMDBDefinition.BUCKET_TABLE.getTable(store); - this.directoryTable = OMDBDefinition.DIRECTORY_TABLE.getTable(store); - this.fileTable = OMDBDefinition.FILE_TABLE.getTable(store); - this.deletedDirectoryTable = OMDBDefinition.DELETED_DIR_TABLE.getTable(store); - this.deletedTable = OMDBDefinition.DELETED_TABLE.getTable(store); - this.snapshotInfoTable = OMDBDefinition.SNAPSHOT_INFO_TABLE.getTable(store); + this.volumeTable = OMDBDefinition.VOLUME_TABLE_DEF.getTable(store); + this.bucketTable = OMDBDefinition.BUCKET_TABLE_DEF.getTable(store); + this.directoryTable = OMDBDefinition.DIRECTORY_TABLE_DEF.getTable(store); + this.fileTable = OMDBDefinition.FILE_TABLE_DEF.getTable(store); + this.deletedDirectoryTable = OMDBDefinition.DELETED_DIR_TABLE_DEF.getTable(store); + this.deletedTable = OMDBDefinition.DELETED_TABLE_DEF.getTable(store); + this.snapshotInfoTable = OMDBDefinition.SNAPSHOT_INFO_TABLE_DEF.getTable(store); } public Report run() throws Exception { diff --git a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/repair/TestTransactionInfoRepair.java b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/repair/TestTransactionInfoRepair.java index ed91c2fe079f..da197e2a70eb 100644 --- a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/repair/TestTransactionInfoRepair.java +++ b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/repair/TestTransactionInfoRepair.java @@ -142,7 +142,7 @@ private void testCommand(String component, ManagedRocksDB mdb, ColumnFamilyHandl private String getColumnFamilyName(String component) { switch (component) { - case "om": return OMDBDefinition.TRANSACTION_INFO_TABLE.getName(); + case "om": return OMDBDefinition.TRANSACTION_INFO_TABLE_DEF.getName(); case "scm": return SCMDBDefinition.TRANSACTIONINFO.getName(); default: return ""; }