diff --git a/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/codec/package-info.java b/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/codec/package-info.java deleted file mode 100644 index 463b80c01c80..000000000000 --- a/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/codec/package-info.java +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * Utility classes to encode/decode DTO objects to/from byte array. - */ -package org.apache.hadoop.ozone.om.codec; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java index 8a7b446c013c..2053049573cc 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java @@ -136,99 +136,6 @@ public class OmMetadataManagerImpl implements OMMetadataManager, private static final Logger LOG = LoggerFactory.getLogger(OmMetadataManagerImpl.class); - /** - * OM RocksDB Structure . - *
- * OM DB stores metadata as KV pairs iThis n different column families. - *
- * OM DB Schema: - * - *
- * {@code
- * Common Tables:
- * |----------------------------------------------------------------------|
- * | Column Family | VALUE |
- * |----------------------------------------------------------------------|
- * | userTable | /user->UserVolumeInfo |
- * |----------------------------------------------------------------------|
- * | volumeTable | /volume->VolumeInfo |
- * |----------------------------------------------------------------------|
- * | bucketTable | /volume/bucket-> BucketInfo |
- * |----------------------------------------------------------------------|
- * | s3SecretTable | s3g_access_key_id -> s3Secret |
- * |----------------------------------------------------------------------|
- * | dTokenTable | OzoneTokenID -> renew_time |
- * |----------------------------------------------------------------------|
- * | prefixInfoTable | prefix -> PrefixInfo |
- * |----------------------------------------------------------------------|
- * | multipartInfoTable | /volumeName/bucketName/keyName/uploadId ->... |
- * |----------------------------------------------------------------------|
- * | transactionInfoTable| #TRANSACTIONINFO -> OMTransactionInfo |
- * |----------------------------------------------------------------------|
- * }
- *
- *
- * {@code
- * Multi-Tenant Tables:
- * |----------------------------------------------------------------------|
- * | tenantStateTable | tenantId -> OmDBTenantState |
- * |----------------------------------------------------------------------|
- * | tenantAccessIdTable | accessId -> OmDBAccessIdInfo |
- * |----------------------------------------------------------------------|
- * | principalToAccessIdsTable | userPrincipal -> OmDBUserPrincipalInfo |
- * |----------------------------------------------------------------------|
- * }
- *
- *
- * {@code
- * Simple Tables:
- * |----------------------------------------------------------------------|
- * | Column Family | VALUE |
- * |----------------------------------------------------------------------|
- * | keyTable | /volumeName/bucketName/keyName->KeyInfo |
- * |----------------------------------------------------------------------|
- * | deletedTable | /volumeName/bucketName/keyName->RepeatedKeyInfo |
- * |----------------------------------------------------------------------|
- * | openKey | /volumeName/bucketName/keyName/id->KeyInfo |
- * |----------------------------------------------------------------------|
- * }
- *
- *
- * {@code
- * Prefix Tables:
- * |----------------------------------------------------------------------|
- * | Column Family | VALUE |
- * |----------------------------------------------------------------------|
- * | directoryTable | /volumeId/bucketId/parentId/dirName -> DirInfo |
- * |----------------------------------------------------------------------|
- * | fileTable | /volumeId/bucketId/parentId/fileName -> KeyInfo |
- * |----------------------------------------------------------------------|
- * | openFileTable | /volumeId/bucketId/parentId/fileName/id -> KeyInfo|
- * |----------------------------------------------------------------------|
- * | deletedDirTable | /volumeId/bucketId/parentId/dirName/objectId -> |
- * | | KeyInfo |
- * |----------------------------------------------------------------------|
- * }
- *
- *
- * {@code
- * Snapshot Tables:
- * |-------------------------------------------------------------------------|
- * | Column Family | VALUE |
- * |-------------------------------------------------------------------------|
- * | snapshotInfoTable | /volume/bucket/snapshotName -> SnapshotInfo |
- * |-------------------------------------------------------------------------|
- * | snapshotRenamedTable | /volumeName/bucketName/objectID -> One of: |
- * | | 1. /volumeId/bucketId/parentId/dirName |
- * | | 2. /volumeId/bucketId/parentId/fileName |
- * | | 3. /volumeName/bucketName/keyName |
- * |-------------------------------------------------------------------------|
- * | compactionLogTable | dbTrxId-compactionTime -> compactionLogEntry |
- * |-------------------------------------------------------------------------|
- * }
- *
- */
-
public static final String USER_TABLE = "userTable";
public static final String VOLUME_TABLE = "volumeTable";
public static final String BUCKET_TABLE = "bucketTable";
@@ -566,48 +473,48 @@ protected void initializeOmTables(CacheType cacheType,
throws IOException {
final TableInitializer initializer = new TableInitializer(addCacheMetrics);
- userTable = initializer.get(OMDBDefinition.USER_TABLE);
+ userTable = initializer.get(OMDBDefinition.USER_TABLE_DEF);
- volumeTable = initializer.get(OMDBDefinition.VOLUME_TABLE, cacheType);
- bucketTable = initializer.get(OMDBDefinition.BUCKET_TABLE, cacheType);
- keyTable = initializer.get(OMDBDefinition.KEY_TABLE);
+ volumeTable = initializer.get(OMDBDefinition.VOLUME_TABLE_DEF, cacheType);
+ bucketTable = initializer.get(OMDBDefinition.BUCKET_TABLE_DEF, cacheType);
+ keyTable = initializer.get(OMDBDefinition.KEY_TABLE_DEF);
- openKeyTable = initializer.get(OMDBDefinition.OPEN_KEY_TABLE);
- multipartInfoTable = initializer.get(OMDBDefinition.MULTIPART_INFO_TABLE);
- deletedTable = initializer.get(OMDBDefinition.DELETED_TABLE);
+ openKeyTable = initializer.get(OMDBDefinition.OPEN_KEY_TABLE_DEF);
+ multipartInfoTable = initializer.get(OMDBDefinition.MULTIPART_INFO_TABLE_DEF);
+ deletedTable = initializer.get(OMDBDefinition.DELETED_TABLE_DEF);
- dirTable = initializer.get(OMDBDefinition.DIRECTORY_TABLE);
- fileTable = initializer.get(OMDBDefinition.FILE_TABLE);
- openFileTable = initializer.get(OMDBDefinition.OPEN_FILE_TABLE);
- deletedDirTable = initializer.get(OMDBDefinition.DELETED_DIR_TABLE);
+ dirTable = initializer.get(OMDBDefinition.DIRECTORY_TABLE_DEF);
+ fileTable = initializer.get(OMDBDefinition.FILE_TABLE_DEF);
+ openFileTable = initializer.get(OMDBDefinition.OPEN_FILE_TABLE_DEF);
+ deletedDirTable = initializer.get(OMDBDefinition.DELETED_DIR_TABLE_DEF);
- dTokenTable = initializer.get(OMDBDefinition.DELEGATION_TOKEN_TABLE);
- s3SecretTable = initializer.get(OMDBDefinition.S3_SECRET_TABLE);
- prefixTable = initializer.get(OMDBDefinition.PREFIX_TABLE);
+ dTokenTable = initializer.get(OMDBDefinition.DELEGATION_TOKEN_TABLE_DEF);
+ s3SecretTable = initializer.get(OMDBDefinition.S3_SECRET_TABLE_DEF);
+ prefixTable = initializer.get(OMDBDefinition.PREFIX_TABLE_DEF);
- transactionInfoTable = initializer.get(OMDBDefinition.TRANSACTION_INFO_TABLE);
+ transactionInfoTable = initializer.get(OMDBDefinition.TRANSACTION_INFO_TABLE_DEF);
- metaTable = initializer.get(OMDBDefinition.META_TABLE);
+ metaTable = initializer.get(OMDBDefinition.META_TABLE_DEF);
// accessId -> OmDBAccessIdInfo (tenantId, secret, Kerberos principal)
- tenantAccessIdTable = initializer.get(OMDBDefinition.TENANT_ACCESS_ID_TABLE);
+ tenantAccessIdTable = initializer.get(OMDBDefinition.TENANT_ACCESS_ID_TABLE_DEF);
// User principal -> OmDBUserPrincipalInfo (a list of accessIds)
- principalToAccessIdsTable = initializer.get(OMDBDefinition.PRINCIPAL_TO_ACCESS_IDS_TABLE);
+ principalToAccessIdsTable = initializer.get(OMDBDefinition.PRINCIPAL_TO_ACCESS_IDS_TABLE_DEF);
// tenant name -> tenant (tenant states)
- tenantStateTable = initializer.get(OMDBDefinition.TENANT_STATE_TABLE);
+ tenantStateTable = initializer.get(OMDBDefinition.TENANT_STATE_TABLE_DEF);
// TODO: [SNAPSHOT] Consider FULL_CACHE for snapshotInfoTable since
// exclusiveSize in SnapshotInfo can be frequently updated.
// path -> snapshotInfo (snapshot info for snapshot)
- snapshotInfoTable = initializer.get(OMDBDefinition.SNAPSHOT_INFO_TABLE);
+ snapshotInfoTable = initializer.get(OMDBDefinition.SNAPSHOT_INFO_TABLE_DEF);
// volumeName/bucketName/objectID -> renamedKey or renamedDir
- snapshotRenamedTable = initializer.get(OMDBDefinition.SNAPSHOT_RENAMED_TABLE);
+ snapshotRenamedTable = initializer.get(OMDBDefinition.SNAPSHOT_RENAMED_TABLE_DEF);
// TODO: [SNAPSHOT] Initialize table lock for snapshotRenamedTable.
- compactionLogTable = initializer.get(OMDBDefinition.COMPACTION_LOG_TABLE);
+ compactionLogTable = initializer.get(OMDBDefinition.COMPACTION_LOG_TABLE_DEF);
}
/**
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/codec/OMDBDefinition.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/codec/OMDBDefinition.java
index b0a6f79b9489..6d053e1e5e05 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/codec/OMDBDefinition.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/codec/OMDBDefinition.java
@@ -26,7 +26,6 @@
import org.apache.hadoop.hdds.utils.db.StringCodec;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.om.OMConfigKeys;
-import org.apache.hadoop.ozone.om.OmMetadataManagerImpl;
import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
import org.apache.hadoop.ozone.om.helpers.OmDBAccessIdInfo;
import org.apache.hadoop.ozone.om.helpers.OmDBTenantState;
@@ -45,166 +44,255 @@
import org.apache.ozone.compaction.log.CompactionLogEntry;
/**
- * Class defines the structure and types of the om.db.
+ * OM database definitions.
+ *
+ * {@code
+ * User, Token and Secret Tables:
+ * |------------------------------------------------------------------------|
+ * | Column Family | Mapping |
+ * |------------------------------------------------------------------------|
+ * | userTable | /user :- UserVolumeInfo |
+ * | dTokenTable | OzoneTokenID :- renew_time |
+ * | s3SecretTable | s3g_access_key_id :- s3Secret |
+ * |------------------------------------------------------------------------|
+ * }
+ *
+ *
+ *
+ * {@code
+ * Volume, Bucket, Prefix and Transaction Tables:
+ * |------------------------------------------------------------------------|
+ * | Column Family | Mapping |
+ * |------------------------------------------------------------------------|
+ * | volumeTable | /volume :- VolumeInfo |
+ * | bucketTable | /volume/bucket :- BucketInfo |
+ * |------------------------------------------------------------------------|
+ * | prefixTable | prefix :- PrefixInfo |
+ * |------------------------------------------------------------------------|
+ * | transactionInfoTable | #TRANSACTIONINFO :- OMTransactionInfo |
+ * | metaTable | metaDataKey :- metaDataValue |
+ * |------------------------------------------------------------------------|
+ * }
+ *
+ *
+ *
+ * {@code
+ * Object Store (OBS) Tables:
+ * |-----------------------------------------------------------------------|
+ * | Column Family | Mapping |
+ * |-----------------------------------------------------------------------|
+ * | keyTable | /volume/bucket/key :- KeyInfo |
+ * | deletedTable | /volume/bucket/key :- RepeatedKeyInfo |
+ * | openKeyTable | /volume/bucket/key/id :- KeyInfo |
+ * | multipartInfoTable | /volume/bucket/key/uploadId :- parts |
+ * |-----------------------------------------------------------------------|
+ * }
+ *
+ * Note that "volume", "bucket" and "key" in OBS tables are names.
+ *
+ *
+ * {@code
+ * File System Optimized (FSO) Tables:
+ * |-----------------------------------------------------------------------------------|
+ * | Column Family | Mapping |
+ * |-----------------------------------------------------------------------------------|
+ * | fileTable | /volumeId/bucketId/parentId/fileName :- KeyInfo |
+ * | openFileTable | /volumeId/bucketId/parentId/fileName/id :- KeyInfo |
+ * | directoryTable | /volumeId/bucketId/parentId/dirName :- DirInfo |
+ * | deletedDirectoryTable | /volumeId/bucketId/parentId/dirName/objectId :- KeyInfo |
+ * |-----------------------------------------------------------------------------------|
+ * }
+ *
+ *
+ *
+ * {@code
+ * S3 Multi-Tenant Tables:
+ * |----------------------------------------------------------------------|
+ * | Column Family | Mapping |
+ * |----------------------------------------------------------------------|
+ * | tenantStateTable | tenantId :- OmDBTenantState |
+ * | tenantAccessIdTable | accessId :- OmDBAccessIdInfo |
+ * | principalToAccessIdsTable | userPrincipal :- OmDBUserPrincipalInfo |
+ * |----------------------------------------------------------------------|
+ * }
+ *
+ *
+ *
+ * {@code
+ * Snapshot Tables:
+ * |----------------------------------------------------------------------------------|
+ * | Column Family | Mapping |
+ * |----------------------------------------------------------------------------------|
+ * | snapshotInfoTable | /volumeName/bucketName/snapshotName :- SnapshotInfo |
+ * | snapshotRenamedTable | /volumeName/bucketName/objectID :- renameFrom |
+ * | compactionLogTable | dbTrxId-compactionTime :- compactionLogEntry |
+ * |----------------------------------------------------------------------------------|
+ * }
+ *
+ * Note that renameFrom is one of the following:
+ * 1. /volumeId/bucketId/parentId/dirName
+ * 2. /volumeId/bucketId/parentId/fileName
+ * 3. /volumeName/bucketName/keyName
*/
public final class OMDBDefinition extends DBDefinition.WithMap {
- public static final DBColumnFamilyDefinition+ * This table complements the keyTable (or fileTable) * and dirTable entries of the immediately previous snapshot in the * same snapshot scope (bucket or volume). *
@@ -213,39 +301,43 @@ public final class OMDBDefinition extends DBDefinition.WithMap {
* renamedKey or renamedDir is present in the previous snapshot's keyTable
* (or fileTable).
*/
- public static final DBColumnFamilyDefinition