diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java index dbac5f3abcbe92..f0b54e0c22d44e 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java @@ -100,6 +100,17 @@ public interface OMMetadataManager { String getOzoneKey(String volume, String bucket, String key); + /** + * Given a volume, bucket and a key, return the corresponding DB directory + * key. + * + * @param volume - volume name + * @param bucket - bucket name + * @param key - key name + * @return DB directory key as String. + */ + String getOzoneDirKey(String volume, String bucket, String key); + /** * Returns the DB key name of a open key in OM metadata store. Should be diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java index 0741777a61a723..88fd24cf42cb30 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java @@ -46,6 +46,7 @@ import org.apache.hadoop.ozone.om.helpers.OmMultipartKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmPrefixInfo; import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; +import org.apache.hadoop.ozone.om.helpers.OzoneFSUtils; import org.apache.hadoop.ozone.om.helpers.S3SecretValue; import org.apache.hadoop.ozone.om.lock.OzoneManagerLock; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.VolumeList; @@ -381,6 +382,12 @@ public String getOzoneKey(String volume, String bucket, String key) { return builder.toString(); } + @Override + public String getOzoneDirKey(String volume, String bucket, String key) { + key = OzoneFSUtils.addTrailingSlashIfNeeded(key); + return getOzoneKey(volume, bucket, key); + } + @Override public String getOpenKey(String volume, String bucket, String key, long id) { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java index 668a955e887dc1..abdfd12f4f206d 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java @@ -23,6 +23,7 @@ import org.apache.hadoop.ozone.om.request.bucket.OMBucketDeleteRequest; import org.apache.hadoop.ozone.om.request.bucket.OMBucketSetPropertyRequest; import org.apache.hadoop.ozone.om.request.OMClientRequest; +import org.apache.hadoop.ozone.om.request.file.OMDirectoryCreateRequest; import org.apache.hadoop.ozone.om.request.key.OMAllocateBlockRequest; import org.apache.hadoop.ozone.om.request.key.OMKeyCommitRequest; import org.apache.hadoop.ozone.om.request.key.OMKeyCreateRequest; @@ -90,6 +91,8 @@ public static OMClientRequest createClientRequest(OMRequest omRequest) { return new OMKeyDeleteRequest(omRequest); case RenameKey: return new OMKeyRenameRequest(omRequest); + case CreateDirectory: + return new OMDirectoryCreateRequest(omRequest); default: // TODO: will update once all request types are implemented. return null; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMDirectoryCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMDirectoryCreateRequest.java new file mode 100644 index 00000000000000..d1bf41b62a8191 --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMDirectoryCreateRequest.java @@ -0,0 +1,282 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om.request.file; + +import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Map; + +import com.google.common.base.Optional; +import com.google.common.base.Preconditions; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.hadoop.fs.FileEncryptionInfo; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos; +import org.apache.hadoop.ozone.audit.AuditLogger; +import org.apache.hadoop.ozone.audit.OMAction; +import org.apache.hadoop.ozone.om.OMMetadataManager; +import org.apache.hadoop.ozone.om.OMMetrics; +import org.apache.hadoop.ozone.om.OzoneManager; +import org.apache.hadoop.ozone.om.exceptions.OMException; +import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; +import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup; +import org.apache.hadoop.ozone.om.helpers.OzoneFSUtils; +import org.apache.hadoop.ozone.om.request.OMClientRequest; +import org.apache.hadoop.ozone.om.request.key.OMKeyRequest; +import org.apache.hadoop.ozone.om.response.OMClientResponse; +import org.apache.hadoop.ozone.om.response.file.OMDirectoryCreateResponse; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos + .CreateDirectoryRequest; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos + .CreateDirectoryResponse; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos + .KeyArgs; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos + .OMRequest; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos + .OMResponse; +import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer; +import org.apache.hadoop.ozone.security.acl.OzoneObj; +import org.apache.hadoop.util.Time; +import org.apache.hadoop.utils.db.cache.CacheKey; +import org.apache.hadoop.utils.db.cache.CacheValue; + + +import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.BUCKET_NOT_FOUND; +import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.FILE_ALREADY_EXISTS; +import static org.apache.hadoop.ozone.om.lock.OzoneManagerLock.Resource.BUCKET_LOCK; +/** + * Handle create directory request. + */ +public class OMDirectoryCreateRequest extends OMClientRequest + implements OMKeyRequest { + + private static final Logger LOG = + LoggerFactory.getLogger(OMDirectoryCreateRequest.class); + + public OMDirectoryCreateRequest(OMRequest omRequest) { + super(omRequest); + } + + @Override + public OMRequest preExecute(OzoneManager ozoneManager) { + CreateDirectoryRequest createDirectoryRequest = + getOmRequest().getCreateDirectoryRequest(); + Preconditions.checkNotNull(createDirectoryRequest); + + KeyArgs.Builder newKeyArgs = createDirectoryRequest.getKeyArgs() + .toBuilder().setModificationTime(Time.now()); + + CreateDirectoryRequest.Builder newCreateDirectoryRequest = + createDirectoryRequest.toBuilder().setKeyArgs(newKeyArgs); + + return getOmRequest().toBuilder().setCreateDirectoryRequest( + newCreateDirectoryRequest).build(); + + } + + @Override + public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, + long transactionLogIndex) { + + KeyArgs keyArgs = getOmRequest().getCreateDirectoryRequest().getKeyArgs(); + + String volumeName = keyArgs.getVolumeName(); + String bucketName = keyArgs.getBucketName(); + String keyName = keyArgs.getKeyName(); + + OMResponse.Builder omResponse = + OzoneManagerProtocolProtos.OMResponse.newBuilder().setCmdType( + OzoneManagerProtocolProtos.Type.CreateDirectory).setStatus( + OzoneManagerProtocolProtos.Status.OK); + + OMMetrics omMetrics = ozoneManager.getMetrics(); + omMetrics.incNumCreateDirectory(); + + AuditLogger auditLogger = ozoneManager.getAuditLogger(); + OzoneManagerProtocolProtos.UserInfo userInfo = getOmRequest().getUserInfo(); + + Map auditMap = buildKeyArgsAuditMap(keyArgs); + OMMetadataManager omMetadataManager = ozoneManager.getMetadataManager(); + boolean acquiredLock = false; + IOException exception = null; + OmKeyInfo dirKeyInfo = null; + + try { + // check Acl + if (ozoneManager.getAclsEnabled()) { + checkAcls(ozoneManager, OzoneObj.ResourceType.BUCKET, + OzoneObj.StoreType.OZONE, IAccessAuthorizer.ACLType.WRITE, + volumeName, bucketName, keyName); + } + + // Check if this is the root of the filesystem. + if (keyName.length() == 0) { + return new OMDirectoryCreateResponse(null, + omResponse.setCreateDirectoryResponse( + CreateDirectoryResponse.newBuilder()).build()); + } + // acquire lock + acquiredLock = omMetadataManager.getLock().acquireLock(BUCKET_LOCK, + volumeName, bucketName); + + // TODO: Not checking volume exist here, once we have full cache we can + // add volume exist check also. + + OmBucketInfo omBucketInfo = omMetadataManager.getBucketTable().get( + omMetadataManager.getBucketKey(volumeName, bucketName)); + + if (omBucketInfo == null) { + throw new OMException("Bucket not found " + bucketName, + BUCKET_NOT_FOUND); + } + + + // Need to check if any files exist in the given path, if they exist we + // cannot create a directory with the given key. + OMDirectoryResult omDirectoryResult = verifyFilesInPath(omMetadataManager, + volumeName, bucketName, omMetadataManager.getOzoneDirKey(volumeName, + bucketName, keyName), Paths.get(keyName)); + + if (omDirectoryResult == OMDirectoryResult.FILE_ALREADY_EXISTS) { + throw new OMException("Unable to create directory: " +keyName + + " in volume/bucket: " + volumeName + "/" + bucketName, + FILE_ALREADY_EXISTS); + } else if (omDirectoryResult == OMDirectoryResult.SUB_DIRECTORY_EXISTS || + omDirectoryResult == OMDirectoryResult.NONE) { + dirKeyInfo = createDirectoryKeyInfo(ozoneManager, omBucketInfo, + volumeName, bucketName, keyName, keyArgs); + + omMetadataManager.getKeyTable().addCacheEntry( + new CacheKey<>(omMetadataManager.getOzoneKey(volumeName, bucketName, + dirKeyInfo.getKeyName())), + new CacheValue<>(Optional.of(dirKeyInfo), transactionLogIndex)); + } + // if directory already exists do nothing or do we need to throw + // exception? Current KeyManagerImpl code does just return, following + // similar approach. + + } catch (IOException ex) { + exception = ex; + } finally { + if (acquiredLock) { + omMetadataManager.getLock().releaseLock(BUCKET_LOCK, volumeName, + bucketName); + } + } + + auditLog(auditLogger, buildAuditMessage(OMAction.CREATE_DIRECTORY, + auditMap, exception, userInfo)); + + if (exception == null) { + LOG.debug("Directory is successfully created for Key: {} in " + + "volume/bucket:{}/{}", keyName, volumeName, bucketName); + omResponse.setCreateDirectoryResponse( + CreateDirectoryResponse.newBuilder()); + return new OMDirectoryCreateResponse(dirKeyInfo, + omResponse.build()); + } else { + LOG.error("CreateDirectory failed for Key: {} in volume/bucket:{}/{}", + keyName, volumeName, bucketName, exception); + omMetrics.incNumCreateDirectoryFails(); + return new OMDirectoryCreateResponse(null, + createErrorOMResponse(omResponse, exception)); + } + } + + /** + * Verify any files exist in the given path in the specified volume/bucket. + * @param omMetadataManager + * @param volumeName + * @param bucketName + * @param keyPath + * @return true - if file exist in the given path, else false. + * @throws IOException + */ + private OMDirectoryResult verifyFilesInPath( + OMMetadataManager omMetadataManager, String volumeName, String bucketName, + String directoryName, Path keyPath) throws IOException { + + while (keyPath != null) { + String keyName = keyPath.toString(); + + String dbKeyName = omMetadataManager.getOzoneKey(volumeName, + bucketName, keyName); + String dbDirKeyName = omMetadataManager.getOzoneDirKey(volumeName, + bucketName, keyName); + + if (omMetadataManager.getKeyTable().get(dbKeyName) != null) { + // Found a file in the given path. + return OMDirectoryResult.FILE_ALREADY_EXISTS; + } else if (omMetadataManager.getKeyTable().get(dbDirKeyName) != null) { + if (dbDirKeyName.equals(directoryName)) { + return OMDirectoryResult.DIRECTORY_ALREADY_EXISTS; + } else { + return OMDirectoryResult.SUB_DIRECTORY_EXISTS; + } + } + keyPath = keyPath.getParent(); + } + + // Found no files/ directories in the given path. + return OMDirectoryResult.NONE; + } + + + private OmKeyInfo createDirectoryKeyInfo(OzoneManager ozoneManager, + OmBucketInfo omBucketInfo, String volumeName, String bucketName, + String keyName, KeyArgs keyArgs) + throws IOException { + FileEncryptionInfo encryptionInfo = getFileEncryptionInfo(ozoneManager, + omBucketInfo); + String dirName = OzoneFSUtils.addTrailingSlashIfNeeded(keyName); + + return new OmKeyInfo.Builder() + .setVolumeName(volumeName) + .setBucketName(bucketName) + .setKeyName(dirName) + .setOmKeyLocationInfos(Collections.singletonList( + new OmKeyLocationInfoGroup(0, new ArrayList<>()))) + .setCreationTime(keyArgs.getModificationTime()) + .setModificationTime(keyArgs.getModificationTime()) + .setDataSize(0) + .setReplicationType(HddsProtos.ReplicationType.RATIS) + .setReplicationFactor(HddsProtos.ReplicationFactor.ONE) + .setFileEncryptionInfo(encryptionInfo) + .setAcls(keyArgs.getAclsList()) + .build(); + } + + /** + * Return codes used by verifyFilesInPath method. + */ + enum OMDirectoryResult { + DIRECTORY_ALREADY_EXISTS, + FILE_ALREADY_EXISTS, + SUB_DIRECTORY_EXISTS, + NONE + } + +} diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/package-info.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/package-info.java new file mode 100644 index 00000000000000..3184500f38501e --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/package-info.java @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +/** + * Package contains classes related to file requests. + */ +package org.apache.hadoop.ozone.om.request.file; \ No newline at end of file diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/file/OMDirectoryCreateResponse.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/file/OMDirectoryCreateResponse.java new file mode 100644 index 00000000000000..95eaa781a073cd --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/file/OMDirectoryCreateResponse.java @@ -0,0 +1,67 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om.response.file; + +import org.apache.hadoop.ozone.om.OMMetadataManager; +import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.response.OMClientResponse; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos + .OMResponse; +import org.apache.hadoop.utils.db.BatchOperation; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.annotation.Nullable; +import java.io.IOException; + +/** + * Response for create directory request. + */ +public class OMDirectoryCreateResponse extends OMClientResponse { + + public static final Logger LOG = + LoggerFactory.getLogger(OMDirectoryCreateResponse.class); + private OmKeyInfo dirKeyInfo; + + public OMDirectoryCreateResponse(@Nullable OmKeyInfo dirKeyInfo, + OMResponse omResponse) { + super(omResponse); + this.dirKeyInfo = dirKeyInfo; + } + + @Override + public void addToDBBatch(OMMetadataManager omMetadataManager, + BatchOperation batchOperation) throws IOException { + if (getOMResponse().getStatus() == OzoneManagerProtocolProtos.Status.OK) { + if (dirKeyInfo != null) { + String dirKey = + omMetadataManager.getOzoneKey(dirKeyInfo.getVolumeName(), + dirKeyInfo.getBucketName(), dirKeyInfo.getKeyName()); + omMetadataManager.getKeyTable().putWithBatch(batchOperation, dirKey, + dirKeyInfo); + } else { + // When directory already exists, we don't add it to cache. And it is + // not an error, in this case dirKeyInfo will be null. + LOG.debug("Response Status is OK, dirKeyInfo is null in " + + "OMDirectoryCreateResponse"); + } + } + } +} diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/file/package-info.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/file/package-info.java new file mode 100644 index 00000000000000..135eca9d3cb8d8 --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/file/package-info.java @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +/** + * Package contains classes related to file responses. + */ +package org.apache.hadoop.ozone.om.response.file; \ No newline at end of file diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerHARequestHandlerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerHARequestHandlerImpl.java index 029746782d9502..68029f89473b08 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerHARequestHandlerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerHARequestHandlerImpl.java @@ -108,6 +108,7 @@ public OMResponse handleApplyTransaction(OMRequest omRequest, case CommitKey: case DeleteKey: case RenameKey: + case CreateDirectory: //TODO: We don't need to pass transactionID, this will be removed when // complete write requests is changed to new model. And also we can // return OMClientResponse, then adding to doubleBuffer can be taken diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerProtocolServerSideTranslatorPB.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerProtocolServerSideTranslatorPB.java index 08c1cfd856e1f9..56522b75ca4f7d 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerProtocolServerSideTranslatorPB.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerProtocolServerSideTranslatorPB.java @@ -85,7 +85,6 @@ public OMResponse submitRequest(RpcController controller, if (OmUtils.isReadOnly(request)) { return submitReadRequestToOM(request); } else { - // PreExecute if needed. if (omRatisServer.isLeader()) { try { OMClientRequest omClientRequest = @@ -93,7 +92,7 @@ public OMResponse submitRequest(RpcController controller, if (omClientRequest != null) { request = omClientRequest.preExecute(ozoneManager); } - } catch (IOException ex) { + } catch(IOException ex) { // As some of the preExecute returns error. So handle here. return createErrorResponse(request, ex); } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/file/TestOMDirectoryCreateRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/file/TestOMDirectoryCreateRequest.java new file mode 100644 index 00000000000000..20cb96eba7ccba --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/file/TestOMDirectoryCreateRequest.java @@ -0,0 +1,327 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om.request.file; + +import java.util.UUID; + +import org.apache.commons.lang.RandomStringUtils; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos; +import org.apache.hadoop.ozone.om.helpers.OzoneFSUtils; +import org.apache.hadoop.ozone.om.response.OMClientResponse; +import org.apache.hadoop.utils.db.cache.CacheKey; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.mockito.Mockito; + +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.ozone.audit.AuditLogger; +import org.apache.hadoop.ozone.audit.AuditMessage; +import org.apache.hadoop.ozone.om.OMConfigKeys; +import org.apache.hadoop.ozone.om.OMMetadataManager; +import org.apache.hadoop.ozone.om.OMMetrics; +import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; +import org.apache.hadoop.ozone.om.OzoneManager; +import org.apache.hadoop.ozone.om.request.TestOMRequestUtils; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos + .CreateDirectoryRequest; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos + .KeyArgs; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos + .OMRequest; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.when; + +/** + * Test OM directory create request. + */ +public class TestOMDirectoryCreateRequest { + + @Rule + public TemporaryFolder folder = new TemporaryFolder(); + + private OzoneManager ozoneManager; + private OMMetrics omMetrics; + private OMMetadataManager omMetadataManager; + private AuditLogger auditLogger; + + + @Before + public void setup() throws Exception { + ozoneManager = Mockito.mock(OzoneManager.class); + omMetrics = OMMetrics.create(); + OzoneConfiguration ozoneConfiguration = new OzoneConfiguration(); + ozoneConfiguration.set(OMConfigKeys.OZONE_OM_DB_DIRS, + folder.newFolder().getAbsolutePath()); + omMetadataManager = new OmMetadataManagerImpl(ozoneConfiguration); + when(ozoneManager.getMetrics()).thenReturn(omMetrics); + when(ozoneManager.getMetadataManager()).thenReturn(omMetadataManager); + auditLogger = Mockito.mock(AuditLogger.class); + when(ozoneManager.getAuditLogger()).thenReturn(auditLogger); + Mockito.doNothing().when(auditLogger).logWrite(any(AuditMessage.class)); + } + + @After + public void stop() { + omMetrics.unRegister(); + Mockito.framework().clearInlineMocks(); + } + + @Test + public void testPreExecute() throws Exception { + + String volumeName = "vol1"; + String bucketName = "bucket1"; + String keyName = "a/b/c"; + + TestOMRequestUtils.addVolumeAndBucketToDB(volumeName, bucketName, + omMetadataManager); + + OMRequest omRequest = createDirectoryRequest(volumeName, bucketName, + keyName); + OMDirectoryCreateRequest omDirectoryCreateRequest = + new OMDirectoryCreateRequest(omRequest); + + OMRequest modifiedOmRequest = + omDirectoryCreateRequest.preExecute(ozoneManager); + + // As in preExecute, we modify original request. + Assert.assertNotEquals(omRequest, modifiedOmRequest); + + } + + + @Test + public void testValidateAndUpdateCache() throws Exception { + String volumeName = "vol1"; + String bucketName = "bucket1"; + String keyName = RandomStringUtils.randomAlphabetic(5); + for (int i =0; i< 3; i++) { + keyName += "/" + RandomStringUtils.randomAlphabetic(5); + } + + // Add volume and bucket entries to DB. + TestOMRequestUtils.addVolumeAndBucketToDB(volumeName, bucketName, + omMetadataManager); + + OMRequest omRequest = createDirectoryRequest(volumeName, bucketName, + keyName); + OMDirectoryCreateRequest omDirectoryCreateRequest = + new OMDirectoryCreateRequest(omRequest); + + OMRequest modifiedOmRequest = + omDirectoryCreateRequest.preExecute(ozoneManager); + + omDirectoryCreateRequest = new OMDirectoryCreateRequest(modifiedOmRequest); + + OMClientResponse omClientResponse = + omDirectoryCreateRequest.validateAndUpdateCache(ozoneManager, 100L); + + Assert.assertTrue(omClientResponse.getOMResponse().getStatus() + == OzoneManagerProtocolProtos.Status.OK); + Assert.assertTrue(omMetadataManager.getKeyTable().get( + omMetadataManager.getOzoneDirKey( + volumeName, bucketName, keyName)) != null); + + } + + + @Test + public void testValidateAndUpdateCacheWithBucketNotFound() throws Exception { + String volumeName = "vol1"; + String bucketName = "bucket1"; + String keyName = RandomStringUtils.randomAlphabetic(5); + for (int i =0; i< 3; i++) { + keyName += "/" + RandomStringUtils.randomAlphabetic(5); + } + + OMRequest omRequest = createDirectoryRequest(volumeName, bucketName, + keyName); + OMDirectoryCreateRequest omDirectoryCreateRequest = + new OMDirectoryCreateRequest(omRequest); + + OMRequest modifiedOmRequest = + omDirectoryCreateRequest.preExecute(ozoneManager); + + omDirectoryCreateRequest = new OMDirectoryCreateRequest(modifiedOmRequest); + + OMClientResponse omClientResponse = + omDirectoryCreateRequest.validateAndUpdateCache(ozoneManager, 100L); + + Assert.assertTrue(omClientResponse.getOMResponse().getStatus() + == OzoneManagerProtocolProtos.Status.BUCKET_NOT_FOUND); + + // Key should not exist in DB + Assert.assertTrue(omMetadataManager.getKeyTable().get( + omMetadataManager.getOzoneDirKey( + volumeName, bucketName, keyName)) == null); + + } + + @Test + public void testValidateAndUpdateCacheWithSubDirectoryInPath() + throws Exception { + String volumeName = "vol1"; + String bucketName = "bucket1"; + String keyName = RandomStringUtils.randomAlphabetic(5); + for (int i =0; i< 3; i++) { + keyName += "/" + RandomStringUtils.randomAlphabetic(5); + } + + // Add volume and bucket entries to DB. + TestOMRequestUtils.addVolumeAndBucketToDB(volumeName, bucketName, + omMetadataManager); + + TestOMRequestUtils.addKeyToTable(false, volumeName, bucketName, + keyName.substring(0, 12), 1L, HddsProtos.ReplicationType.RATIS, + HddsProtos.ReplicationFactor.ONE, omMetadataManager); + OMRequest omRequest = createDirectoryRequest(volumeName, bucketName, + keyName); + OMDirectoryCreateRequest omDirectoryCreateRequest = + new OMDirectoryCreateRequest(omRequest); + + OMRequest modifiedOmRequest = + omDirectoryCreateRequest.preExecute(ozoneManager); + + omDirectoryCreateRequest = new OMDirectoryCreateRequest(modifiedOmRequest); + + OMClientResponse omClientResponse = + omDirectoryCreateRequest.validateAndUpdateCache(ozoneManager, 100L); + + Assert.assertTrue(omClientResponse.getOMResponse().getStatus() + == OzoneManagerProtocolProtos.Status.OK); + + // Key should exist in DB and cache. + Assert.assertTrue(omMetadataManager.getKeyTable().get( + omMetadataManager.getOzoneDirKey( + volumeName, bucketName, keyName)) != null); + Assert.assertTrue(omMetadataManager.getKeyTable().getCacheValue( + new CacheKey<>(omMetadataManager.getOzoneDirKey( + volumeName, bucketName, keyName))) != null); + + } + + @Test + public void testValidateAndUpdateCacheWithDirectoryAlreadyExists() + throws Exception { + String volumeName = "vol1"; + String bucketName = "bucket1"; + String keyName = RandomStringUtils.randomAlphabetic(5); + for (int i =0; i< 3; i++) { + keyName += "/" + RandomStringUtils.randomAlphabetic(5); + } + + // Add volume and bucket entries to DB. + TestOMRequestUtils.addVolumeAndBucketToDB(volumeName, bucketName, + omMetadataManager); + + TestOMRequestUtils.addKeyToTable(false, volumeName, bucketName, + OzoneFSUtils.addTrailingSlashIfNeeded(keyName), 1L, + HddsProtos.ReplicationType.RATIS, HddsProtos.ReplicationFactor.ONE, + omMetadataManager); + OMRequest omRequest = createDirectoryRequest(volumeName, bucketName, + keyName); + OMDirectoryCreateRequest omDirectoryCreateRequest = + new OMDirectoryCreateRequest(omRequest); + + OMRequest modifiedOmRequest = + omDirectoryCreateRequest.preExecute(ozoneManager); + + omDirectoryCreateRequest = new OMDirectoryCreateRequest(modifiedOmRequest); + + OMClientResponse omClientResponse = + omDirectoryCreateRequest.validateAndUpdateCache(ozoneManager, 100L); + + Assert.assertTrue(omClientResponse.getOMResponse().getStatus() + == OzoneManagerProtocolProtos.Status.OK); + + // Key should exist in DB + Assert.assertTrue(omMetadataManager.getKeyTable().get( + omMetadataManager.getOzoneDirKey( + volumeName, bucketName, keyName)) != null); + + // As it already exists, it should not be in cache. + Assert.assertTrue(omMetadataManager.getKeyTable().getCacheValue( + new CacheKey<>(omMetadataManager.getOzoneDirKey( + volumeName, bucketName, keyName))) == null); + + } + + @Test + public void testValidateAndUpdateCacheWithFilesInPath() throws Exception { + String volumeName = "vol1"; + String bucketName = "bucket1"; + String keyName = RandomStringUtils.randomAlphabetic(5); + for (int i =0; i< 3; i++) { + keyName += "/" + RandomStringUtils.randomAlphabetic(5); + } + + // Add volume and bucket entries to DB. + TestOMRequestUtils.addVolumeAndBucketToDB(volumeName, bucketName, + omMetadataManager); + // Add a key with first two levels. + TestOMRequestUtils.addKeyToTable(false, volumeName, bucketName, + keyName.substring(0, 11), 1L, HddsProtos.ReplicationType.RATIS, + HddsProtos.ReplicationFactor.ONE, omMetadataManager); + OMRequest omRequest = createDirectoryRequest(volumeName, bucketName, + keyName); + OMDirectoryCreateRequest omDirectoryCreateRequest = + new OMDirectoryCreateRequest(omRequest); + + OMRequest modifiedOmRequest = + omDirectoryCreateRequest.preExecute(ozoneManager); + + omDirectoryCreateRequest = new OMDirectoryCreateRequest(modifiedOmRequest); + + OMClientResponse omClientResponse = + omDirectoryCreateRequest.validateAndUpdateCache(ozoneManager, 100L); + + Assert.assertTrue(omClientResponse.getOMResponse().getStatus() + == OzoneManagerProtocolProtos.Status.FILE_ALREADY_EXISTS); + + // Key should not exist in DB + Assert.assertTrue(omMetadataManager.getKeyTable().get( + omMetadataManager.getOzoneDirKey( + volumeName, bucketName, keyName)) == null); + + } + + /** + * Create OMRequest which encapsulates CreateDirectory request. + * @param volumeName + * @param bucketName + * @param keyName + * @return OMRequest + */ + private OMRequest createDirectoryRequest(String volumeName, String bucketName, + String keyName) { + return OMRequest.newBuilder().setCreateDirectoryRequest( + CreateDirectoryRequest.newBuilder().setKeyArgs( + KeyArgs.newBuilder().setVolumeName(volumeName) + .setBucketName(bucketName).setKeyName(keyName))) + .setCmdType(OzoneManagerProtocolProtos.Type.CreateDirectory) + .setClientId(UUID.randomUUID().toString()).build(); + } + +} diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/file/package-info.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/file/package-info.java new file mode 100644 index 00000000000000..ab81a7ed42153c --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/file/package-info.java @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +/** + * Package contains test classes for file requests. + */ +package org.apache.hadoop.ozone.om.request.file; \ No newline at end of file diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/file/TestOMDirectoryCreateResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/file/TestOMDirectoryCreateResponse.java new file mode 100644 index 00000000000000..c9e9746a627741 --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/file/TestOMDirectoryCreateResponse.java @@ -0,0 +1,123 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om.response.file; + +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos; +import org.apache.hadoop.ozone.om.OMConfigKeys; +import org.apache.hadoop.ozone.om.OMMetadataManager; +import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; +import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.OzoneFSUtils; +import org.apache.hadoop.ozone.om.request.TestOMRequestUtils; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos + .OMResponse; +import org.apache.hadoop.test.GenericTestUtils; +import org.apache.hadoop.utils.db.BatchOperation; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.slf4j.event.Level; + +import java.util.UUID; + +/** + * Tests OMDirectoryCreateResponse. + */ +public class TestOMDirectoryCreateResponse { + @Rule + public TemporaryFolder folder = new TemporaryFolder(); + + private OMMetadataManager omMetadataManager; + private BatchOperation batchOperation; + + @Before + public void setup() throws Exception { + OzoneConfiguration ozoneConfiguration = new OzoneConfiguration(); + ozoneConfiguration.set(OMConfigKeys.OZONE_OM_DB_DIRS, + folder.newFolder().getAbsolutePath()); + omMetadataManager = new OmMetadataManagerImpl(ozoneConfiguration); + batchOperation = omMetadataManager.getStore().initBatchOperation(); + } + + @Test + public void testAddToDBBatch() throws Exception { + + String volumeName = UUID.randomUUID().toString(); + String keyName = UUID.randomUUID().toString(); + String bucketName = UUID.randomUUID().toString(); + + OmKeyInfo omKeyInfo = TestOMRequestUtils.createOmKeyInfo(volumeName, + bucketName, OzoneFSUtils.addTrailingSlashIfNeeded(keyName), + HddsProtos.ReplicationType.RATIS, HddsProtos.ReplicationFactor.ONE); + + OMResponse omResponse = OMResponse.newBuilder().setCreateDirectoryResponse( + OzoneManagerProtocolProtos.CreateDirectoryResponse.getDefaultInstance()) + .setStatus(OzoneManagerProtocolProtos.Status.OK) + .setCmdType(OzoneManagerProtocolProtos.Type.CreateDirectory) + .build(); + + OMDirectoryCreateResponse omDirectoryCreateResponse = + new OMDirectoryCreateResponse(omKeyInfo, omResponse); + + omDirectoryCreateResponse.addToDBBatch(omMetadataManager, batchOperation); + + // Do manual commit and see whether addToBatch is successful or not. + omMetadataManager.getStore().commitBatchOperation(batchOperation); + + Assert.assertNotNull(omMetadataManager.getKeyTable().get( + omMetadataManager.getOzoneDirKey(volumeName, bucketName, keyName))); + } + + @Test + public void testAddToDBBatchWithNullOmkeyInfo() throws Exception { + + GenericTestUtils.setLogLevel(OMDirectoryCreateResponse.LOG, Level.DEBUG); + GenericTestUtils.LogCapturer logCapturer = GenericTestUtils.LogCapturer + .captureLogs(OMDirectoryCreateResponse.LOG); + + + String volumeName = UUID.randomUUID().toString(); + String keyName = UUID.randomUUID().toString(); + String bucketName = UUID.randomUUID().toString(); + + OMResponse omResponse = OMResponse.newBuilder().setCreateDirectoryResponse( + OzoneManagerProtocolProtos.CreateDirectoryResponse.getDefaultInstance()) + .setStatus(OzoneManagerProtocolProtos.Status.OK) + .setCmdType(OzoneManagerProtocolProtos.Type.CreateDirectory) + .build(); + + OMDirectoryCreateResponse omDirectoryCreateResponse = + new OMDirectoryCreateResponse(null, omResponse); + + omDirectoryCreateResponse.addToDBBatch(omMetadataManager, batchOperation); + + // Do manual commit and see whether addToBatch is successful or not. + omMetadataManager.getStore().commitBatchOperation(batchOperation); + + Assert.assertNull(omMetadataManager.getKeyTable().get( + omMetadataManager.getOzoneDirKey(volumeName, bucketName, keyName))); + + Assert.assertTrue(logCapturer.getOutput().contains("Response Status is " + + "OK, dirKeyInfo is null")); + } +} diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/file/package-info.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/file/package-info.java new file mode 100644 index 00000000000000..4c6c005294da3e --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/file/package-info.java @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +/** + * Package contains test classes for file responses. + */ +package org.apache.hadoop.ozone.om.response.file; \ No newline at end of file