Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
73 changes: 0 additions & 73 deletions hadoop-ozone/ozone-manager/dev-support/findbugsExcludeFile.xml
Original file line number Diff line number Diff line change
Expand Up @@ -16,77 +16,4 @@
limitations under the License.
-->
<FindBugsFilter>
<!-- Test -->
<Match>
<Class name="org.apache.hadoop.ozone.om.TestChunkStreams"/>
<Bug pattern="URF_UNREAD_PUBLIC_OR_PROTECTED_FIELD" />
</Match>
<Match>
<Class name="org.apache.hadoop.ozone.om.TestOMStorage"/>
<Bug pattern="RV_RETURN_VALUE_IGNORED_BAD_PRACTICE" />
</Match>
<Match>
<Class name="org.apache.hadoop.ozone.om.TestOmMetadataManager"/>
<Bug pattern="UC_USELESS_OBJECT" />
</Match>
<Match>
<Class name="org.apache.hadoop.ozone.om.TestOzoneManagerHttpServer"/>
<Bug pattern="REC_CATCH_EXCEPTION" />
</Match>
<Match>
<Class name="org.apache.hadoop.ozone.om.TestOzoneManagerHttpServer"/>
<Bug pattern="RV_RETURN_VALUE_IGNORED_BAD_PRACTICE" />
</Match>
<Match>
<Class name="org.apache.hadoop.ozone.om.ratis.TestOzoneManagerDoubleBufferWithOMResponse"/>
<Bug pattern="UPM_UNCALLED_PRIVATE_METHOD" />
</Match>
<Match>
<Class name="org.apache.hadoop.ozone.om.ratis.TestOzoneManagerRatisServer"/>
<Bug pattern="URF_UNREAD_FIELD" />
</Match>
<Match>
<Class name="org.apache.hadoop.ozone.om.ratis.TestOzoneManagerDoubleBufferWithDummyResponse"/>
<Bug pattern="URF_UNREAD_PUBLIC_OR_PROTECTED_FIELD" />
</Match>
<Match>
<Class name="org.apache.hadoop.ozone.om.request.TestNormalizePaths"/>
<Bug pattern="URF_UNREAD_PUBLIC_OR_PROTECTED_FIELD" />
</Match>
<Match>
<Class name="~org\.apache\.hadoop\.ozone\.om\.request\.TestOMClientRequestWithUserInfo\$.*"/>
<Bug pattern="UMAC_UNCALLABLE_METHOD_OF_ANONYMOUS_CLASS" />
</Match>
<Match>
<Class name="org.apache.hadoop.ozone.om.request.security.TestOMDelegationTokenRequest"/>
<Bug pattern="UUF_UNUSED_PUBLIC_OR_PROTECTED_FIELD" />
</Match>
<Match>
<Class name="org.apache.hadoop.ozone.om.snapshot.TestSnapshotDiffManager" />
<Bug pattern="RV_RETURN_VALUE_IGNORED_NO_SIDE_EFFECT" />
</Match>
<Match>
<Class name="org.apache.hadoop.ozone.security.TestOzoneManagerBlockToken"/>
<Bug pattern="RV_RETURN_VALUE_IGNORED_BAD_PRACTICE" />
</Match>
<Match>
<Class name="org.apache.hadoop.ozone.security.TestOzoneManagerBlockToken"/>
<Bug pattern="UC_USELESS_OBJECT" />
</Match>
<Match>
<Class name="org.apache.hadoop.ozone.security.TestOzoneTokenIdentifier"/>
<Bug pattern="RV_RETURN_VALUE_IGNORED_BAD_PRACTICE" />
</Match>
<Match>
<Class name="org.apache.hadoop.ozone.security.TestOzoneTokenIdentifier"/>
<Bug pattern="UC_USELESS_OBJECT" />
</Match>
<Match>
<Class name="org.apache.hadoop.ozone.om.snapshot.TestFSODirectoryPathResolver"/>
<Bug pattern="DMI_HARDCODED_ABSOLUTE_FILENAME" />
</Match>
<Match>
<Class name="org.apache.hadoop.ozone.om.upgrade.TestOMLayoutFeatureAspect"/>
<Bug pattern="BC_IMPOSSIBLE_CAST" />
</Match>
</FindBugsFilter>
Original file line number Diff line number Diff line change
Expand Up @@ -205,16 +205,13 @@ public void testListAllVolumes() throws Exception {

@Test
public void testListBuckets() throws Exception {

String volumeName1 = "volumeA";
String prefixBucketNameWithOzoneOwner = "ozoneBucket";
String prefixBucketNameWithHadoopOwner = "hadoopBucket";

OMRequestTestUtils.addVolumeToDB(volumeName1, omMetadataManager);


TreeSet<String> volumeABucketsPrefixWithOzoneOwner = new TreeSet<>();
TreeSet<String> volumeABucketsPrefixWithHadoopOwner = new TreeSet<>();

// Add exact name in prefixBucketNameWithOzoneOwner without postfix.
volumeABucketsPrefixWithOzoneOwner.add(prefixBucketNameWithOzoneOwner);
Expand All @@ -225,24 +222,18 @@ public void testListBuckets() throws Exception {
prefixBucketNameWithOzoneOwner + i);
addBucketsToCache(volumeName1, prefixBucketNameWithOzoneOwner + i);
} else {
volumeABucketsPrefixWithHadoopOwner.add(
prefixBucketNameWithHadoopOwner + i);
addBucketsToCache(volumeName1, prefixBucketNameWithHadoopOwner + i);
}
}

String volumeName2 = "volumeB";
TreeSet<String> volumeBBucketsPrefixWithOzoneOwner = new TreeSet<>();
TreeSet<String> volumeBBucketsPrefixWithHadoopOwner = new TreeSet<>();
OMRequestTestUtils.addVolumeToDB(volumeName2, omMetadataManager);

// Add exact name in prefixBucketNameWithOzoneOwner without postfix.
volumeBBucketsPrefixWithOzoneOwner.add(prefixBucketNameWithOzoneOwner);
addBucketsToCache(volumeName2, prefixBucketNameWithOzoneOwner);
for (int i = 1; i < 100; i++) {
if (i % 2 == 0) { // This part adds 49 buckets.
volumeBBucketsPrefixWithOzoneOwner.add(
prefixBucketNameWithOzoneOwner + i);
addBucketsToCache(volumeName2, prefixBucketNameWithOzoneOwner + i);
} else {
volumeBBucketsPrefixWithHadoopOwner.add(
Expand All @@ -251,6 +242,8 @@ public void testListBuckets() throws Exception {
}
}

// VOLUME A

// List all buckets which have prefix ozoneBucket
List<OmBucketInfo> omBucketInfoList =
omMetadataManager.listBuckets(volumeName1,
Expand Down Expand Up @@ -291,7 +284,7 @@ public void testListBuckets() throws Exception {
assertNotEquals(prefixBucketNameWithOzoneOwner + 10, omBucketInfo.getBucketName());
}


// VOLUME B

omBucketInfoList = omMetadataManager.listBuckets(volumeName2,
null, prefixBucketNameWithHadoopOwner, 100, false);
Expand Down Expand Up @@ -370,26 +363,19 @@ public void testListKeys() throws Exception {
String prefixKeyB = "key-b";
String prefixKeyC = "key-c";
TreeSet<String> keysASet = new TreeSet<>();
TreeSet<String> keysBSet = new TreeSet<>();
TreeSet<String> keysCSet = new TreeSet<>();
for (int i = 1; i <= 100; i++) {
if (i % 2 == 0) {
keysASet.add(prefixKeyA + i);
addKeysToOM(volumeNameA, ozoneBucket, prefixKeyA + i, i);
} else {
keysBSet.add(prefixKeyB + i);
addKeysToOM(volumeNameA, hadoopBucket, prefixKeyB + i, i);
}
}
keysCSet.add(prefixKeyC + 1);
addKeysToOM(volumeNameA, ozoneTestBucket, prefixKeyC + 0, 0);

TreeSet<String> keysAVolumeBSet = new TreeSet<>();
TreeSet<String> keysBVolumeBSet = new TreeSet<>();
for (int i = 1; i <= 100; i++) {
if (i % 2 == 0) {
keysAVolumeBSet.add(
prefixKeyA + i);
addKeysToOM(volumeNameB, ozoneBucket, prefixKeyA + i, i);
} else {
keysBVolumeBSet.add(
Expand Down Expand Up @@ -524,6 +510,7 @@ public void testListKeysWithFewDeleteEntriesInCache() throws Exception {

// As in total 100, 50 are marked for delete. It should list only 50 keys.
assertEquals(50, omKeyInfoList.size());
assertEquals(50, deleteKeySet.size());

TreeSet<String> expectedKeys = new TreeSet<>();

Expand Down Expand Up @@ -593,7 +580,6 @@ public void testListOpenFiles(BucketLayout bucketLayout) throws Exception {
}

int numOpenKeys = 3;
List<String> openKeys = new ArrayList<>();
for (int i = 0; i < numOpenKeys; i++) {
final OmKeyInfo keyInfo = OMRequestTestUtils.createOmKeyInfo(volumeName, bucketName, keyPrefix + i,
RatisReplicationConfig.getInstance(ONE))
Expand All @@ -613,7 +599,6 @@ public void testListOpenFiles(BucketLayout bucketLayout) throws Exception {
dbOpenKeyName = omMetadataManager.getOpenKey(volumeName, bucketName,
keyInfo.getKeyName(), clientID);
}
openKeys.add(dbOpenKeyName);
}

String dbPrefix;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
import static org.junit.jupiter.api.Assertions.assertTrue;

import java.io.File;
import java.io.IOException;
import java.net.InetSocketAddress;
import java.net.URL;
import java.net.URLConnection;
Expand Down Expand Up @@ -59,10 +60,9 @@ public static Collection<Object[]> policy() {
}

@BeforeAll public static void setUp(@TempDir File baseDir) throws Exception {

// Create metadata directory
ozoneMetadataDirectory = new File(baseDir.getPath(), "metadata");
ozoneMetadataDirectory.mkdirs();
assertTrue(ozoneMetadataDirectory.mkdirs());

// Initialize the OzoneConfiguration
conf = new OzoneConfiguration();
Expand Down Expand Up @@ -146,15 +146,14 @@ private static boolean canAccess(String scheme, InetSocketAddress addr) {
return false;
}
try {
URL url =
new URL(scheme + "://" + NetUtils.getHostPortString(addr) + "/jmx");
URL url = new URL(scheme + "://" + NetUtils.getHostPortString(addr) + "/jmx");
URLConnection conn = connectionFactory.openConnection(url);
conn.connect();
conn.getContent();
} catch (Exception e) {
return true;
} catch (IOException e) {
return false;
}
return true;
}

private static boolean implies(boolean a, boolean b) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -61,8 +61,6 @@
import org.apache.hadoop.ozone.om.response.volume.OMVolumeCreateResponse;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.BucketInfo;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.DeleteBucketResponse;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.Daemon;
import org.apache.ozone.test.GenericTestUtils;
Expand Down Expand Up @@ -487,18 +485,5 @@ private OMBucketCreateResponse createBucket(String volumeName,
doubleBuffer.add(omClientResponse, termIndex);
return (OMBucketCreateResponse) omClientResponse;
}

/**
* Create OMBucketDeleteResponse for specified volume and bucket.
* @return OMBucketDeleteResponse
*/
private OMBucketDeleteResponse deleteBucket(String volumeName,
String bucketName) {
return new OMBucketDeleteResponse(OMResponse.newBuilder()
.setCmdType(OzoneManagerProtocolProtos.Type.DeleteBucket)
.setStatus(OzoneManagerProtocolProtos.Status.OK)
.setDeleteBucketResponse(DeleteBucketResponse.newBuilder().build())
.build(), volumeName, bucketName);
}
}

Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,6 @@ public class TestOzoneManagerRatisServer {
private OMMetadataManager omMetadataManager;
private OzoneManager ozoneManager;
private OMNodeDetails omNodeDetails;
private TermIndex initialTermIndex;
private SecurityConfig secConfig;
private OMCertificateClient certClient;

Expand Down Expand Up @@ -115,7 +114,6 @@ public void init(@TempDir Path metaDirPath) throws Exception {
omMetadataManager = new OmMetadataManagerImpl(ozoneConfiguration,
ozoneManager);
when(ozoneManager.getMetadataManager()).thenReturn(omMetadataManager);
initialTermIndex = TermIndex.valueOf(0, 0);
when(ozoneManager.getTransactionInfo()).thenReturn(TransactionInfo.DEFAULT_VALUE);
when(ozoneManager.getConfiguration()).thenReturn(conf);
final OmConfig omConfig = conf.getObject(OmConfig.class);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,6 @@
import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.ozone.om.OMMetadataManager;
import org.apache.hadoop.ozone.om.OMMetrics;
import org.apache.hadoop.ozone.om.OmMetadataManagerImpl;
import org.apache.hadoop.ozone.om.OzoneManager;
import org.junit.jupiter.api.AfterEach;
Expand All @@ -43,7 +42,6 @@ public class TestOMDelegationTokenRequest {
private Path folder;

protected OzoneManager ozoneManager;
protected OMMetrics omMetrics;
protected OMMetadataManager omMetadataManager;
protected ConfigurationSource conf;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -132,15 +132,23 @@ public void testGetAbsolutePathForValidObjectIDs() throws IOException {
Map<Long, Path> absolutePathMap = fsoDirectoryPathResolver
.getAbsolutePathForObjectIDs(Optional.of(objIds));

assertEquals(ImmutableMap.of(
17L, Paths.get("/dir3/dir14/dir17"),
9L, Paths.get("/dir2/dir9"),
10L, Paths.get("/dir2/dir10"),
15L, Paths.get("/dir3/dir15"),
4L, Paths.get("/dir4"),
3L, Paths.get("/dir3"),
1L, Paths.get("/")
), absolutePathMap);
Map<Long, String> pathMapping = ImmutableMap.<Long, String>builder()
.put(17L, "/dir3/dir14/dir17")
.put(9L, "/dir2/dir9")
.put(10L, "/dir2/dir10")
.put(15L, "/dir3/dir15")
.put(4L, "/dir4")
.put(3L, "/dir3")
.put(1L, "/")
.build();

Map<Long, Path> expectedPaths = pathMapping.entrySet().stream()
.collect(ImmutableMap.toImmutableMap(
Map.Entry::getKey,
e -> Paths.get(e.getValue())
));

assertEquals(expectedPaths, absolutePathMap);
assertEquals(objIds.size(), absolutePathMap.size());
// Invalid Obj Id 19 with dirInfo dir19 which is not present in the bucket.
assertThrows(IllegalArgumentException.class,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@
import static org.apache.ratis.util.JavaUtils.attempt;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
Expand Down Expand Up @@ -680,12 +681,13 @@ public void testObjectIdMapWithTombstoneEntries(boolean nativeLibraryLoaded,

SnapshotDiffManager spy = spy(snapshotDiffManager);

doAnswer(invocation -> {
Boolean isKeyInBucket = doAnswer(invocation -> {
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

String[] split = invocation.getArgument(0, String.class).split("/");
String keyName = split[split.length - 1];
return Integer.parseInt(keyName.substring(3)) % 2 == 0;
}
).when(spy).isKeyInBucket(anyString(), anyMap(), anyString());
assertFalse(isKeyInBucket);

PersistentMap<byte[], byte[]> oldObjectIdKeyMap =
new StubbedPersistentMap<>();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -208,7 +208,6 @@ public void testSymmetricTokenPerf() {
public void testSymmetricTokenPerfHelper(String hmacAlgorithm, int keyLen) {
final int testTokenCount = 1000;
List<OzoneTokenIdentifier> tokenIds = new ArrayList<>();
List<byte[]> tokenPasswordSym = new ArrayList<>();
for (int i = 0; i < testTokenCount; i++) {
tokenIds.add(generateTestToken());
}
Expand All @@ -234,8 +233,7 @@ public void testSymmetricTokenPerfHelper(String hmacAlgorithm, int keyLen) {

long startTime = Time.monotonicNowNanos();
for (int i = 0; i < testTokenCount; i++) {
tokenPasswordSym.add(
signTokenSymmetric(tokenIds.get(i), mac, secretKey));
signTokenSymmetric(tokenIds.get(i), mac, secretKey);
}
long duration = Time.monotonicNowNanos() - startTime;
LOG.info("Average token sign time with {}({} symmetric key) is {} ns",
Expand Down