Skip to content

HBASE-27821 Split TestFuzzyRowFilterEndToEnd #5209

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Apache9 merged 1 commit into from
May 4, 2023
Merged

HBASE-27821 Split TestFuzzyRowFilterEndToEnd #5209

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
178 changes: 4 additions & 174 deletions hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFuzzyRowFilterEndToEnd.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,16 +41,12 @@
import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.filter.FilterList.Operator;
import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.regionserver.RegionScanner;
import org.apache.hadoop.hbase.testclassification.FilterTests;
import org.apache.hadoop.hbase.testclassification.LargeTests;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.apache.hadoop.hbase.util.Pair;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.ClassRule;
import org.junit.Rule;
Expand All @@ -62,31 +58,22 @@

import org.apache.hbase.thirdparty.com.google.common.collect.Lists;

@Category({ FilterTests.class, LargeTests.class })
@Category({ FilterTests.class, MediumTests.class })
public class TestFuzzyRowFilterEndToEnd {

@ClassRule
public static final HBaseClassTestRule CLASS_RULE =
HBaseClassTestRule.forClass(TestFuzzyRowFilterEndToEnd.class);

private final static HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil();
private final static byte fuzzyValue = (byte) 63;
private static final Logger LOG = LoggerFactory.getLogger(TestFuzzyRowFilterEndToEnd.class);

private static int firstPartCardinality = 50;
private static int secondPartCardinality = 50;
private static int thirdPartCardinality = 50;
private static int colQualifiersTotal = 5;
private static int totalFuzzyKeys = thirdPartCardinality / 2;
private static final HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil();

private static String table = "TestFuzzyRowFilterEndToEnd";
private static final byte fuzzyValue = (byte) 63;

@Rule
public TestName name = new TestName();

/**
* @throws java.lang.Exception
*/
@BeforeClass
public static void setUpBeforeClass() throws Exception {
Configuration conf = TEST_UTIL.getConfiguration();
Expand All @@ -99,30 +86,11 @@ public static void setUpBeforeClass() throws Exception {
TEST_UTIL.startMiniCluster();
}

/**
* @throws java.lang.Exception
*/
@AfterClass
public static void tearDownAfterClass() throws Exception {
TEST_UTIL.shutdownMiniCluster();
}

/**
* @throws java.lang.Exception
*/
@Before
public void setUp() throws Exception {
// Nothing to do.
}

/**
* @throws java.lang.Exception
*/
@After
public void tearDown() throws Exception {
// Nothing to do.
}

// HBASE-15676 Test that fuzzy info of all fixed bits (0s) finds matching row.
@Test
public void testAllFixedBits() throws IOException {
Expand Down Expand Up @@ -217,144 +185,6 @@ public void testHBASE14782() throws IOException {
TEST_UTIL.deleteTable(TableName.valueOf(name.getMethodName()));
}

@Test
public void testEndToEnd() throws Exception {
String cf = "f";

Table ht =
TEST_UTIL.createTable(TableName.valueOf(table), Bytes.toBytes(cf), Integer.MAX_VALUE);

// 10 byte row key - (2 bytes 4 bytes 4 bytes)
// 4 byte qualifier
// 4 byte value

for (int i0 = 0; i0 < firstPartCardinality; i0++) {

for (int i1 = 0; i1 < secondPartCardinality; i1++) {

for (int i2 = 0; i2 < thirdPartCardinality; i2++) {
byte[] rk = new byte[10];

ByteBuffer buf = ByteBuffer.wrap(rk);
buf.clear();
buf.putShort((short) i0);
buf.putInt(i1);
buf.putInt(i2);
for (int c = 0; c < colQualifiersTotal; c++) {
byte[] cq = new byte[4];
Bytes.putBytes(cq, 0, Bytes.toBytes(c), 0, 4);

Put p = new Put(rk);
p.setDurability(Durability.SKIP_WAL);
p.addColumn(Bytes.toBytes(cf), cq, Bytes.toBytes(c));
ht.put(p);
}
}
}
}

TEST_UTIL.flush();

// test passes
runTest1(ht);
runTest2(ht);

}

private void runTest1(Table hTable) throws IOException {
// [0, 2, ?, ?, ?, ?, 0, 0, 0, 1]

byte[] mask = new byte[] { 0, 0, 1, 1, 1, 1, 0, 0, 0, 0 };

List<Pair<byte[], byte[]>> list = new ArrayList<>();
for (int i = 0; i < totalFuzzyKeys; i++) {
byte[] fuzzyKey = new byte[10];
ByteBuffer buf = ByteBuffer.wrap(fuzzyKey);
buf.clear();
buf.putShort((short) 2);
for (int j = 0; j < 4; j++) {
buf.put(fuzzyValue);
}
buf.putInt(i);

Pair<byte[], byte[]> pair = new Pair<>(fuzzyKey, mask);
list.add(pair);
}

int expectedSize = secondPartCardinality * totalFuzzyKeys * colQualifiersTotal;
FuzzyRowFilter fuzzyRowFilter0 = new FuzzyRowFilter(list);
// Filters are not stateless - we can't reuse them
FuzzyRowFilter fuzzyRowFilter1 = new FuzzyRowFilter(list);

// regular test
runScanner(hTable, expectedSize, fuzzyRowFilter0);
// optimized from block cache
runScanner(hTable, expectedSize, fuzzyRowFilter1);

}

private void runTest2(Table hTable) throws IOException {
// [0, 0, ?, ?, ?, ?, 0, 0, 0, 0] , [0, 1, ?, ?, ?, ?, 0, 0, 0, 1]...

byte[] mask = new byte[] { 0, 0, 1, 1, 1, 1, 0, 0, 0, 0 };

List<Pair<byte[], byte[]>> list = new ArrayList<>();

for (int i = 0; i < totalFuzzyKeys; i++) {
byte[] fuzzyKey = new byte[10];
ByteBuffer buf = ByteBuffer.wrap(fuzzyKey);
buf.clear();
buf.putShort((short) (i * 2));
for (int j = 0; j < 4; j++) {
buf.put(fuzzyValue);
}
buf.putInt(i * 2);

Pair<byte[], byte[]> pair = new Pair<>(fuzzyKey, mask);
list.add(pair);
}

int expectedSize = totalFuzzyKeys * secondPartCardinality * colQualifiersTotal;

FuzzyRowFilter fuzzyRowFilter0 = new FuzzyRowFilter(list);
// Filters are not stateless - we can't reuse them
FuzzyRowFilter fuzzyRowFilter1 = new FuzzyRowFilter(list);

// regular test
runScanner(hTable, expectedSize, fuzzyRowFilter0);
// optimized from block cache
runScanner(hTable, expectedSize, fuzzyRowFilter1);

}

private void runScanner(Table hTable, int expectedSize, Filter filter) throws IOException {

String cf = "f";
Scan scan = new Scan();
scan.addFamily(Bytes.toBytes(cf));
scan.setFilter(filter);
List<HRegion> regions = TEST_UTIL.getHBaseCluster().getRegions(TableName.valueOf(table));
HRegion first = regions.get(0);
first.getScanner(scan);
RegionScanner scanner = first.getScanner(scan);
List<Cell> results = new ArrayList<>();
// Result result;
long timeBeforeScan = EnvironmentEdgeManager.currentTime();
int found = 0;
while (scanner.next(results)) {
found += results.size();
results.clear();
}
found += results.size();
long scanTime = EnvironmentEdgeManager.currentTime() - timeBeforeScan;
scanner.close();

LOG.info("\nscan time = " + scanTime + "ms");
LOG.info("found " + found + " results\n");

assertEquals(expectedSize, found);
}

@Test
public void testFilterList() throws Exception {
String cf = "f";
Expand Down
Loading