Skip to content

Commit 5d46eee

Browse files
DaveCTurnerDaveCTurner
committed
Introduce DocHistoryEntry
Today, each test has its own process for deciding whether to refresh/flush/GC deletes after applying an operation. This change moves this decision into generateSingleDocHistory().
1 parent d53649b commit 5d46eee

File tree

1 file changed

+60
-32
lines changed

1 file changed

+60
-32
lines changed

‎server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java‎

Lines changed: 60 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -1384,11 +1384,26 @@ public void testVersioningCreateExistsException() throws IOException {
13841384
assertThat(indexResult.getFailure(), instanceOf(VersionConflictEngineException.class));
13851385
}
13861386

1387-
protected List<Engine.Operation> generateSingleDocHistory(boolean forReplica, VersionType versionType,
1388-
boolean partialOldPrimary, long primaryTerm,
1389-
int minOpCount, int maxOpCount) {
1387+
private class DocHistoryEntry {
1388+
final Engine.Operation op;
1389+
final boolean refreshAfterOperation;
1390+
final boolean flushAfterOperation;
1391+
final boolean gcDeletesAfterOperation;
1392+
1393+
private DocHistoryEntry(Engine.Operation op, boolean refreshAfterOperation,
1394+
boolean flushAfterOperation, boolean gcDeletesAfterOperation) {
1395+
this.op = op;
1396+
this.refreshAfterOperation = refreshAfterOperation;
1397+
this.flushAfterOperation = flushAfterOperation;
1398+
this.gcDeletesAfterOperation = gcDeletesAfterOperation;
1399+
}
1400+
}
1401+
1402+
protected List<DocHistoryEntry> generateSingleDocHistory(boolean forReplica, VersionType versionType,
1403+
boolean partialOldPrimary, long primaryTerm,
1404+
int minOpCount, int maxOpCount) {
13901405
final int numOfOps = randomIntBetween(minOpCount, maxOpCount);
1391-
final List<Engine.Operation> ops = new ArrayList<>();
1406+
final List<DocHistoryEntry> ops = new ArrayList<>();
13921407
final Term id = newUid("1");
13931408
final int startWithSeqNo;
13941409
if (partialOldPrimary) {
@@ -1435,19 +1450,19 @@ protected List<Engine.Operation> generateSingleDocHistory(boolean forReplica, Ve
14351450
forReplica ? REPLICA : PRIMARY,
14361451
System.currentTimeMillis());
14371452
}
1438-
ops.add(op);
1453+
ops.add(new DocHistoryEntry(op, randomBoolean(), randomBoolean(), rarely()));
14391454
}
14401455
return ops;
14411456
}
14421457

14431458
public void testOutOfOrderDocsOnReplica() throws IOException {
1444-
final List<Engine.Operation> ops = generateSingleDocHistory(true,
1459+
final List<DocHistoryEntry> ops = generateSingleDocHistory(true,
14451460
randomFrom(VersionType.INTERNAL, VersionType.EXTERNAL, VersionType.EXTERNAL_GTE, VersionType.FORCE), false, 2, 2, 20);
14461461
assertOpsOnReplica(ops, replicaEngine, true);
14471462
}
14481463

1449-
private void assertOpsOnReplica(List<Engine.Operation> ops, InternalEngine replicaEngine, boolean shuffleOps) throws IOException {
1450-
final Engine.Operation lastOp = ops.get(ops.size() - 1);
1464+
private void assertOpsOnReplica(List<DocHistoryEntry> ops, InternalEngine replicaEngine, boolean shuffleOps) throws IOException {
1465+
final Engine.Operation lastOp = ops.get(ops.size() - 1).op;
14511466
final String lastFieldValue;
14521467
if (lastOp instanceof Engine.Index) {
14531468
Engine.Index index = (Engine.Index) lastOp;
@@ -1458,15 +1473,16 @@ private void assertOpsOnReplica(List<Engine.Operation> ops, InternalEngine repli
14581473
}
14591474
if (shuffleOps) {
14601475
int firstOpWithSeqNo = 0;
1461-
while (firstOpWithSeqNo < ops.size() && ops.get(firstOpWithSeqNo).seqNo() < 0) {
1476+
while (firstOpWithSeqNo < ops.size() && ops.get(firstOpWithSeqNo).op.seqNo() < 0) {
14621477
firstOpWithSeqNo++;
14631478
}
14641479
// shuffle ops but make sure legacy ops are first
14651480
shuffle(ops.subList(0, firstOpWithSeqNo), random());
14661481
shuffle(ops.subList(firstOpWithSeqNo, ops.size()), random());
14671482
}
14681483
boolean firstOp = true;
1469-
for (Engine.Operation op : ops) {
1484+
for (final DocHistoryEntry docHistoryEntry : ops) {
1485+
final Engine.Operation op = docHistoryEntry.op;
14701486
logger.info("performing [{}], v [{}], seq# [{}], term [{}]",
14711487
op.operationType().name().charAt(0), op.version(), op.seqNo(), op.primaryTerm());
14721488
if (op instanceof Engine.Index) {
@@ -1491,13 +1507,14 @@ private void assertOpsOnReplica(List<Engine.Operation> ops, InternalEngine repli
14911507
assertThat(result.getVersion(), equalTo(op.version()));
14921508
assertThat(result.hasFailure(), equalTo(false));
14931509
}
1494-
if (randomBoolean()) {
1510+
if (docHistoryEntry.refreshAfterOperation) {
14951511
engine.refresh("test");
14961512
}
1497-
if (randomBoolean()) {
1513+
if (docHistoryEntry.flushAfterOperation) {
14981514
engine.flush();
14991515
engine.refresh("test");
15001516
}
1517+
// TODO GC deletes?
15011518
firstOp = false;
15021519
}
15031520

@@ -1512,8 +1529,8 @@ private void assertOpsOnReplica(List<Engine.Operation> ops, InternalEngine repli
15121529
}
15131530

15141531
public void testConcurrentOutOfDocsOnReplica() throws IOException, InterruptedException {
1515-
final List<Engine.Operation> ops = generateSingleDocHistory(true, randomFrom(VersionType.INTERNAL, VersionType.EXTERNAL), false, 2, 100, 300);
1516-
final Engine.Operation lastOp = ops.get(ops.size() - 1);
1532+
final List<DocHistoryEntry> ops = generateSingleDocHistory(true, randomFrom(VersionType.INTERNAL, VersionType.EXTERNAL), false, 2, 100, 300);
1533+
final Engine.Operation lastOp = ops.get(ops.size() - 1).op;
15171534
final String lastFieldValue;
15181535
if (lastOp instanceof Engine.Index) {
15191536
Engine.Index index = (Engine.Index) lastOp;
@@ -1535,7 +1552,7 @@ public void testConcurrentOutOfDocsOnReplica() throws IOException, InterruptedEx
15351552
}
15361553
}
15371554

1538-
private void concurrentlyApplyOps(List<Engine.Operation> ops, InternalEngine engine) throws InterruptedException {
1555+
private void concurrentlyApplyOps(List<DocHistoryEntry> ops, InternalEngine engine) throws InterruptedException {
15391556
Thread[] thread = new Thread[randomIntBetween(3, 5)];
15401557
CountDownLatch startGun = new CountDownLatch(thread.length);
15411558
AtomicInteger offset = new AtomicInteger(-1);
@@ -1550,15 +1567,21 @@ private void concurrentlyApplyOps(List<Engine.Operation> ops, InternalEngine eng
15501567
int docOffset;
15511568
while ((docOffset = offset.incrementAndGet()) < ops.size()) {
15521569
try {
1553-
final Engine.Operation op = ops.get(docOffset);
1570+
final DocHistoryEntry docHistoryEntry = ops.get(docOffset);
1571+
final Engine.Operation op = docHistoryEntry.op;
15541572
if (op instanceof Engine.Index) {
15551573
engine.index((Engine.Index) op);
15561574
} else {
15571575
engine.delete((Engine.Delete) op);
15581576
}
1559-
if ((docOffset + 1) % 4 == 0) {
1577+
if (docHistoryEntry.refreshAfterOperation) {
15601578
engine.refresh("test");
15611579
}
1580+
if (docHistoryEntry.flushAfterOperation) {
1581+
engine.flush();
1582+
engine.refresh("test");
1583+
}
1584+
// TODO GC deletes?
15621585
} catch (IOException e) {
15631586
throw new AssertionError(e);
15641587
}
@@ -1572,11 +1595,11 @@ private void concurrentlyApplyOps(List<Engine.Operation> ops, InternalEngine eng
15721595
}
15731596

15741597
public void testInternalVersioningOnPrimary() throws IOException {
1575-
final List<Engine.Operation> ops = generateSingleDocHistory(false, VersionType.INTERNAL, false, 2, 2, 20);
1598+
final List<DocHistoryEntry> ops = generateSingleDocHistory(false, VersionType.INTERNAL, false, 2, 2, 20);
15761599
assertOpsOnPrimary(ops, Versions.NOT_FOUND, true, engine);
15771600
}
15781601

1579-
private int assertOpsOnPrimary(List<Engine.Operation> ops, long currentOpVersion, boolean docDeleted, InternalEngine engine)
1602+
private int assertOpsOnPrimary(List<DocHistoryEntry> ops, long currentOpVersion, boolean docDeleted, InternalEngine engine)
15801603
throws IOException {
15811604
String lastFieldValue = null;
15821605
int opsPerformed = 0;
@@ -1586,7 +1609,8 @@ private int assertOpsOnPrimary(List<Engine.Operation> ops, long currentOpVersion
15861609
index.getAutoGeneratedIdTimestamp(), index.isRetry());
15871610
BiFunction<Long, Engine.Delete, Engine.Delete> delWithVersion = (version, delete) -> new Engine.Delete(delete.type(), delete.id(),
15881611
delete.uid(), delete.seqNo(), delete.primaryTerm(), version, delete.versionType(), delete.origin(), delete.startTime());
1589-
for (Engine.Operation op : ops) {
1612+
for (final DocHistoryEntry docHistoryEntry : ops) {
1613+
final Engine.Operation op = docHistoryEntry.op;
15901614
final boolean versionConflict = rarely();
15911615
final boolean versionedOp = versionConflict || randomBoolean();
15921616
final long conflictingVersion = docDeleted || randomBoolean() ?
@@ -1650,12 +1674,14 @@ private int assertOpsOnPrimary(List<Engine.Operation> ops, long currentOpVersion
16501674
}
16511675
}
16521676
}
1653-
if (randomBoolean()) {
1677+
1678+
// TODO pure refresh?
1679+
if (docHistoryEntry.flushAfterOperation) {
16541680
engine.flush();
16551681
engine.refresh("test");
16561682
}
16571683

1658-
if (rarely()) {
1684+
if (docHistoryEntry.gcDeletesAfterOperation) {
16591685
// simulate GC deletes
16601686
engine.refresh("gc_simulation", Engine.SearcherScope.INTERNAL);
16611687
engine.clearDeletedTombstones();
@@ -1680,8 +1706,8 @@ public void testNonInternalVersioningOnPrimary() throws IOException {
16801706
final Set<VersionType> nonInternalVersioning = new HashSet<>(Arrays.asList(VersionType.values()));
16811707
nonInternalVersioning.remove(VersionType.INTERNAL);
16821708
final VersionType versionType = randomFrom(nonInternalVersioning);
1683-
final List<Engine.Operation> ops = generateSingleDocHistory(false, versionType, false, 2, 2, 20);
1684-
final Engine.Operation lastOp = ops.get(ops.size() - 1);
1709+
final List<DocHistoryEntry> ops = generateSingleDocHistory(false, versionType, false, 2, 2, 20);
1710+
final Engine.Operation lastOp = ops.get(ops.size() - 1).op;
16851711
final String lastFieldValue;
16861712
if (lastOp instanceof Engine.Index) {
16871713
Engine.Index index = (Engine.Index) lastOp;
@@ -1697,7 +1723,8 @@ public void testNonInternalVersioningOnPrimary() throws IOException {
16971723
long highestOpVersion = Versions.NOT_FOUND;
16981724
long seqNo = -1;
16991725
boolean docDeleted = true;
1700-
for (Engine.Operation op : ops) {
1726+
for (final DocHistoryEntry docHistoryEntry : ops) {
1727+
final Engine.Operation op = docHistoryEntry.op;
17011728
logger.info("performing [{}], v [{}], seq# [{}], term [{}]",
17021729
op.operationType().name().charAt(0), op.version(), op.seqNo(), op.primaryTerm());
17031730
if (op instanceof Engine.Index) {
@@ -1737,13 +1764,14 @@ public void testNonInternalVersioningOnPrimary() throws IOException {
17371764
assertThat(result.getFailure(), instanceOf(VersionConflictEngineException.class));
17381765
}
17391766
}
1740-
if (randomBoolean()) {
1767+
if (docHistoryEntry.refreshAfterOperation) {
17411768
engine.refresh("test");
17421769
}
1743-
if (randomBoolean()) {
1770+
if (docHistoryEntry.flushAfterOperation) {
17441771
engine.flush();
17451772
engine.refresh("test");
17461773
}
1774+
// TODO GC deletes?
17471775
}
17481776

17491777
assertVisibleCount(engine, docDeleted ? 0 : 1);
@@ -1758,9 +1786,9 @@ public void testNonInternalVersioningOnPrimary() throws IOException {
17581786
}
17591787

17601788
public void testVersioningPromotedReplica() throws IOException {
1761-
final List<Engine.Operation> replicaOps = generateSingleDocHistory(true, VersionType.INTERNAL, false, 1, 2, 20);
1762-
List<Engine.Operation> primaryOps = generateSingleDocHistory(false, VersionType.INTERNAL, false, 2, 2, 20);
1763-
Engine.Operation lastReplicaOp = replicaOps.get(replicaOps.size() - 1);
1789+
final List<DocHistoryEntry> replicaOps = generateSingleDocHistory(true, VersionType.INTERNAL, false, 1, 2, 20);
1790+
List<DocHistoryEntry> primaryOps = generateSingleDocHistory(false, VersionType.INTERNAL, false, 2, 2, 20);
1791+
Engine.Operation lastReplicaOp = replicaOps.get(replicaOps.size() - 1).op;
17641792
final boolean deletedOnReplica = lastReplicaOp instanceof Engine.Delete;
17651793
final long finalReplicaVersion = lastReplicaOp.version();
17661794
final long finalReplicaSeqNo = lastReplicaOp.seqNo();
@@ -1779,8 +1807,8 @@ public void testVersioningPromotedReplica() throws IOException {
17791807
}
17801808

17811809
public void testConcurrentExternalVersioningOnPrimary() throws IOException, InterruptedException {
1782-
final List<Engine.Operation> ops = generateSingleDocHistory(false, VersionType.EXTERNAL, false, 2, 100, 300);
1783-
final Engine.Operation lastOp = ops.get(ops.size() - 1);
1810+
final List<DocHistoryEntry> ops = generateSingleDocHistory(false, VersionType.EXTERNAL, false, 2, 100, 300);
1811+
final Engine.Operation lastOp = ops.get(ops.size() - 1).op;
17841812
final String lastFieldValue;
17851813
if (lastOp instanceof Engine.Index) {
17861814
Engine.Index index = (Engine.Index) lastOp;

0 commit comments

Comments
 (0)