Skip to content
Merged
Show file tree
Hide file tree
Changes from 7 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -346,6 +346,8 @@ public void apply(Settings value, Settings current, Settings previous) {
RecoverySettings.INDICES_RECOVERY_MAX_CONCURRENT_REMOTE_STORE_STREAMS_SETTING,
RecoverySettings.INDICES_INTERNAL_REMOTE_UPLOAD_TIMEOUT,
RecoverySettings.INDICES_RECOVERY_CHUNK_SIZE_SETTING,
RecoverySettings.INDICES_TRANSLOG_CONCURRENT_RECOVERY_ENABLE,
RecoverySettings.INDICES_TRANSLOG_CONCURRENT_RECOVERY_BATCH_SIZE,
ThrottlingAllocationDecider.CLUSTER_ROUTING_ALLOCATION_NODE_INITIAL_PRIMARIES_RECOVERIES_SETTING,
ThrottlingAllocationDecider.CLUSTER_ROUTING_ALLOCATION_NODE_INITIAL_REPLICAS_RECOVERIES_SETTING,
ThrottlingAllocationDecider.CLUSTER_ROUTING_ALLOCATION_NODE_CONCURRENT_INCOMING_RECOVERIES_SETTING,
Expand Down
65 changes: 58 additions & 7 deletions server/src/main/java/org/opensearch/index/shard/IndexShard.java
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,7 @@
import org.opensearch.common.util.concurrent.AbstractRunnable;
import org.opensearch.common.util.concurrent.AsyncIOProcessor;
import org.opensearch.common.util.concurrent.BufferedAsyncIOProcessor;
import org.opensearch.common.util.concurrent.OpenSearchThreadPoolExecutor;
import org.opensearch.common.util.concurrent.RunOnce;
import org.opensearch.common.util.concurrent.ThreadContext;
import org.opensearch.common.util.io.IOUtils;
Expand Down Expand Up @@ -240,6 +241,7 @@
import java.util.Set;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicBoolean;
Expand Down Expand Up @@ -5274,14 +5276,63 @@ public void close() throws IOException {
newEngineReference.set(engineFactory.newReadWriteEngine(newEngineConfig(replicationTracker)));
onNewEngine(newEngineReference.get());
}
final TranslogRecoveryRunner translogRunner = (snapshot) -> runTranslogRecovery(
newEngineReference.get(),
snapshot,
Engine.Operation.Origin.LOCAL_RESET,
() -> {
// TODO: add a dedicate recovery stats for the reset translog
final TranslogRecoveryRunner translogRunner = (snapshot) -> {
Engine engine = newEngineReference.get();
assert null != engine;
int totalOperations = snapshot.totalOperations();
long batchSize = recoverySettings.getTranslogConcurrentRecoveryBatchSize();
boolean threadPoolNotBusy = ((OpenSearchThreadPoolExecutor) threadPool.executor(ThreadPool.Names.TRANSLOG_RECOVERY)).getQueue()
Comment thread
guojialiang92 marked this conversation as resolved.
Resolved
.isEmpty();
// When the total totalOperations <= batchSize, there is no need to use concurrent execution.
if (recoverySettings.isTranslogConcurrentRecoveryEnable()
&& indexSettings.isSegRepEnabledOrRemoteNode()
&& totalOperations > batchSize
&& threadPoolNotBusy) {
long localCheckpoint = engine.getProcessedLocalCheckpoint();
final long batches = (totalOperations + batchSize - 1) / batchSize;
List<Translog.Snapshot> translogSnapshotList = new ArrayList<>();
List<Future<Integer>> translogRecoveryFutureList = new ArrayList<>();
try {
// Since the translog does not change at this time, it is safe to re-partition the translog snapshot here.
for (int i = 0; i < batches; i++) {
long start = localCheckpoint + 1 + (long) i * batchSize;
long end = (i == batches - 1) ? Long.MAX_VALUE : start + batchSize - 1;
Translog.Snapshot translogSnapshot = engine.translogManager().newChangesSnapshot(start, end, false);
translogSnapshotList.add(translogSnapshot);
Comment thread
guojialiang92 marked this conversation as resolved.
Resolved
Comment thread
guojialiang92 marked this conversation as resolved.
Resolved
translogRecoveryFutureList.add(
threadPool.executor(ThreadPool.Names.TRANSLOG_RECOVERY)
Comment thread
guojialiang92 marked this conversation as resolved.
Resolved
.submit(() -> runTranslogRecovery(engine, translogSnapshot, Engine.Operation.Origin.LOCAL_RESET, () -> {
// TODO: add a dedicate recovery stats for the reset translog
}))
);
}
Exception exception = null;
int totalRecovered = 0;
for (Future<Integer> translogRecoveryFuture : translogRecoveryFutureList) {
try {
int recoveredOps = translogRecoveryFuture.get();
totalRecovered += recoveredOps;
} catch (Exception e) {
Comment thread
guojialiang92 marked this conversation as resolved.
Resolved
if (exception == null) {
exception = e;
} else {
exception.addSuppressed(e);
}
}
}
if (exception != null) {
throw new IOException("Failed to concurrent recovery translog", exception);
}
return totalRecovered;
} finally {
IOUtils.closeWhileHandlingException(translogSnapshotList);
}
} else {
return runTranslogRecovery(newEngineReference.get(), snapshot, Engine.Operation.Origin.LOCAL_RESET, () -> {
// TODO: add a dedicate recovery stats for the reset translog
});
}
);
};
Comment thread
guojialiang92 marked this conversation as resolved.
Resolved

// When the new engine is created, translogs are synced from remote store onto local. Since remote store is the source
// of truth for translog, we play all translogs that exists locally. Otherwise, the recoverUpto happens upto global checkpoint.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -230,6 +230,21 @@ public class RecoverySettings {
Property.NodeScope
);

public static final Setting<Boolean> INDICES_TRANSLOG_CONCURRENT_RECOVERY_ENABLE = Setting.boolSetting(
"indices.translog_concurrent_recovery.enable",
false,
Property.Dynamic,
Property.NodeScope
);

public static final Setting<Long> INDICES_TRANSLOG_CONCURRENT_RECOVERY_BATCH_SIZE = Setting.longSetting(
"indices.translog_concurrent_recovery.batch_size",
500000,
1,
Property.Dynamic,
Property.NodeScope
);

private volatile ByteSizeValue mergedSegmentWarmerMinSegmentSizeThreshold;
private volatile ByteSizeValue recoveryMaxBytesPerSec;
private volatile ByteSizeValue replicationMaxBytesPerSec;
Expand All @@ -252,6 +267,9 @@ public class RecoverySettings {
private volatile TimeValue internalRemoteUploadTimeout;
private volatile TimeValue mergedSegmentReplicationTimeout;

private volatile boolean isTranslogConcurrentRecoveryEnable;
private volatile long translogConcurrentRecoveryBatchSize;

public RecoverySettings(Settings settings, ClusterSettings clusterSettings) {
this.retryDelayStateSync = INDICES_RECOVERY_RETRY_DELAY_STATE_SYNC_SETTING.get(settings);
this.maxConcurrentFileChunks = INDICES_RECOVERY_MAX_CONCURRENT_FILE_CHUNKS_SETTING.get(settings);
Expand Down Expand Up @@ -286,6 +304,9 @@ public RecoverySettings(Settings settings, ClusterSettings clusterSettings) {
this.internalRemoteUploadTimeout = INDICES_INTERNAL_REMOTE_UPLOAD_TIMEOUT.get(settings);
this.chunkSize = INDICES_RECOVERY_CHUNK_SIZE_SETTING.get(settings);

this.isTranslogConcurrentRecoveryEnable = INDICES_TRANSLOG_CONCURRENT_RECOVERY_ENABLE.get(settings);
this.translogConcurrentRecoveryBatchSize = INDICES_TRANSLOG_CONCURRENT_RECOVERY_BATCH_SIZE.get(settings);

clusterSettings.addSettingsUpdateConsumer(INDICES_RECOVERY_MAX_BYTES_PER_SEC_SETTING, this::setRecoveryMaxBytesPerSec);
clusterSettings.addSettingsUpdateConsumer(INDICES_REPLICATION_MAX_BYTES_PER_SEC_SETTING, this::setReplicationMaxBytesPerSec);
clusterSettings.addSettingsUpdateConsumer(
Expand Down Expand Up @@ -324,6 +345,11 @@ public RecoverySettings(Settings settings, ClusterSettings clusterSettings) {
INDICES_RECOVERY_INTERNAL_ACTION_RETRY_TIMEOUT_SETTING,
this::setInternalActionRetryTimeout
);
clusterSettings.addSettingsUpdateConsumer(INDICES_TRANSLOG_CONCURRENT_RECOVERY_ENABLE, this::setTranslogConcurrentRecoveryEnable);
clusterSettings.addSettingsUpdateConsumer(
INDICES_TRANSLOG_CONCURRENT_RECOVERY_BATCH_SIZE,
this::setTranslogConcurrentRecoveryBatchSize
);
}

private void setMergedSegmentWarmerMinSegmentSizeThreshold(ByteSizeValue value) {
Expand Down Expand Up @@ -493,4 +519,19 @@ public void setIndicesMergedSegmentReplicationWarmerEnabled(boolean mergedSegmen
this.mergedSegmentReplicationWarmerEnabled = mergedSegmentReplicationWarmerEnabled;
}

public boolean isTranslogConcurrentRecoveryEnable() {
return isTranslogConcurrentRecoveryEnable;
}

private void setTranslogConcurrentRecoveryEnable(boolean translogConcurrentRecoveryEnable) {
isTranslogConcurrentRecoveryEnable = translogConcurrentRecoveryEnable;
}

public long getTranslogConcurrentRecoveryBatchSize() {
return translogConcurrentRecoveryBatchSize;
}

private void setTranslogConcurrentRecoveryBatchSize(long translogConcurrentRecoveryBatchSize) {
this.translogConcurrentRecoveryBatchSize = translogConcurrentRecoveryBatchSize;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,7 @@ public class ThreadPool implements ReportingService<ThreadPoolInfo>, Scheduler {
public static class Names {
public static final String SAME = "same";
public static final String GENERIC = "generic";
public static final String TRANSLOG_RECOVERY = "translog_recovery";
@Deprecated
public static final String LISTENER = "listener";
public static final String GET = "get";
Expand Down Expand Up @@ -179,6 +180,7 @@ public static ThreadPoolType fromType(String type) {
HashMap<String, ThreadPoolType> map = new HashMap<>();
map.put(Names.SAME, ThreadPoolType.DIRECT);
map.put(Names.GENERIC, ThreadPoolType.SCALING);
map.put(Names.TRANSLOG_RECOVERY, ThreadPoolType.FIXED);
map.put(Names.LISTENER, ThreadPoolType.FIXED);
map.put(Names.GET, ThreadPoolType.FIXED);
map.put(Names.ANALYZE, ThreadPoolType.FIXED);
Expand Down Expand Up @@ -258,6 +260,7 @@ public ThreadPool(
final int genericThreadPoolMax = boundedBy(4 * allocatedProcessors, 128, 512);
final int snapshotDeletionPoolMax = boundedBy(4 * allocatedProcessors, 64, 256);
builders.put(Names.GENERIC, new ScalingExecutorBuilder(Names.GENERIC, 4, genericThreadPoolMax, TimeValue.timeValueSeconds(30)));
builders.put(Names.TRANSLOG_RECOVERY, new FixedExecutorBuilder(settings, Names.TRANSLOG_RECOVERY, allocatedProcessors, -1));
Comment thread
guojialiang92 marked this conversation as resolved.
Resolved
builders.put(Names.WRITE, new FixedExecutorBuilder(settings, Names.WRITE, allocatedProcessors, 10000));
builders.put(Names.GET, new FixedExecutorBuilder(settings, Names.GET, allocatedProcessors, 1000));
builders.put(Names.ANALYZE, new FixedExecutorBuilder(settings, Names.ANALYZE, 1, 16));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@
import org.opensearch.indices.replication.SegmentReplicationSourceFactory;
import org.opensearch.indices.replication.SegmentReplicationTarget;
import org.opensearch.indices.replication.SegmentReplicationTargetService;
import org.opensearch.indices.replication.checkpoint.MergedSegmentPublisher;
import org.opensearch.indices.replication.checkpoint.ReplicationCheckpoint;
import org.opensearch.threadpool.ThreadPool;
import org.opensearch.transport.TransportService;
Expand Down Expand Up @@ -531,69 +532,94 @@ public void testPrimaryRelocation() throws Exception {
// Todo: Move this test to SegmentReplicationIndexShardTests so that it runs for both node-node & remote store
public void testNRTReplicaPromotedAsPrimary() throws Exception {
try (ReplicationGroup shards = createGroup(2, getIndexSettings(), new NRTReplicationEngineFactory())) {
shards.startAll();
IndexShard oldPrimary = shards.getPrimary();
final IndexShard nextPrimary = shards.getReplicas().get(0);
final IndexShard replica = shards.getReplicas().get(1);

// 1. Create ops that are in the index and xlog of both shards but not yet part of a commit point.
final int numDocs = shards.indexDocs(randomInt(10));

// refresh and copy the segments over.
oldPrimary.refresh("Test");
replicateSegments(oldPrimary, shards.getReplicas());

// at this point both shards should have numDocs persisted and searchable.
assertDocCounts(oldPrimary, numDocs, numDocs);
for (IndexShard shard : shards.getReplicas()) {
assertDocCounts(shard, numDocs, numDocs);
}
assertEqualTranslogOperations(shards, oldPrimary);

// 2. Create ops that are in the replica's xlog, not in the index.
// index some more into both but don't replicate. replica will have only numDocs searchable, but should have totalDocs
// persisted.
final int additonalDocs = shards.indexDocs(randomInt(10));
final int totalDocs = numDocs + additonalDocs;
doPrimaryPromotion(shards, randomInt(10), randomInt(10));
}
}
Comment thread
guojialiang92 marked this conversation as resolved.
Resolved

assertDocCounts(oldPrimary, totalDocs, totalDocs);
assertEqualTranslogOperations(shards, oldPrimary);
for (IndexShard shard : shards.getReplicas()) {
assertDocCounts(shard, totalDocs, numDocs);
}
assertEquals(totalDocs, oldPrimary.translogStats().estimatedNumberOfOperations());
assertEquals(totalDocs, oldPrimary.translogStats().estimatedNumberOfOperations());
assertEquals(totalDocs, nextPrimary.translogStats().estimatedNumberOfOperations());
assertEquals(totalDocs, replica.translogStats().estimatedNumberOfOperations());
assertEquals(totalDocs, nextPrimary.translogStats().getUncommittedOperations());
assertEquals(totalDocs, replica.translogStats().getUncommittedOperations());

// promote the replica
shards.syncGlobalCheckpoint();
shards.promoteReplicaToPrimary(nextPrimary);
public void testPrimaryPromotionWithConcurrentTranslogRecovery() throws Exception {
final RecoverySettings recoverySettings = new RecoverySettings(
Settings.builder()
.put(RecoverySettings.INDICES_TRANSLOG_CONCURRENT_RECOVERY_ENABLE.getKey(), true)
.put(RecoverySettings.INDICES_TRANSLOG_CONCURRENT_RECOVERY_BATCH_SIZE.getKey(), 10)
.build(),
new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)
);
try (
ReplicationGroup shards = createGroup(
2,
getIndexSettings(),
indexMapping,
new NRTReplicationEngineFactory(),
recoverySettings,
MergedSegmentPublisher.EMPTY
)
) {
doPrimaryPromotion(shards, randomInt(10), randomIntBetween(200, 300));
}
}
Comment thread
guojialiang92 marked this conversation as resolved.
Resolved

// close and start the oldPrimary as a replica.
oldPrimary.close("demoted", false, false);
oldPrimary.store().close();
oldPrimary = shards.addReplicaWithExistingPath(oldPrimary.shardPath(), oldPrimary.routingEntry().currentNodeId());
shards.recoverReplica(oldPrimary);
private void doPrimaryPromotion(ReplicationGroup shards, int numDocsInFirstBatch, int numDocsInSecondBatch) throws Exception {
shards.startAll();
IndexShard oldPrimary = shards.getPrimary();
final IndexShard nextPrimary = shards.getReplicas().get(0);
final IndexShard replica = shards.getReplicas().get(1);

assertEquals(NRTReplicationEngine.class, oldPrimary.getEngine().getClass());
assertEquals(InternalEngine.class, nextPrimary.getEngine().getClass());
assertDocCounts(nextPrimary, totalDocs, totalDocs);
assertEquals(0, nextPrimary.translogStats().estimatedNumberOfOperations());
// 1. Create ops that are in the index and xlog of both shards but not yet part of a commit point.
final int numDocs = shards.indexDocs(numDocsInFirstBatch);

// refresh and push segments to our other replica.
nextPrimary.refresh("test");
replicateSegments(nextPrimary, asList(replica));
// refresh and copy the segments over.
oldPrimary.refresh("Test");
replicateSegments(oldPrimary, shards.getReplicas());

for (IndexShard shard : shards) {
assertConsistentHistoryBetweenTranslogAndLucene(shard);
}
final List<DocIdSeqNoAndSource> docsAfterRecovery = getDocIdAndSeqNos(shards.getPrimary());
for (IndexShard shard : shards.getReplicas()) {
assertThat(shard.routingEntry().toString(), getDocIdAndSeqNos(shard), equalTo(docsAfterRecovery));
}
// at this point both shards should have numDocs persisted and searchable.
assertDocCounts(oldPrimary, numDocs, numDocs);
for (IndexShard shard : shards.getReplicas()) {
assertDocCounts(shard, numDocs, numDocs);
}
assertEqualTranslogOperations(shards, oldPrimary);

// 2. Create ops that are in the replica's xlog, not in the index.
// index some more into both but don't replicate. replica will have only numDocs searchable, but should have totalDocs
// persisted.
final int additionalDocs = shards.indexDocs(numDocsInSecondBatch);
final int totalDocs = numDocs + additionalDocs;

assertDocCounts(oldPrimary, totalDocs, totalDocs);
assertEqualTranslogOperations(shards, oldPrimary);
for (IndexShard shard : shards.getReplicas()) {
assertDocCounts(shard, totalDocs, numDocs);
}
assertEquals(totalDocs, oldPrimary.translogStats().estimatedNumberOfOperations());
assertEquals(totalDocs, nextPrimary.translogStats().estimatedNumberOfOperations());
assertEquals(totalDocs, replica.translogStats().estimatedNumberOfOperations());
assertEquals(totalDocs, nextPrimary.translogStats().getUncommittedOperations());
assertEquals(totalDocs, replica.translogStats().getUncommittedOperations());

// promote the replica
shards.syncGlobalCheckpoint();
shards.promoteReplicaToPrimary(nextPrimary);

// close and start the oldPrimary as a replica.
oldPrimary.close("demoted", false, false);
oldPrimary.store().close();
oldPrimary = shards.addReplicaWithExistingPath(oldPrimary.shardPath(), oldPrimary.routingEntry().currentNodeId());
shards.recoverReplica(oldPrimary);

assertEquals(NRTReplicationEngine.class, oldPrimary.getEngine().getClass());
assertEquals(InternalEngine.class, nextPrimary.getEngine().getClass());
assertDocCounts(nextPrimary, totalDocs, totalDocs);
assertEquals(0, nextPrimary.translogStats().estimatedNumberOfOperations());

// refresh and push segments to our other replica.
nextPrimary.refresh("test");
replicateSegments(nextPrimary, asList(replica));

for (IndexShard shard : shards) {
assertConsistentHistoryBetweenTranslogAndLucene(shard);
}
final List<DocIdSeqNoAndSource> docsAfterRecovery = getDocIdAndSeqNos(shards.getPrimary());
for (IndexShard shard : shards.getReplicas()) {
assertThat(shard.routingEntry().toString(), getDocIdAndSeqNos(shard), equalTo(docsAfterRecovery));
}
}

Expand Down Expand Up @@ -712,5 +738,4 @@ public void testSegmentReplication_Index_Update_Delete() throws Exception {
}
}
}

}
Loading
Loading