diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/cleanup/TransportCleanupRepositoryAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/cleanup/TransportCleanupRepositoryAction.java index 738b261d7fd8e..8892a60d4c6f4 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/cleanup/TransportCleanupRepositoryAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/cleanup/TransportCleanupRepositoryAction.java @@ -42,7 +42,6 @@ import org.elasticsearch.repositories.Repository; import org.elasticsearch.repositories.RepositoryCleanupResult; import org.elasticsearch.repositories.blobstore.BlobStoreRepository; -import org.elasticsearch.snapshots.SnapshotsService; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; @@ -203,7 +202,6 @@ public void clusterStateProcessed(String source, ClusterState oldState, ClusterS threadPool.executor(ThreadPool.Names.SNAPSHOT).execute(ActionRunnable.wrap(listener, l -> blobStoreRepository.cleanup( repositoryStateId, - newState.nodes().getMinNodeVersion().onOrAfter(SnapshotsService.SHARD_GEN_IN_REPO_DATA_VERSION), ActionListener.wrap(result -> after(null, result), e -> after(e, null))))); } diff --git a/server/src/main/java/org/elasticsearch/cluster/SnapshotsInProgress.java b/server/src/main/java/org/elasticsearch/cluster/SnapshotsInProgress.java index b40d771c64739..dfb41c2e92e8b 100644 --- a/server/src/main/java/org/elasticsearch/cluster/SnapshotsInProgress.java +++ b/server/src/main/java/org/elasticsearch/cluster/SnapshotsInProgress.java @@ -92,13 +92,12 @@ public static class Entry implements ToXContent { private final long startTime; private final long repositoryStateId; // see #useShardGenerations - private final boolean useShardGenerations; @Nullable private final Map userMetadata; @Nullable private final String failure; public Entry(Snapshot snapshot, boolean includeGlobalState, boolean partial, State state, List indices, long startTime, long repositoryStateId, ImmutableOpenMap shards, - String failure, Map userMetadata, boolean useShardGenerations) { + String failure, Map userMetadata) { this.state = state; this.snapshot = snapshot; this.includeGlobalState = includeGlobalState; @@ -116,7 +115,6 @@ public Entry(Snapshot snapshot, boolean includeGlobalState, boolean partial, Sta this.repositoryStateId = repositoryStateId; this.failure = failure; this.userMetadata = userMetadata; - this.useShardGenerations = useShardGenerations; } private static boolean assertShardsConsistent(State state, List indices, @@ -134,19 +132,18 @@ private static boolean assertShardsConsistent(State state, List indices public Entry(Snapshot snapshot, boolean includeGlobalState, boolean partial, State state, List indices, long startTime, long repositoryStateId, ImmutableOpenMap shards, - Map userMetadata, boolean useShardGenerations) { - this(snapshot, includeGlobalState, partial, state, indices, startTime, repositoryStateId, shards, null, userMetadata, - useShardGenerations); + Map userMetadata) { + this(snapshot, includeGlobalState, partial, state, indices, startTime, repositoryStateId, shards, null, userMetadata); } public Entry(Entry entry, State state, ImmutableOpenMap shards) { this(entry.snapshot, entry.includeGlobalState, entry.partial, state, entry.indices, entry.startTime, - entry.repositoryStateId, shards, entry.failure, entry.userMetadata, entry.useShardGenerations); + entry.repositoryStateId, shards, entry.failure, entry.userMetadata); } public Entry(Entry entry, State state, ImmutableOpenMap shards, String failure) { this(entry.snapshot, entry.includeGlobalState, entry.partial, state, entry.indices, entry.startTime, - entry.repositoryStateId, shards, failure, entry.userMetadata, entry.useShardGenerations); + entry.repositoryStateId, shards, failure, entry.userMetadata); } public Entry(Entry entry, ImmutableOpenMap shards) { @@ -197,16 +194,6 @@ public String failure() { return failure; } - /** - * Whether to write to the repository in a format only understood by versions newer than - * {@link SnapshotsService#SHARD_GEN_IN_REPO_DATA_VERSION}. - * - * @return true if writing to repository in new format - */ - public boolean useShardGenerations() { - return useShardGenerations; - } - @Override public boolean equals(Object o) { if (this == o) return true; @@ -222,7 +209,6 @@ public boolean equals(Object o) { if (!snapshot.equals(entry.snapshot)) return false; if (state != entry.state) return false; if (repositoryStateId != entry.repositoryStateId) return false; - if (useShardGenerations != entry.useShardGenerations) return false; return true; } @@ -237,7 +223,6 @@ public int hashCode() { result = 31 * result + indices.hashCode(); result = 31 * result + Long.hashCode(startTime); result = 31 * result + Long.hashCode(repositoryStateId); - result = 31 * result + (useShardGenerations ? 1 : 0); return result; } @@ -520,11 +505,9 @@ public SnapshotsInProgress(StreamInput in) throws IOException { if (in.getVersion().onOrAfter(METADATA_FIELD_INTRODUCED)) { userMetadata = in.readMap(); } - final boolean useShardGenerations; - if (in.getVersion().onOrAfter(SnapshotsService.SHARD_GEN_IN_REPO_DATA_VERSION)) { - useShardGenerations = in.readBoolean(); - } else { - useShardGenerations = false; + final Version version = in.getVersion(); + if (version.before(Version.V_8_0_0) && version.onOrAfter(SnapshotsService.SHARD_GEN_IN_REPO_DATA_VERSION)) { + in.readBoolean(); } entries[i] = new Entry(snapshot, includeGlobalState, @@ -535,8 +518,7 @@ public SnapshotsInProgress(StreamInput in) throws IOException { repositoryStateId, builder.build(), failure, - userMetadata, - useShardGenerations + userMetadata ); } this.entries = Arrays.asList(entries); @@ -562,11 +544,13 @@ public void writeTo(StreamOutput out) throws IOException { } out.writeLong(entry.repositoryStateId); out.writeOptionalString(entry.failure); - if (out.getVersion().onOrAfter(METADATA_FIELD_INTRODUCED)) { + final Version version = out.getVersion(); + if (version.onOrAfter(METADATA_FIELD_INTRODUCED)) { out.writeMap(entry.userMetadata); } - if (out.getVersion().onOrAfter(SnapshotsService.SHARD_GEN_IN_REPO_DATA_VERSION)) { - out.writeBoolean(entry.useShardGenerations); + if (version.before(Version.V_8_0_0) && version.onOrAfter(SnapshotsService.SHARD_GEN_IN_REPO_DATA_VERSION)) { + // BwC logic: we always write shard generations in all versions that we support rolling upgrades from + out.writeBoolean(true); } } } diff --git a/server/src/main/java/org/elasticsearch/repositories/FilterRepository.java b/server/src/main/java/org/elasticsearch/repositories/FilterRepository.java index 59092cc573bf2..cfbfedde6944c 100644 --- a/server/src/main/java/org/elasticsearch/repositories/FilterRepository.java +++ b/server/src/main/java/org/elasticsearch/repositories/FilterRepository.java @@ -76,14 +76,14 @@ public RepositoryData getRepositoryData() { public void finalizeSnapshot(SnapshotId snapshotId, ShardGenerations shardGenerations, long startTime, String failure, int totalShards, List shardFailures, long repositoryStateId, boolean includeGlobalState, MetaData metaData, Map userMetadata, - boolean writeShardGens, ActionListener listener) { + ActionListener listener) { in.finalizeSnapshot(snapshotId, shardGenerations, startTime, failure, totalShards, shardFailures, repositoryStateId, - includeGlobalState, metaData, userMetadata, writeShardGens, listener); + includeGlobalState, metaData, userMetadata, listener); } @Override - public void deleteSnapshot(SnapshotId snapshotId, long repositoryStateId, boolean writeShardGens, ActionListener listener) { - in.deleteSnapshot(snapshotId, repositoryStateId, writeShardGens, listener); + public void deleteSnapshot(SnapshotId snapshotId, long repositoryStateId, ActionListener listener) { + in.deleteSnapshot(snapshotId, repositoryStateId, listener); } @Override @@ -118,9 +118,8 @@ public boolean isReadOnly() { @Override public void snapshotShard(Store store, MapperService mapperService, SnapshotId snapshotId, IndexId indexId, - IndexCommit snapshotIndexCommit, IndexShardSnapshotStatus snapshotStatus, boolean writeShardGens, - ActionListener listener) { - in.snapshotShard(store, mapperService, snapshotId, indexId, snapshotIndexCommit, snapshotStatus, writeShardGens, listener); + IndexCommit snapshotIndexCommit, IndexShardSnapshotStatus snapshotStatus, ActionListener listener) { + in.snapshotShard(store, mapperService, snapshotId, indexId, snapshotIndexCommit, snapshotStatus, listener); } @Override public void restoreShard(Store store, SnapshotId snapshotId, IndexId indexId, ShardId snapshotShardId, RecoveryState recoveryState) { diff --git a/server/src/main/java/org/elasticsearch/repositories/Repository.java b/server/src/main/java/org/elasticsearch/repositories/Repository.java index 31687fbe0f8ec..905c78baff0bd 100644 --- a/server/src/main/java/org/elasticsearch/repositories/Repository.java +++ b/server/src/main/java/org/elasticsearch/repositories/Repository.java @@ -122,23 +122,21 @@ default Repository create(RepositoryMetaData metaData, Function shardFailures, long repositoryStateId, boolean includeGlobalState, MetaData clusterMetaData, Map userMetadata, - boolean writeShardGens, ActionListener listener); + ActionListener listener); /** * Deletes snapshot * * @param snapshotId snapshot id * @param repositoryStateId the unique id identifying the state of the repository when the snapshot deletion began - * @param writeShardGens if shard generations should be written to the repository * @param listener completion listener */ - void deleteSnapshot(SnapshotId snapshotId, long repositoryStateId, boolean writeShardGens, ActionListener listener); + void deleteSnapshot(SnapshotId snapshotId, long repositoryStateId, ActionListener listener); /** * Returns snapshot throttle time in nanoseconds @@ -200,7 +198,7 @@ void finalizeSnapshot(SnapshotId snapshotId, ShardGenerations shardGenerations, * @param listener listener invoked on completion */ void snapshotShard(Store store, MapperService mapperService, SnapshotId snapshotId, IndexId indexId, IndexCommit snapshotIndexCommit, - IndexShardSnapshotStatus snapshotStatus, boolean writeShardGens, ActionListener listener); + IndexShardSnapshotStatus snapshotStatus, ActionListener listener); /** * Restores snapshot of the shard. diff --git a/server/src/main/java/org/elasticsearch/repositories/RepositoryData.java b/server/src/main/java/org/elasticsearch/repositories/RepositoryData.java index 8589d2efdfc6d..ec951765305ae 100644 --- a/server/src/main/java/org/elasticsearch/repositories/RepositoryData.java +++ b/server/src/main/java/org/elasticsearch/repositories/RepositoryData.java @@ -314,10 +314,7 @@ public List resolveNewIndices(final List indicesToResolve) { /** * Writes the snapshots metadata and the related indices metadata to x-content. */ - public XContentBuilder snapshotsToXContent(final XContentBuilder builder, final boolean shouldWriteShardGens) throws IOException { - assert shouldWriteShardGens || shardGenerations.indices().isEmpty() : - "Should not build shard generations in BwC mode but saw generations [" + shardGenerations + "]"; - + public XContentBuilder snapshotsToXContent(final XContentBuilder builder) throws IOException { builder.startObject(); // write the snapshots list builder.startArray(SNAPSHOTS); @@ -343,13 +340,11 @@ public XContentBuilder snapshotsToXContent(final XContentBuilder builder, final builder.value(snapshotId.getUUID()); } builder.endArray(); - if (shouldWriteShardGens) { - builder.startArray(SHARD_GENERATIONS); - for (String gen : shardGenerations.getGens(indexId)) { - builder.value(gen); - } - builder.endArray(); + builder.startArray(SHARD_GENERATIONS); + for (String gen : shardGenerations.getGens(indexId)) { + builder.value(gen); } + builder.endArray(); builder.endObject(); } builder.endObject(); diff --git a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java index e4626578864a8..27f798917f2bf 100644 --- a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java +++ b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java @@ -361,7 +361,7 @@ public RepositoryMetaData getMetadata() { } @Override - public void deleteSnapshot(SnapshotId snapshotId, long repositoryStateId, boolean writeShardGens, ActionListener listener) { + public void deleteSnapshot(SnapshotId snapshotId, long repositoryStateId, ActionListener listener) { if (isReadOnly()) { listener.onFailure(new RepositoryException(metadata.name(), "cannot delete snapshot from a readonly repository")); } else { @@ -371,7 +371,7 @@ public void deleteSnapshot(SnapshotId snapshotId, long repositoryStateId, boolea // Cache the indices that were found before writing out the new index-N blob so that a stuck master will never // delete an index that was created by another master node after writing this index-N blob. final Map foundIndices = blobStore().blobContainer(indicesPath()).children(); - doDeleteShardSnapshots(snapshotId, repositoryStateId, foundIndices, rootBlobs, repositoryData, writeShardGens, listener); + doDeleteShardSnapshots(snapshotId, repositoryStateId, foundIndices, rootBlobs, repositoryData, listener); } catch (Exception ex) { listener.onFailure(new RepositoryException(metadata.name(), "failed to delete snapshot [" + snapshotId + "]", ex)); } @@ -392,51 +392,37 @@ public void deleteSnapshot(SnapshotId snapshotId, long repositoryStateId, boolea * @param listener Listener to invoke once finished */ private void doDeleteShardSnapshots(SnapshotId snapshotId, long repositoryStateId, Map foundIndices, - Map rootBlobs, RepositoryData repositoryData, boolean writeShardGens, - ActionListener listener) throws IOException { - - if (writeShardGens) { - // First write the new shard state metadata (with the removed snapshot) and compute deletion targets - final StepListener> writeShardMetaDataAndComputeDeletesStep = new StepListener<>(); - writeUpdatedShardMetaDataAndComputeDeletes(snapshotId, repositoryData, true, writeShardMetaDataAndComputeDeletesStep); - // Once we have put the new shard-level metadata into place, we can update the repository metadata as follows: - // 1. Remove the snapshot from the list of existing snapshots - // 2. Update the index shard generations of all updated shard folders - // - // Note: If we fail updating any of the individual shard paths, none of them are changed since the newly created - // index-${gen_uuid} will not be referenced by the existing RepositoryData and new RepositoryData is only - // written if all shard paths have been successfully updated. - final StepListener writeUpdatedRepoDataStep = new StepListener<>(); - writeShardMetaDataAndComputeDeletesStep.whenComplete(deleteResults -> { - final ShardGenerations.Builder builder = ShardGenerations.builder(); - for (ShardSnapshotMetaDeleteResult newGen : deleteResults) { - builder.put(newGen.indexId, newGen.shardId, newGen.newGeneration); - } - final RepositoryData updatedRepoData = repositoryData.removeSnapshot(snapshotId, builder.build()); - writeIndexGen(updatedRepoData, repositoryStateId, true); - writeUpdatedRepoDataStep.onResponse(updatedRepoData); - }, listener::onFailure); - // Once we have updated the repository, run the clean-ups - writeUpdatedRepoDataStep.whenComplete(updatedRepoData -> { - // Run unreferenced blobs cleanup in parallel to shard-level snapshot deletion - final ActionListener afterCleanupsListener = - new GroupedActionListener<>(ActionListener.wrap(() -> listener.onResponse(null)), 2); - asyncCleanupUnlinkedRootAndIndicesBlobs(foundIndices, rootBlobs, updatedRepoData, afterCleanupsListener); - asyncCleanupUnlinkedShardLevelBlobs(snapshotId, writeShardMetaDataAndComputeDeletesStep.result(), afterCleanupsListener); - }, listener::onFailure); - } else { - // Write the new repository data first (with the removed snapshot), using no shard generations - final RepositoryData updatedRepoData = repositoryData.removeSnapshot(snapshotId, ShardGenerations.EMPTY); - writeIndexGen(updatedRepoData, repositoryStateId, false); + Map rootBlobs, RepositoryData repositoryData, + ActionListener listener) { + + // First write the new shard state metadata (with the removed snapshot) and compute deletion targets + final StepListener> writeShardMetaDataAndComputeDeletesStep = new StepListener<>(); + writeUpdatedShardMetaDataAndComputeDeletes(snapshotId, repositoryData, writeShardMetaDataAndComputeDeletesStep); + // Once we have put the new shard-level metadata into place, we can update the repository metadata as follows: + // 1. Remove the snapshot from the list of existing snapshots + // 2. Update the index shard generations of all updated shard folders + // + // Note: If we fail updating any of the individual shard paths, none of them are changed since the newly created + // index-${gen_uuid} will not be referenced by the existing RepositoryData and new RepositoryData is only + // written if all shard paths have been successfully updated. + final StepListener writeUpdatedRepoDataStep = new StepListener<>(); + writeShardMetaDataAndComputeDeletesStep.whenComplete(deleteResults -> { + final ShardGenerations.Builder builder = ShardGenerations.builder(); + for (ShardSnapshotMetaDeleteResult newGen : deleteResults) { + builder.put(newGen.indexId, newGen.shardId, newGen.newGeneration); + } + final RepositoryData updatedRepoData = repositoryData.removeSnapshot(snapshotId, builder.build()); + writeIndexGen(updatedRepoData, repositoryStateId); + writeUpdatedRepoDataStep.onResponse(updatedRepoData); + }, listener::onFailure); + // Once we have updated the repository, run the clean-ups + writeUpdatedRepoDataStep.whenComplete(updatedRepoData -> { // Run unreferenced blobs cleanup in parallel to shard-level snapshot deletion final ActionListener afterCleanupsListener = new GroupedActionListener<>(ActionListener.wrap(() -> listener.onResponse(null)), 2); asyncCleanupUnlinkedRootAndIndicesBlobs(foundIndices, rootBlobs, updatedRepoData, afterCleanupsListener); - final StepListener> writeMetaAndComputeDeletesStep = new StepListener<>(); - writeUpdatedShardMetaDataAndComputeDeletes(snapshotId, repositoryData, false, writeMetaAndComputeDeletesStep); - writeMetaAndComputeDeletesStep.whenComplete(deleteResults -> - asyncCleanupUnlinkedShardLevelBlobs(snapshotId, deleteResults, afterCleanupsListener), afterCleanupsListener::onFailure); - } + asyncCleanupUnlinkedShardLevelBlobs(snapshotId, writeShardMetaDataAndComputeDeletesStep.result(), afterCleanupsListener); + }, listener::onFailure); } private void asyncCleanupUnlinkedRootAndIndicesBlobs(Map foundIndices, Map rootBlobs, @@ -465,7 +451,7 @@ private void asyncCleanupUnlinkedShardLevelBlobs(SnapshotId snapshotId, Collecti // updates the shard state metadata for shards of a snapshot that is to be deleted. Also computes the files to be cleaned up. private void writeUpdatedShardMetaDataAndComputeDeletes(SnapshotId snapshotId, RepositoryData oldRepositoryData, - boolean useUUIDs, ActionListener> onAllShardsCompleted) { + ActionListener> onAllShardsCompleted) { final Executor executor = threadPool.executor(ThreadPool.Names.SNAPSHOT); final List indices = oldRepositoryData.indicesToUpdateAfterRemovingSnapshot(snapshotId); @@ -510,20 +496,10 @@ private void writeUpdatedShardMetaDataAndComputeDeletes(SnapshotId snapshotId, R protected void doRun() throws Exception { final BlobContainer shardContainer = shardContainer(indexId, shard); final Set blobs = getShardBlobs(shard, shardContainer); - final BlobStoreIndexShardSnapshots blobStoreIndexShardSnapshots; - final String newGen; - if (useUUIDs) { - newGen = UUIDs.randomBase64UUID(); - blobStoreIndexShardSnapshots = buildBlobStoreIndexShardSnapshots(blobs, shardContainer, - oldRepositoryData.shardGenerations().getShardGen(indexId, shard.getId())).v1(); - } else { - Tuple tuple = - buildBlobStoreIndexShardSnapshots(blobs, shardContainer); - newGen = Long.toString(tuple.v2() + 1); - blobStoreIndexShardSnapshots = tuple.v1(); - } + final BlobStoreIndexShardSnapshots blobStoreIndexShardSnapshots = buildBlobStoreIndexShardSnapshots(blobs, + shardContainer, oldRepositoryData.shardGenerations().getShardGen(indexId, shard.getId())).v1(); allShardsListener.onResponse(deleteFromShardSnapshotMeta(survivingSnapshots, indexId, shard, snapshotId, - shardContainer, blobs, blobStoreIndexShardSnapshots, newGen)); + shardContainer, blobs, blobStoreIndexShardSnapshots)); } @Override @@ -597,10 +573,9 @@ private void cleanupStaleBlobs(Map foundIndices, MapDeleting unreferenced root level blobs {@link #cleanupStaleRootFiles} * * @param repositoryStateId Current repository state id - * @param writeShardGens If shard generations should be written to the repository * @param listener Listener to complete when done */ - public void cleanup(long repositoryStateId, boolean writeShardGens, ActionListener listener) { + public void cleanup(long repositoryStateId, ActionListener listener) { try { if (isReadOnly()) { throw new RepositoryException(metadata.name(), "cannot run cleanup on readonly repository"); @@ -622,7 +597,7 @@ public void cleanup(long repositoryStateId, boolean writeShardGens, ActionListen listener.onResponse(new RepositoryCleanupResult(DeleteResult.ZERO)); } else { // write new index-N blob to ensure concurrent operations will fail - writeIndexGen(repositoryData, repositoryStateId, writeShardGens); + writeIndexGen(repositoryData, repositoryStateId); cleanupStaleBlobs(foundIndices, rootBlobs, repositoryData, ActionListener.map(listener, RepositoryCleanupResult::new)); } } catch (Exception e) { @@ -717,7 +692,6 @@ public void finalizeSnapshot(final SnapshotId snapshotId, final boolean includeGlobalState, final MetaData clusterMetaData, final Map userMetadata, - boolean writeShardGens, final ActionListener listener) { final Collection indices = shardGenerations.indices(); @@ -732,10 +706,8 @@ public void finalizeSnapshot(final SnapshotId snapshotId, final RepositoryData existingRepositoryData = getRepositoryData(); final RepositoryData updatedRepositoryData = existingRepositoryData.addSnapshot(snapshotId, snapshotInfo.state(), shardGenerations); - writeIndexGen(updatedRepositoryData, repositoryStateId, writeShardGens); - if (writeShardGens) { - cleanupOldShardGens(existingRepositoryData, updatedRepositoryData); - } + writeIndexGen(updatedRepositoryData, repositoryStateId); + cleanupOldShardGens(existingRepositoryData, updatedRepositoryData); listener.onResponse(snapshotInfo); }, e -> listener.onFailure(new SnapshotException(metadata.name(), snapshotId, "failed to update snapshot in repository", e))), @@ -934,8 +906,7 @@ public boolean isReadOnly() { return readOnly; } - protected void writeIndexGen(final RepositoryData repositoryData, final long expectedGen, - final boolean writeShardGens) throws IOException { + protected void writeIndexGen(final RepositoryData repositoryData, final long expectedGen) throws IOException { assert isReadOnly() == false; // can not write to a read only repository final long currentGen = repositoryData.getGenId(); if (currentGen != expectedGen) { @@ -950,7 +921,7 @@ protected void writeIndexGen(final RepositoryData repositoryData, final long exp final String indexBlob = INDEX_FILE_PREFIX + Long.toString(newGen); logger.debug("Repository [{}] writing new index generational blob [{}]", metadata.name(), indexBlob); writeAtomic(indexBlob, - BytesReference.bytes(repositoryData.snapshotsToXContent(XContentFactory.jsonBuilder(), writeShardGens)), true); + BytesReference.bytes(repositoryData.snapshotsToXContent(XContentFactory.jsonBuilder())), true); // write the current generation to the index-latest file final BytesReference genBytes; try (BytesStreamOutput bStream = new BytesStreamOutput()) { @@ -1038,8 +1009,7 @@ private void writeAtomic(final String blobName, final BytesReference bytesRef, b @Override public void snapshotShard(Store store, MapperService mapperService, SnapshotId snapshotId, IndexId indexId, - IndexCommit snapshotIndexCommit, IndexShardSnapshotStatus snapshotStatus, boolean writeShardGens, - ActionListener listener) { + IndexCommit snapshotIndexCommit, IndexShardSnapshotStatus snapshotStatus, ActionListener listener) { final ShardId shardId = store.shardId(); final long startTime = threadPool.absoluteTimeInMillis(); final ActionListener snapshotDoneListener = ActionListener.wrap(listener::onResponse, e -> { @@ -1063,7 +1033,6 @@ public void snapshotShard(Store store, MapperService mapperService, SnapshotId s Tuple tuple = buildBlobStoreIndexShardSnapshots(blobs, shardContainer, generation); BlobStoreIndexShardSnapshots snapshots = tuple.v1(); - String fileListGeneration = tuple.v2(); if (snapshots.snapshots().stream().anyMatch(sf -> sf.snapshot().equals(snapshotId.getName()))) { throw new IndexShardSnapshotFailedException(shardId, @@ -1161,20 +1130,7 @@ public void snapshotShard(Store store, MapperService mapperService, SnapshotId s for (SnapshotFiles point : snapshots) { newSnapshotsList.add(point); } - final List blobsToDelete; - final String indexGeneration; - if (writeShardGens) { - indexGeneration = UUIDs.randomBase64UUID(); - blobsToDelete = Collections.emptyList(); - } else { - indexGeneration = Long.toString(Long.parseLong(fileListGeneration) + 1); - // Delete all previous index-N blobs - blobsToDelete = blobs.stream().filter(blob -> blob.startsWith(SNAPSHOT_INDEX_PREFIX)).collect(Collectors.toList()); - assert blobsToDelete.stream().mapToLong(b -> Long.parseLong(b.replaceFirst(SNAPSHOT_INDEX_PREFIX, ""))) - .max().orElse(-1L) < Long.parseLong(indexGeneration) - : "Tried to delete an index-N blob newer than the current generation [" + indexGeneration - + "] when deleting index-N blobs " + blobsToDelete; - } + final String indexGeneration = UUIDs.randomBase64UUID(); try { writeShardIndexBlob(shardContainer, indexGeneration, new BlobStoreIndexShardSnapshots(newSnapshotsList)); } catch (IOException e) { @@ -1182,14 +1138,6 @@ public void snapshotShard(Store store, MapperService mapperService, SnapshotId s "Failed to finalize snapshot creation [" + snapshotId + "] with shard index [" + indexShardSnapshotsFormat.blobName(indexGeneration) + "]", e); } - if (writeShardGens == false) { - try { - shardContainer.deleteBlobsIgnoringIfNotExists(blobsToDelete); - } catch (IOException e) { - logger.warn(() -> new ParameterizedMessage("[{}][{}] failed to delete old index-N blobs during finalization", - snapshotId, shardId), e); - } - } snapshotStatus.moveToDone(threadPool.absoluteTimeInMillis(), indexGeneration); snapshotDoneListener.onResponse(indexGeneration); }, snapshotDoneListener::onFailure); @@ -1350,7 +1298,7 @@ public String toString() { private ShardSnapshotMetaDeleteResult deleteFromShardSnapshotMeta(Set survivingSnapshots, IndexId indexId, ShardId snapshotShardId, SnapshotId snapshotId, BlobContainer shardContainer, Set blobs, - BlobStoreIndexShardSnapshots snapshots, String indexGeneration) { + BlobStoreIndexShardSnapshots snapshots) { // Build a list of snapshots that should be preserved List newSnapshotsList = new ArrayList<>(); final Set survivingSnapshotNames = survivingSnapshots.stream().map(SnapshotId::getName).collect(Collectors.toSet()); @@ -1359,6 +1307,7 @@ private ShardSnapshotMetaDeleteResult deleteFromShardSnapshotMeta(Set() { + snapshot(shardId, snapshot, indexId, snapshotStatus, new ActionListener<>() { @Override public void onResponse(String newGeneration) { assert newGeneration != null; @@ -313,7 +311,7 @@ public void onFailure(Exception e) { * @param snapshotStatus snapshot status */ private void snapshot(final ShardId shardId, final Snapshot snapshot, final IndexId indexId, - final IndexShardSnapshotStatus snapshotStatus, boolean writeShardGens, ActionListener listener) { + final IndexShardSnapshotStatus snapshotStatus, ActionListener listener) { try { final IndexShard indexShard = indicesService.indexServiceSafe(shardId.getIndex()).getShardOrNull(shardId.id()); if (indexShard.routingEntry().primary() == false) { @@ -336,7 +334,7 @@ private void snapshot(final ShardId shardId, final Snapshot snapshot, final Inde // we flush first to make sure we get the latest writes snapshotted snapshotRef = indexShard.acquireLastIndexCommit(true); repository.snapshotShard(indexShard.store(), indexShard.mapperService(), snapshot.getSnapshotId(), indexId, - snapshotRef.getIndexCommit(), snapshotStatus, writeShardGens, ActionListener.runBefore(listener, snapshotRef::close)); + snapshotRef.getIndexCommit(), snapshotStatus, ActionListener.runBefore(listener, snapshotRef::close)); } catch (Exception e) { IOUtils.close(snapshotRef); throw e; diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java index e942c4ac2d33c..b1f71a337e8f8 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java @@ -294,8 +294,8 @@ public ClusterState execute(ClusterState currentState) { threadPool.absoluteTimeInMillis(), repositoryData.getGenId(), null, - request.userMetadata(), - clusterService.state().nodes().getMinNodeVersion().onOrAfter(SHARD_GEN_IN_REPO_DATA_VERSION)); + request.userMetadata() + ); initializingSnapshots.add(newSnapshot.snapshot()); snapshots = new SnapshotsInProgress(newSnapshot); } else { @@ -566,7 +566,6 @@ private void cleanupAfterError(Exception exception) { snapshot.includeGlobalState(), metaDataForSnapshot(snapshot, clusterService.state().metaData()), snapshot.userMetadata(), - snapshot.useShardGenerations(), ActionListener.runAfter(ActionListener.wrap(ignored -> { }, inner -> { inner.addSuppressed(exception); @@ -771,8 +770,7 @@ private void finalizeSnapshotDeletionFromPreviousMaster(ClusterState state) { if (deletionsInProgress != null && deletionsInProgress.hasDeletionsInProgress()) { assert deletionsInProgress.getEntries().size() == 1 : "only one in-progress deletion allowed per cluster"; SnapshotDeletionsInProgress.Entry entry = deletionsInProgress.getEntries().get(0); - deleteSnapshotFromRepository(entry.getSnapshot(), null, entry.getRepositoryStateId(), - state.nodes().getMinNodeVersion()); + deleteSnapshotFromRepository(entry.getSnapshot(), null, entry.getRepositoryStateId()); } } @@ -1030,7 +1028,6 @@ protected void doRun() { entry.includeGlobalState(), metaDataForSnapshot(entry, metaData), entry.userMetadata(), - entry.useShardGenerations(), ActionListener.wrap(snapshotInfo -> { removeSnapshotFromClusterState(snapshot, snapshotInfo, null); logger.info("snapshot [{}] completed with state [{}]", snapshot, snapshotInfo.state()); @@ -1313,7 +1310,7 @@ public void clusterStateProcessed(String source, ClusterState oldState, ClusterS )); } else { logger.debug("deleted snapshot is not running - deleting files"); - deleteSnapshotFromRepository(snapshot, listener, repositoryStateId, newState.nodes().getMinNodeVersion()); + deleteSnapshotFromRepository(snapshot, listener, repositoryStateId); } } }); @@ -1352,18 +1349,15 @@ public static boolean isRepositoryInUse(ClusterState clusterState, String reposi * @param snapshot snapshot * @param listener listener * @param repositoryStateId the unique id representing the state of the repository at the time the deletion began - * @param version minimum ES version the repository should be readable by */ - private void deleteSnapshotFromRepository(Snapshot snapshot, @Nullable ActionListener listener, long repositoryStateId, - Version version) { + private void deleteSnapshotFromRepository(Snapshot snapshot, @Nullable ActionListener listener, long repositoryStateId) { threadPool.executor(ThreadPool.Names.SNAPSHOT).execute(ActionRunnable.wrap(listener, l -> { Repository repository = repositoriesService.repository(snapshot.getRepository()); - repository.deleteSnapshot(snapshot.getSnapshotId(), repositoryStateId, version.onOrAfter(SHARD_GEN_IN_REPO_DATA_VERSION), - ActionListener.wrap(v -> { - logger.info("snapshot [{}] deleted", snapshot); - removeSnapshotDeletionFromClusterState(snapshot, null, l); - }, ex -> removeSnapshotDeletionFromClusterState(snapshot, ex, l) - )); + repository.deleteSnapshot(snapshot.getSnapshotId(), repositoryStateId, ActionListener.wrap(v -> { + logger.info("snapshot [{}] deleted", snapshot); + removeSnapshotDeletionFromClusterState(snapshot, null, l); + }, ex -> removeSnapshotDeletionFromClusterState(snapshot, ex, l) + )); })); } @@ -1438,16 +1432,12 @@ private static ImmutableOpenMap> waitingIndices = entry.waitingIndices(); assertEquals(2, waitingIndices.get(idx1Name).size()); diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataDeleteIndexServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataDeleteIndexServiceTests.java index b77653c34c769..70d1ef34b092b 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataDeleteIndexServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataDeleteIndexServiceTests.java @@ -62,7 +62,7 @@ public void testDeleteSnapshotting() { SnapshotsInProgress snaps = new SnapshotsInProgress(new SnapshotsInProgress.Entry(snapshot, true, false, SnapshotsInProgress.State.INIT, singletonList(new IndexId(index, "doesn't matter")), System.currentTimeMillis(), (long) randomIntBetween(0, 1000), ImmutableOpenMap.of(), - SnapshotInfoTests.randomUserMetadata(), randomBoolean())); + SnapshotInfoTests.randomUserMetadata())); ClusterState state = ClusterState.builder(clusterState(index)) .putCustom(SnapshotsInProgress.TYPE, snaps) .build(); diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataIndexStateServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataIndexStateServiceTests.java index f5aad6f05cd3b..1a385fb956f27 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataIndexStateServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataIndexStateServiceTests.java @@ -472,7 +472,7 @@ private static ClusterState addSnapshotIndex(final String index, final int numSh final SnapshotsInProgress.Entry entry = new SnapshotsInProgress.Entry(snapshot, randomBoolean(), false, SnapshotsInProgress.State.INIT, Collections.singletonList(new IndexId(index, index)), randomNonNegativeLong(), randomLong(), shardsBuilder.build(), - SnapshotInfoTests.randomUserMetadata(), randomBoolean()); + SnapshotInfoTests.randomUserMetadata()); return ClusterState.builder(newState).putCustom(SnapshotsInProgress.TYPE, new SnapshotsInProgress(entry)).build(); } diff --git a/server/src/test/java/org/elasticsearch/repositories/RepositoriesServiceTests.java b/server/src/test/java/org/elasticsearch/repositories/RepositoriesServiceTests.java index 63e5aa820d137..ea279f16a5a02 100644 --- a/server/src/test/java/org/elasticsearch/repositories/RepositoriesServiceTests.java +++ b/server/src/test/java/org/elasticsearch/repositories/RepositoriesServiceTests.java @@ -157,12 +157,12 @@ public RepositoryData getRepositoryData() { public void finalizeSnapshot(SnapshotId snapshotId, ShardGenerations indices, long startTime, String failure, int totalShards, List shardFailures, long repositoryStateId, boolean includeGlobalState, MetaData metaData, Map userMetadata, - boolean writeShardGens, ActionListener listener) { + ActionListener listener) { listener.onResponse(null); } @Override - public void deleteSnapshot(SnapshotId snapshotId, long repositoryStateId, boolean writeShardGens, ActionListener listener) { + public void deleteSnapshot(SnapshotId snapshotId, long repositoryStateId, ActionListener listener) { listener.onResponse(null); } @@ -198,8 +198,7 @@ public boolean isReadOnly() { @Override public void snapshotShard(Store store, MapperService mapperService, SnapshotId snapshotId, IndexId indexId, IndexCommit - snapshotIndexCommit, IndexShardSnapshotStatus snapshotStatus, boolean writeShardGens, - ActionListener listener) { + snapshotIndexCommit, IndexShardSnapshotStatus snapshotStatus, ActionListener listener) { } diff --git a/server/src/test/java/org/elasticsearch/repositories/RepositoryDataTests.java b/server/src/test/java/org/elasticsearch/repositories/RepositoryDataTests.java index 1067013d20e97..12f2a59953e57 100644 --- a/server/src/test/java/org/elasticsearch/repositories/RepositoryDataTests.java +++ b/server/src/test/java/org/elasticsearch/repositories/RepositoryDataTests.java @@ -72,7 +72,7 @@ public void testIndicesToUpdateAfterRemovingSnapshot() { public void testXContent() throws IOException { RepositoryData repositoryData = generateRandomRepoData(); XContentBuilder builder = JsonXContent.contentBuilder(); - repositoryData.snapshotsToXContent(builder, true); + repositoryData.snapshotsToXContent(builder); try (XContentParser parser = createParser(JsonXContent.jsonXContent, BytesReference.bytes(builder))) { long gen = (long) randomIntBetween(0, 500); RepositoryData fromXContent = RepositoryData.snapshotsFromXContent(parser, gen); @@ -178,7 +178,7 @@ public void testIndexThatReferencesAnUnknownSnapshot() throws IOException { final RepositoryData repositoryData = generateRandomRepoData(); XContentBuilder builder = XContentBuilder.builder(xContent); - repositoryData.snapshotsToXContent(builder, true); + repositoryData.snapshotsToXContent(builder); RepositoryData parsedRepositoryData; try (XContentParser xParser = createParser(builder)) { parsedRepositoryData = RepositoryData.snapshotsFromXContent(xParser, repositoryData.getGenId()); @@ -214,7 +214,7 @@ public void testIndexThatReferencesAnUnknownSnapshot() throws IOException { indexSnapshots, shardGenBuilder.build()); final XContentBuilder corruptedBuilder = XContentBuilder.builder(xContent); - corruptedRepositoryData.snapshotsToXContent(corruptedBuilder, true); + corruptedRepositoryData.snapshotsToXContent(corruptedBuilder); try (XContentParser xParser = createParser(corruptedBuilder)) { ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> diff --git a/server/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryRestoreTests.java b/server/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryRestoreTests.java index 59a977efe5f5d..2a225622501db 100644 --- a/server/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryRestoreTests.java +++ b/server/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryRestoreTests.java @@ -172,7 +172,7 @@ public void testSnapshotWithConflictingName() throws IOException { repository.finalizeSnapshot(snapshot.getSnapshotId(), ShardGenerations.builder().put(indexId, 0, shardGen).build(), 0L, null, 1, Collections.emptyList(), -1L, false, - MetaData.builder().put(shard.indexSettings().getIndexMetaData(), false).build(), Collections.emptyMap(), true, + MetaData.builder().put(shard.indexSettings().getIndexMetaData(), false).build(), Collections.emptyMap(), future); future.actionGet(); IndexShardSnapshotFailedException isfe = expectThrows(IndexShardSnapshotFailedException.class, diff --git a/server/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryTests.java b/server/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryTests.java index 32b65880c8249..037ac72bbb70e 100644 --- a/server/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryTests.java +++ b/server/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryTests.java @@ -141,7 +141,7 @@ public void testReadAndWriteSnapshotsThroughIndexFile() throws Exception { // write to and read from a index file with no entries assertThat(repository.getRepositoryData().getSnapshotIds().size(), equalTo(0)); final RepositoryData emptyData = RepositoryData.EMPTY; - repository.writeIndexGen(emptyData, emptyData.getGenId(), true); + repository.writeIndexGen(emptyData, emptyData.getGenId()); RepositoryData repoData = repository.getRepositoryData(); assertEquals(repoData, emptyData); assertEquals(repoData.getIndices().size(), 0); @@ -150,12 +150,12 @@ public void testReadAndWriteSnapshotsThroughIndexFile() throws Exception { // write to and read from an index file with snapshots but no indices repoData = addRandomSnapshotsToRepoData(repoData, false); - repository.writeIndexGen(repoData, repoData.getGenId(), true); + repository.writeIndexGen(repoData, repoData.getGenId()); assertEquals(repoData, repository.getRepositoryData()); // write to and read from a index file with random repository data repoData = addRandomSnapshotsToRepoData(repository.getRepositoryData(), true); - repository.writeIndexGen(repoData, repoData.getGenId(), true); + repository.writeIndexGen(repoData, repoData.getGenId()); assertEquals(repoData, repository.getRepositoryData()); } @@ -164,14 +164,14 @@ public void testIndexGenerationalFiles() throws Exception { // write to index generational file RepositoryData repositoryData = generateRandomRepoData(); - repository.writeIndexGen(repositoryData, repositoryData.getGenId(), true); + repository.writeIndexGen(repositoryData, repositoryData.getGenId()); assertThat(repository.getRepositoryData(), equalTo(repositoryData)); assertThat(repository.latestIndexBlobId(), equalTo(0L)); assertThat(repository.readSnapshotIndexLatestBlob(), equalTo(0L)); // adding more and writing to a new index generational file repositoryData = addRandomSnapshotsToRepoData(repository.getRepositoryData(), true); - repository.writeIndexGen(repositoryData, repositoryData.getGenId(), true); + repository.writeIndexGen(repositoryData, repositoryData.getGenId()); assertEquals(repository.getRepositoryData(), repositoryData); assertThat(repository.latestIndexBlobId(), equalTo(1L)); assertThat(repository.readSnapshotIndexLatestBlob(), equalTo(1L)); @@ -179,7 +179,7 @@ public void testIndexGenerationalFiles() throws Exception { // removing a snapshot and writing to a new index generational file repositoryData = repository.getRepositoryData().removeSnapshot( repositoryData.getSnapshotIds().iterator().next(), ShardGenerations.EMPTY); - repository.writeIndexGen(repositoryData, repositoryData.getGenId(), true); + repository.writeIndexGen(repositoryData, repositoryData.getGenId()); assertEquals(repository.getRepositoryData(), repositoryData); assertThat(repository.latestIndexBlobId(), equalTo(2L)); assertThat(repository.readSnapshotIndexLatestBlob(), equalTo(2L)); @@ -191,12 +191,12 @@ public void testRepositoryDataConcurrentModificationNotAllowed() throws IOExcept // write to index generational file RepositoryData repositoryData = generateRandomRepoData(); final long startingGeneration = repositoryData.getGenId(); - repository.writeIndexGen(repositoryData, startingGeneration, true); + repository.writeIndexGen(repositoryData, startingGeneration); // write repo data again to index generational file, errors because we already wrote to the // N+1 generation from which this repository data instance was created expectThrows(RepositoryException.class, () -> repository.writeIndexGen( - repositoryData.withGenId(startingGeneration + 1), repositoryData.getGenId(), true)); + repositoryData.withGenId(startingGeneration + 1), repositoryData.getGenId())); } public void testBadChunksize() throws Exception { diff --git a/server/src/test/java/org/elasticsearch/repositories/fs/FsRepositoryTests.java b/server/src/test/java/org/elasticsearch/repositories/fs/FsRepositoryTests.java index 5694629325557..3ff766e205d65 100644 --- a/server/src/test/java/org/elasticsearch/repositories/fs/FsRepositoryTests.java +++ b/server/src/test/java/org/elasticsearch/repositories/fs/FsRepositoryTests.java @@ -103,7 +103,7 @@ public void testSnapshotAndRestore() throws IOException, InterruptedException { final PlainActionFuture future1 = PlainActionFuture.newFuture(); runGeneric(threadPool, () -> { IndexShardSnapshotStatus snapshotStatus = IndexShardSnapshotStatus.newInitializing(null); - repository.snapshotShard(store, null, snapshotId, indexId, indexCommit, snapshotStatus, true, future1); + repository.snapshotShard(store, null, snapshotId, indexId, indexCommit, snapshotStatus, future1); future1.actionGet(); IndexShardSnapshotStatus.Copy copy = snapshotStatus.asCopy(); assertEquals(copy.getTotalFileCount(), copy.getIncrementalFileCount()); @@ -129,7 +129,7 @@ public void testSnapshotAndRestore() throws IOException, InterruptedException { final PlainActionFuture future2 = PlainActionFuture.newFuture(); runGeneric(threadPool, () -> { IndexShardSnapshotStatus snapshotStatus = IndexShardSnapshotStatus.newInitializing(shardGeneration); - repository.snapshotShard(store, null, incSnapshotId, indexId, incIndexCommit, snapshotStatus, true, future2); + repository.snapshotShard(store, null, incSnapshotId, indexId, incIndexCommit, snapshotStatus, future2); future2.actionGet(); IndexShardSnapshotStatus.Copy copy = snapshotStatus.asCopy(); assertEquals(2, copy.getIncrementalFileCount()); diff --git a/server/src/test/java/org/elasticsearch/snapshots/SnapshotsInProgressSerializationTests.java b/server/src/test/java/org/elasticsearch/snapshots/SnapshotsInProgressSerializationTests.java index 6059b44073442..7d59c71ed18ea 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/SnapshotsInProgressSerializationTests.java +++ b/server/src/test/java/org/elasticsearch/snapshots/SnapshotsInProgressSerializationTests.java @@ -77,7 +77,7 @@ private Entry randomSnapshot() { } ImmutableOpenMap shards = builder.build(); return new Entry(snapshot, includeGlobalState, partial, state, indices, startTime, repositoryStateId, shards, - SnapshotInfoTests.randomUserMetadata(), randomBoolean()); + SnapshotInfoTests.randomUserMetadata()); } @Override diff --git a/server/src/test/java/org/elasticsearch/snapshots/mockstore/MockEventuallyConsistentRepositoryTests.java b/server/src/test/java/org/elasticsearch/snapshots/mockstore/MockEventuallyConsistentRepositoryTests.java index b5756c89377ea..f1185858a7c4a 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/mockstore/MockEventuallyConsistentRepositoryTests.java +++ b/server/src/test/java/org/elasticsearch/snapshots/mockstore/MockEventuallyConsistentRepositoryTests.java @@ -151,7 +151,7 @@ public void testOverwriteSnapshotInfoBlob() { final SnapshotId snapshotId = new SnapshotId("foo", UUIDs.randomBase64UUID()); // We try to write another snap- blob for "foo" in the next generation. It fails because the content differs. repository.finalizeSnapshot(snapshotId, ShardGenerations.EMPTY, 1L, null, 5, Collections.emptyList(), - -1L, false, MetaData.EMPTY_META_DATA, Collections.emptyMap(), true, future); + -1L, false, MetaData.EMPTY_META_DATA, Collections.emptyMap(), future); future.actionGet(); // We try to write another snap- blob for "foo" in the next generation. It fails because the content differs. @@ -160,7 +160,7 @@ public void testOverwriteSnapshotInfoBlob() { final PlainActionFuture fut = PlainActionFuture.newFuture(); repository.finalizeSnapshot( snapshotId, ShardGenerations.EMPTY, 1L, null, 6, Collections.emptyList(), - 0, false, MetaData.EMPTY_META_DATA, Collections.emptyMap(), true, fut); + 0, false, MetaData.EMPTY_META_DATA, Collections.emptyMap(), fut); fut.actionGet(); }); assertThat(assertionError.getMessage(), equalTo("\nExpected: <6>\n but: was <5>")); @@ -169,7 +169,7 @@ public void testOverwriteSnapshotInfoBlob() { // It passes cleanly because the content of the blob except for the timestamps. final PlainActionFuture future2 = PlainActionFuture.newFuture(); repository.finalizeSnapshot(snapshotId, ShardGenerations.EMPTY, 1L, null, 5, Collections.emptyList(), - 0, false, MetaData.EMPTY_META_DATA, Collections.emptyMap(),true, future2); + 0, false, MetaData.EMPTY_META_DATA, Collections.emptyMap(), future2); future2.actionGet(); } } diff --git a/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java index 98e2f88292d56..24e6e3a6dca0e 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java @@ -828,7 +828,7 @@ protected String snapshotShard(final IndexShard shard, final String shardGen; try (Engine.IndexCommitRef indexCommitRef = shard.acquireLastIndexCommit(true)) { repository.snapshotShard(shard.store(), shard.mapperService(), snapshot.getSnapshotId(), indexId, - indexCommitRef.getIndexCommit(), snapshotStatus, true, future); + indexCommitRef.getIndexCommit(), snapshotStatus, future); shardGen = future.actionGet(); } diff --git a/test/framework/src/main/java/org/elasticsearch/index/shard/RestoreOnlyRepository.java b/test/framework/src/main/java/org/elasticsearch/index/shard/RestoreOnlyRepository.java index 22bacff49f022..8fb39c25e98bc 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/shard/RestoreOnlyRepository.java +++ b/test/framework/src/main/java/org/elasticsearch/index/shard/RestoreOnlyRepository.java @@ -94,13 +94,13 @@ public RepositoryData getRepositoryData() { @Override public void finalizeSnapshot(SnapshotId snapshotId, ShardGenerations shardGenerations, long startTime, String failure, int totalShards, List shardFailures, long repositoryStateId, - boolean includeGlobalState, MetaData metaData, Map userMetadata, boolean writeShardGens, + boolean includeGlobalState, MetaData metaData, Map userMetadata, ActionListener listener) { listener.onResponse(null); } @Override - public void deleteSnapshot(SnapshotId snapshotId, long repositoryStateId, boolean writeShardGens, ActionListener listener) { + public void deleteSnapshot(SnapshotId snapshotId, long repositoryStateId, ActionListener listener) { listener.onResponse(null); } @@ -130,8 +130,7 @@ public boolean isReadOnly() { @Override public void snapshotShard(Store store, MapperService mapperService, SnapshotId snapshotId, IndexId indexId, - IndexCommit snapshotIndexCommit, IndexShardSnapshotStatus snapshotStatus, boolean writeShardGens, - ActionListener listener) { + IndexCommit snapshotIndexCommit, IndexShardSnapshotStatus snapshotStatus, ActionListener listener) { } @Override diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRepository.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRepository.java index 5ec68fb604bc7..5abcce781aa40 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRepository.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRepository.java @@ -248,13 +248,12 @@ public RepositoryData getRepositoryData() { @Override public void finalizeSnapshot(SnapshotId snapshotId, ShardGenerations shardGenerations, long startTime, String failure, int totalShards, List shardFailures, long repositoryStateId, boolean includeGlobalState, - MetaData metaData, Map userMetadata, boolean writeShardGens, - ActionListener listener) { + MetaData metaData, Map userMetadata, ActionListener listener) { throw new UnsupportedOperationException("Unsupported for repository of type: " + TYPE); } @Override - public void deleteSnapshot(SnapshotId snapshotId, long repositoryStateId, boolean writeShardGens, ActionListener listener) { + public void deleteSnapshot(SnapshotId snapshotId, long repositoryStateId, ActionListener listener) { throw new UnsupportedOperationException("Unsupported for repository of type: " + TYPE); } @@ -289,8 +288,7 @@ public boolean isReadOnly() { @Override public void snapshotShard(Store store, MapperService mapperService, SnapshotId snapshotId, IndexId indexId, - IndexCommit snapshotIndexCommit, IndexShardSnapshotStatus snapshotStatus, boolean writeShardGens, - ActionListener listener) { + IndexCommit snapshotIndexCommit, IndexShardSnapshotStatus snapshotStatus, ActionListener listener) { throw new UnsupportedOperationException("Unsupported for repository of type: " + TYPE); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshotRepository.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshotRepository.java index 35f71a6c7f540..0e1cdcf1b0d03 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshotRepository.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshotRepository.java @@ -83,13 +83,13 @@ public final class SourceOnlySnapshotRepository extends FilterRepository { public void finalizeSnapshot(SnapshotId snapshotId, ShardGenerations shardGenerations, long startTime, String failure, int totalShards, List shardFailures, long repositoryStateId, boolean includeGlobalState, MetaData metaData, Map userMetadata, - boolean writeShardGens, ActionListener listener) { + ActionListener listener) { // we process the index metadata at snapshot time. This means if somebody tries to restore // a _source only snapshot with a plain repository it will be just fine since we already set the // required engine, that the index is read-only and the mapping to a default mapping try { super.finalizeSnapshot(snapshotId, shardGenerations, startTime, failure, totalShards, shardFailures, repositoryStateId, - includeGlobalState, metadataToSnapshot(shardGenerations.indices(), metaData), userMetadata, writeShardGens, listener); + includeGlobalState, metadataToSnapshot(shardGenerations.indices(), metaData), userMetadata, listener); } catch (IOException ex) { listener.onFailure(ex); } @@ -124,8 +124,7 @@ private static MetaData metadataToSnapshot(Collection indices, MetaData @Override public void snapshotShard(Store store, MapperService mapperService, SnapshotId snapshotId, IndexId indexId, - IndexCommit snapshotIndexCommit, IndexShardSnapshotStatus snapshotStatus, boolean writeShardGens, - ActionListener listener) { + IndexCommit snapshotIndexCommit, IndexShardSnapshotStatus snapshotStatus, ActionListener listener) { if (mapperService.documentMapper() != null // if there is no mapping this is null && mapperService.documentMapper().sourceMapper().isComplete() == false) { listener.onFailure( @@ -164,7 +163,7 @@ protected void closeInternal() { Collections.singletonMap(BlockTreeTermsReader.FST_MODE_KEY, BlockTreeTermsReader.FSTLoadMode.OFF_HEAP.name())); toClose.add(reader); IndexCommit indexCommit = reader.getIndexCommit(); - super.snapshotShard(tempStore, mapperService, snapshotId, indexId, indexCommit, snapshotStatus, writeShardGens, + super.snapshotShard(tempStore, mapperService, snapshotId, indexId, indexCommit, snapshotStatus, ActionListener.runBefore(listener, () -> IOUtils.close(toClose))); } catch (IOException e) { try { diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/SourceOnlySnapshotShardTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/SourceOnlySnapshotShardTests.java index c4ccf6bfd832b..ce4d2ed062662 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/SourceOnlySnapshotShardTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/SourceOnlySnapshotShardTests.java @@ -100,7 +100,7 @@ public void testSourceIncomplete() throws IOException { IndexShardSnapshotStatus indexShardSnapshotStatus = IndexShardSnapshotStatus.newInitializing("-1"); final PlainActionFuture future = PlainActionFuture.newFuture(); runAsSnapshot(shard.getThreadPool(), () -> repository.snapshotShard(shard.store(), shard.mapperService(), snapshotId, indexId, - snapshotRef.getIndexCommit(), indexShardSnapshotStatus, true, future)); + snapshotRef.getIndexCommit(), indexShardSnapshotStatus, future)); IllegalStateException illegalStateException = expectThrows(IllegalStateException.class, future::actionGet); assertEquals( "Can't snapshot _source only on an index that has incomplete source ie. has _source disabled or filters the source", @@ -126,7 +126,7 @@ public void testIncrementalSnapshot() throws IOException { SnapshotId snapshotId = new SnapshotId("test", "test"); final PlainActionFuture future = PlainActionFuture.newFuture(); runAsSnapshot(shard.getThreadPool(), () -> repository.snapshotShard(shard.store(), shard.mapperService(), snapshotId, indexId, - snapshotRef.getIndexCommit(), indexShardSnapshotStatus, true, future)); + snapshotRef.getIndexCommit(), indexShardSnapshotStatus, future)); shardGeneration = future.actionGet(); IndexShardSnapshotStatus.Copy copy = indexShardSnapshotStatus.asCopy(); assertEquals(copy.getTotalFileCount(), copy.getIncrementalFileCount()); @@ -142,7 +142,7 @@ public void testIncrementalSnapshot() throws IOException { IndexShardSnapshotStatus indexShardSnapshotStatus = IndexShardSnapshotStatus.newInitializing(shardGeneration); final PlainActionFuture future = PlainActionFuture.newFuture(); runAsSnapshot(shard.getThreadPool(), () -> repository.snapshotShard(shard.store(), shard.mapperService(), snapshotId, indexId, - snapshotRef.getIndexCommit(), indexShardSnapshotStatus, true, future)); + snapshotRef.getIndexCommit(), indexShardSnapshotStatus, future)); shardGeneration = future.actionGet(); IndexShardSnapshotStatus.Copy copy = indexShardSnapshotStatus.asCopy(); // we processed the segments_N file plus _1.si, _1.fdx, _1.fnm, _1.fdt @@ -158,7 +158,7 @@ public void testIncrementalSnapshot() throws IOException { IndexShardSnapshotStatus indexShardSnapshotStatus = IndexShardSnapshotStatus.newInitializing(shardGeneration); final PlainActionFuture future = PlainActionFuture.newFuture(); runAsSnapshot(shard.getThreadPool(), () -> repository.snapshotShard(shard.store(), shard.mapperService(), snapshotId, indexId, - snapshotRef.getIndexCommit(), indexShardSnapshotStatus, true, future)); + snapshotRef.getIndexCommit(), indexShardSnapshotStatus, future)); future.actionGet(); IndexShardSnapshotStatus.Copy copy = indexShardSnapshotStatus.asCopy(); // we processed the segments_N file plus _1_1.liv @@ -206,7 +206,7 @@ public void testRestoreMinmal() throws IOException { final PlainActionFuture future = PlainActionFuture.newFuture(); runAsSnapshot(shard.getThreadPool(), () -> { repository.snapshotShard(shard.store(), shard.mapperService(), snapshotId, indexId, snapshotRef.getIndexCommit(), - indexShardSnapshotStatus, true, future); + indexShardSnapshotStatus, future); future.actionGet(); final PlainActionFuture finFuture = PlainActionFuture.newFuture(); repository.finalizeSnapshot(snapshotId, @@ -214,7 +214,6 @@ public void testRestoreMinmal() throws IOException { indexShardSnapshotStatus.asCopy().getStartTime(), null, 1, Collections.emptyList(), repository.getRepositoryData().getGenId(), true, MetaData.builder().put(shard.indexSettings().getIndexMetaData(), false).build(), Collections.emptyMap(), - true, finFuture); finFuture.actionGet(); }); diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/slm/SnapshotRetentionTaskTests.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/slm/SnapshotRetentionTaskTests.java index 8dea5e2b6adbd..dff01f8e9b0bb 100644 --- a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/slm/SnapshotRetentionTaskTests.java +++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/slm/SnapshotRetentionTaskTests.java @@ -334,8 +334,7 @@ public void testOkToDeleteSnapshots() { new SnapshotsInProgress.Entry( snapshot, true, false, SnapshotsInProgress.State.INIT, Collections.singletonList(new IndexId("name", "id")), 0, 0, - ImmutableOpenMap.builder().build(), Collections.emptyMap(), - randomBoolean())); + ImmutableOpenMap.builder().build(), Collections.emptyMap())); ClusterState state = ClusterState.builder(new ClusterName("cluster")) .putCustom(SnapshotsInProgress.TYPE, inProgress) .build();