Skip to content

Remove Obsolete ShardGenerations BwC Logic #48416

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 4 commits into from
Closed

Remove Obsolete ShardGenerations BwC Logic #48416

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
4e0fcb9
Remove Obsolete ShardGenerations BwC Logic
original-brownbear Oct 23, 2019
73242fa
Merge remote-tracking branch 'elastic/master' into drop-pointless-bwc…
original-brownbear Oct 23, 2019
e7052ad
missed an assertion
original-brownbear Oct 23, 2019
563f8aa
Merge branch 'master' into drop-pointless-bwc-logic
elasticmachine Oct 23, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 0 additions & 2 deletions ...ticsearch/action/admin/cluster/repositories/cleanup/TransportCleanupRepositoryAction.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,6 @@
import org.elasticsearch.repositories.Repository;
import org.elasticsearch.repositories.RepositoryCleanupResult;
import org.elasticsearch.repositories.blobstore.BlobStoreRepository;
import org.elasticsearch.snapshots.SnapshotsService;
import org.elasticsearch.tasks.Task;
import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.transport.TransportService;
Expand Down Expand Up @@ -203,7 +202,6 @@ public void clusterStateProcessed(String source, ClusterState oldState, ClusterS
threadPool.executor(ThreadPool.Names.SNAPSHOT).execute(ActionRunnable.wrap(listener,
l -> blobStoreRepository.cleanup(
repositoryStateId,
newState.nodes().getMinNodeVersion().onOrAfter(SnapshotsService.SHARD_GEN_IN_REPO_DATA_VERSION),
ActionListener.wrap(result -> after(null, result), e -> after(e, null)))));
}

Expand Down
44 changes: 14 additions & 30 deletions server/src/main/java/org/elasticsearch/cluster/SnapshotsInProgress.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -92,13 +92,12 @@ public static class Entry implements ToXContent {
private final long startTime;
private final long repositoryStateId;
// see #useShardGenerations
private final boolean useShardGenerations;
@Nullable private final Map<String, Object> userMetadata;
@Nullable private final String failure;

public Entry(Snapshot snapshot, boolean includeGlobalState, boolean partial, State state, List<IndexId> indices,
long startTime, long repositoryStateId, ImmutableOpenMap<ShardId, ShardSnapshotStatus> shards,
String failure, Map<String, Object> userMetadata, boolean useShardGenerations) {
String failure, Map<String, Object> userMetadata) {
this.state = state;
this.snapshot = snapshot;
this.includeGlobalState = includeGlobalState;
Expand All @@ -116,7 +115,6 @@ public Entry(Snapshot snapshot, boolean includeGlobalState, boolean partial, Sta
this.repositoryStateId = repositoryStateId;
this.failure = failure;
this.userMetadata = userMetadata;
this.useShardGenerations = useShardGenerations;
}

private static boolean assertShardsConsistent(State state, List<IndexId> indices,
Expand All @@ -134,19 +132,18 @@ private static boolean assertShardsConsistent(State state, List<IndexId> indices

public Entry(Snapshot snapshot, boolean includeGlobalState, boolean partial, State state, List<IndexId> indices,
long startTime, long repositoryStateId, ImmutableOpenMap<ShardId, ShardSnapshotStatus> shards,
Map<String, Object> userMetadata, boolean useShardGenerations) {
this(snapshot, includeGlobalState, partial, state, indices, startTime, repositoryStateId, shards, null, userMetadata,
useShardGenerations);
Map<String, Object> userMetadata) {
this(snapshot, includeGlobalState, partial, state, indices, startTime, repositoryStateId, shards, null, userMetadata);
}

public Entry(Entry entry, State state, ImmutableOpenMap<ShardId, ShardSnapshotStatus> shards) {
this(entry.snapshot, entry.includeGlobalState, entry.partial, state, entry.indices, entry.startTime,
entry.repositoryStateId, shards, entry.failure, entry.userMetadata, entry.useShardGenerations);
entry.repositoryStateId, shards, entry.failure, entry.userMetadata);
}

public Entry(Entry entry, State state, ImmutableOpenMap<ShardId, ShardSnapshotStatus> shards, String failure) {
this(entry.snapshot, entry.includeGlobalState, entry.partial, state, entry.indices, entry.startTime,
entry.repositoryStateId, shards, failure, entry.userMetadata, entry.useShardGenerations);
entry.repositoryStateId, shards, failure, entry.userMetadata);
}

public Entry(Entry entry, ImmutableOpenMap<ShardId, ShardSnapshotStatus> shards) {
Expand Down Expand Up @@ -197,16 +194,6 @@ public String failure() {
return failure;
}

/**
* Whether to write to the repository in a format only understood by versions newer than
* {@link SnapshotsService#SHARD_GEN_IN_REPO_DATA_VERSION}.
*
* @return true if writing to repository in new format
*/
public boolean useShardGenerations() {
return useShardGenerations;
}

@Override
public boolean equals(Object o) {
if (this == o) return true;
Expand All @@ -222,7 +209,6 @@ public boolean equals(Object o) {
if (!snapshot.equals(entry.snapshot)) return false;
if (state != entry.state) return false;
if (repositoryStateId != entry.repositoryStateId) return false;
if (useShardGenerations != entry.useShardGenerations) return false;

return true;
}
Expand All @@ -237,7 +223,6 @@ public int hashCode() {
result = 31 * result + indices.hashCode();
result = 31 * result + Long.hashCode(startTime);
result = 31 * result + Long.hashCode(repositoryStateId);
result = 31 * result + (useShardGenerations ? 1 : 0);
return result;
}

Expand Down Expand Up @@ -520,11 +505,9 @@ public SnapshotsInProgress(StreamInput in) throws IOException {
if (in.getVersion().onOrAfter(METADATA_FIELD_INTRODUCED)) {
userMetadata = in.readMap();
}
final boolean useShardGenerations;
if (in.getVersion().onOrAfter(SnapshotsService.SHARD_GEN_IN_REPO_DATA_VERSION)) {
useShardGenerations = in.readBoolean();
} else {
useShardGenerations = false;
final Version version = in.getVersion();
if (version.before(Version.V_8_0_0) && version.onOrAfter(SnapshotsService.SHARD_GEN_IN_REPO_DATA_VERSION)) {
Copy link
Contributor

@ywelsch ywelsch Oct 24, 2019

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

given that this logic can only be removed in 9.0, perhaps simpler to just unconditionally do a in.readBoolean(); here and add an assertion that fails once we branch 9.0. Also note that version.onOrAfter(SnapshotsService.SHARD_GEN_IN_REPO_DATA_VERSION) should be unnecessary once we stop testing master against 7.5. This is also why I prefer to wait with PRs like this until that has happened, it simplifies the BWC conditions.

Copy link
Contributor Author

@original-brownbear original-brownbear Oct 24, 2019 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is also why I prefer to wait with PRs like this until that has happened, it simplifies the BWC conditions.

Right that's a fair point. I'll just close here then and look into this again once 7.6 is out :)

in.readBoolean();
}
entries[i] = new Entry(snapshot,
includeGlobalState,
Expand All @@ -535,8 +518,7 @@ public SnapshotsInProgress(StreamInput in) throws IOException {
repositoryStateId,
builder.build(),
failure,
userMetadata,
useShardGenerations
userMetadata
);
}
this.entries = Arrays.asList(entries);
Expand All @@ -562,11 +544,13 @@ public void writeTo(StreamOutput out) throws IOException {
}
out.writeLong(entry.repositoryStateId);
out.writeOptionalString(entry.failure);
if (out.getVersion().onOrAfter(METADATA_FIELD_INTRODUCED)) {
final Version version = out.getVersion();
if (version.onOrAfter(METADATA_FIELD_INTRODUCED)) {
out.writeMap(entry.userMetadata);
}
if (out.getVersion().onOrAfter(SnapshotsService.SHARD_GEN_IN_REPO_DATA_VERSION)) {
out.writeBoolean(entry.useShardGenerations);
if (version.before(Version.V_8_0_0) && version.onOrAfter(SnapshotsService.SHARD_GEN_IN_REPO_DATA_VERSION)) {
// BwC logic: we always write shard generations in all versions that we support rolling upgrades from
out.writeBoolean(true);
}
}
}
Expand Down
13 changes: 6 additions & 7 deletions server/src/main/java/org/elasticsearch/repositories/FilterRepository.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,14 +76,14 @@ public RepositoryData getRepositoryData() {
public void finalizeSnapshot(SnapshotId snapshotId, ShardGenerations shardGenerations, long startTime, String failure,
int totalShards, List<SnapshotShardFailure> shardFailures, long repositoryStateId,
boolean includeGlobalState, MetaData metaData, Map<String, Object> userMetadata,
boolean writeShardGens, ActionListener<SnapshotInfo> listener) {
ActionListener<SnapshotInfo> listener) {
in.finalizeSnapshot(snapshotId, shardGenerations, startTime, failure, totalShards, shardFailures, repositoryStateId,
includeGlobalState, metaData, userMetadata, writeShardGens, listener);
includeGlobalState, metaData, userMetadata, listener);
}

@Override
public void deleteSnapshot(SnapshotId snapshotId, long repositoryStateId, boolean writeShardGens, ActionListener<Void> listener) {
in.deleteSnapshot(snapshotId, repositoryStateId, writeShardGens, listener);
public void deleteSnapshot(SnapshotId snapshotId, long repositoryStateId, ActionListener<Void> listener) {
in.deleteSnapshot(snapshotId, repositoryStateId, listener);
}

@Override
Expand Down Expand Up @@ -118,9 +118,8 @@ public boolean isReadOnly() {

@Override
public void snapshotShard(Store store, MapperService mapperService, SnapshotId snapshotId, IndexId indexId,
IndexCommit snapshotIndexCommit, IndexShardSnapshotStatus snapshotStatus, boolean writeShardGens,
ActionListener<String> listener) {
in.snapshotShard(store, mapperService, snapshotId, indexId, snapshotIndexCommit, snapshotStatus, writeShardGens, listener);
IndexCommit snapshotIndexCommit, IndexShardSnapshotStatus snapshotStatus, ActionListener<String> listener) {
in.snapshotShard(store, mapperService, snapshotId, indexId, snapshotIndexCommit, snapshotStatus, listener);
}
@Override
public void restoreShard(Store store, SnapshotId snapshotId, IndexId indexId, ShardId snapshotShardId, RecoveryState recoveryState) {
Expand Down
8 changes: 3 additions & 5 deletions server/src/main/java/org/elasticsearch/repositories/Repository.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -122,23 +122,21 @@ default Repository create(RepositoryMetaData metaData, Function<String, Reposito
* @param includeGlobalState include cluster global state
* @param clusterMetaData cluster metadata
* @param userMetadata user metadata
* @param writeShardGens if shard generations should be written to the repository
* @param listener listener to be called on completion of the snapshot
*/
void finalizeSnapshot(SnapshotId snapshotId, ShardGenerations shardGenerations, long startTime, String failure,
int totalShards, List<SnapshotShardFailure> shardFailures, long repositoryStateId,
boolean includeGlobalState, MetaData clusterMetaData, Map<String, Object> userMetadata,
boolean writeShardGens, ActionListener<SnapshotInfo> listener);
ActionListener<SnapshotInfo> listener);

/**
* Deletes snapshot
*
* @param snapshotId snapshot id
* @param repositoryStateId the unique id identifying the state of the repository when the snapshot deletion began
* @param writeShardGens if shard generations should be written to the repository
* @param listener completion listener
*/
void deleteSnapshot(SnapshotId snapshotId, long repositoryStateId, boolean writeShardGens, ActionListener<Void> listener);
void deleteSnapshot(SnapshotId snapshotId, long repositoryStateId, ActionListener<Void> listener);

/**
* Returns snapshot throttle time in nanoseconds
Expand Down Expand Up @@ -200,7 +198,7 @@ void finalizeSnapshot(SnapshotId snapshotId, ShardGenerations shardGenerations,
* @param listener listener invoked on completion
*/
void snapshotShard(Store store, MapperService mapperService, SnapshotId snapshotId, IndexId indexId, IndexCommit snapshotIndexCommit,
IndexShardSnapshotStatus snapshotStatus, boolean writeShardGens, ActionListener<String> listener);
IndexShardSnapshotStatus snapshotStatus, ActionListener<String> listener);

/**
* Restores snapshot of the shard.
Expand Down
15 changes: 5 additions & 10 deletions server/src/main/java/org/elasticsearch/repositories/RepositoryData.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -314,10 +314,7 @@ public List<IndexId> resolveNewIndices(final List<String> indicesToResolve) {
/**
* Writes the snapshots metadata and the related indices metadata to x-content.
*/
public XContentBuilder snapshotsToXContent(final XContentBuilder builder, final boolean shouldWriteShardGens) throws IOException {
assert shouldWriteShardGens || shardGenerations.indices().isEmpty() :
"Should not build shard generations in BwC mode but saw generations [" + shardGenerations + "]";

public XContentBuilder snapshotsToXContent(final XContentBuilder builder) throws IOException {
builder.startObject();
// write the snapshots list
builder.startArray(SNAPSHOTS);
Expand All @@ -343,13 +340,11 @@ public XContentBuilder snapshotsToXContent(final XContentBuilder builder, final
builder.value(snapshotId.getUUID());
}
builder.endArray();
if (shouldWriteShardGens) {
builder.startArray(SHARD_GENERATIONS);
for (String gen : shardGenerations.getGens(indexId)) {
builder.value(gen);
}
builder.endArray();
builder.startArray(SHARD_GENERATIONS);
for (String gen : shardGenerations.getGens(indexId)) {
builder.value(gen);
}
builder.endArray();
builder.endObject();
}
builder.endObject();
Expand Down
Loading