diff --git a/qa/smoke-test-http/src/internalClusterTest/java/org/elasticsearch/http/snapshots/RestGetSnapshotsIT.java b/qa/smoke-test-http/src/internalClusterTest/java/org/elasticsearch/http/snapshots/RestGetSnapshotsIT.java index eb3b95ff27595..de73f973f511f 100644 --- a/qa/smoke-test-http/src/internalClusterTest/java/org/elasticsearch/http/snapshots/RestGetSnapshotsIT.java +++ b/qa/smoke-test-http/src/internalClusterTest/java/org/elasticsearch/http/snapshots/RestGetSnapshotsIT.java @@ -12,6 +12,7 @@ import org.apache.http.client.methods.HttpGet; import org.elasticsearch.action.ActionFuture; import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse; +import org.elasticsearch.action.admin.cluster.snapshots.get.After; import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsRequest; import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsResponse; import org.elasticsearch.action.admin.cluster.snapshots.get.SnapshotSortKey; @@ -386,7 +387,7 @@ private static void assertStablePagination(String repoName, Collection a final GetSnapshotsResponse getSnapshotsResponse = sortedWithLimit( repoName, sort, - sort.encodeAfterQueryParam(after), + After.fromSnapshotInfo(after, sort).toQueryParam(), i, order, includeIndexNames diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/GetSnapshotsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/GetSnapshotsIT.java index d7d731789710e..f066555fa9ce8 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/GetSnapshotsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/GetSnapshotsIT.java @@ -20,6 +20,7 @@ import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotRequest; import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse; import org.elasticsearch.action.admin.cluster.snapshots.create.TransportCreateSnapshotAction; +import org.elasticsearch.action.admin.cluster.snapshots.get.After; import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsRequest; import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsRequestBuilder; import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsResponse; @@ -826,7 +827,7 @@ private static void assertStablePagination(String[] repoNames, Collection client().execute(TransportGetSnapshotsAction.TYPE, nextRequest, l)); diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/After.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/After.java new file mode 100644 index 0000000000000..043901593193a --- /dev/null +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/After.java @@ -0,0 +1,81 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.action.admin.cluster.snapshots.get; + +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.snapshots.SnapshotInfo; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.Base64; + +/** + * Cursor value for the {@code ?after} pagination parameter of the get-snapshots API. Encodes the sort key value plus snapshot and + * repository names for tiebreaking. Can be decoded from a query param and passed in a {@link GetSnapshotsRequest}, and used by + * {@link AfterPredicates} to filter out snapshots that are not after this cursor. + */ +public record After(String value, String repoName, String snapshotName) implements Writeable { + + public After(StreamInput in) throws IOException { + this(in.readString(), in.readString(), in.readString()); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(value); + out.writeString(repoName); + out.writeString(snapshotName); + } + + /** + * Build an {@link After} cursor from the last snapshot in a page, for use when building the {@code ?after} value for the next request. + * + * @param snapshotInfo the snapshot (typically the last in the current page) + * @param sortBy the sort key used for the request + * @return the cursor + */ + public static After fromSnapshotInfo(SnapshotInfo snapshotInfo, SnapshotSortKey sortBy) { + return new After(switch (sortBy) { + case START_TIME -> Long.toString(snapshotInfo.startTime()); + case NAME -> snapshotInfo.snapshotId().getName(); + case DURATION -> Long.toString(snapshotInfo.endTime() - snapshotInfo.startTime()); + case INDICES -> Integer.toString(snapshotInfo.indices().size()); + case SHARDS -> Integer.toString(snapshotInfo.totalShards()); + case FAILED_SHARDS -> Integer.toString(snapshotInfo.failedShards()); + case REPOSITORY -> snapshotInfo.repository(); + }, snapshotInfo.repository(), snapshotInfo.snapshotId().getName()); + } + + /** + * Encode this cursor as the {@code ?after} query parameter value (base64url-encoded "value,repoName,snapshotName"). + * + * @return the encoded string for use as the {@code after} query parameter + */ + public String toQueryParam() { + return Base64.getUrlEncoder().encodeToString((value + "," + repoName + "," + snapshotName).getBytes(StandardCharsets.UTF_8)); + } + + /** + * Decode the {@code ?after} query parameter into an {@link After} instance. + * + * @param param the value of the {@code after} query parameter (base64url-encoded "value,repoName,snapshotName") + * @return the decoded cursor + * @throws IllegalArgumentException if the parameter format is invalid + */ + public static After decodeAfterQueryParam(String param) { + final String[] parts = new String(Base64.getUrlDecoder().decode(param), StandardCharsets.UTF_8).split(","); + if (parts.length != 3) { + throw new IllegalArgumentException("invalid ?after parameter [" + param + "]"); + } + return new After(parts[0], parts[1], parts[2]); + } +} diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/AfterPredicates.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/AfterPredicates.java new file mode 100644 index 0000000000000..8d747a0bb1950 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/AfterPredicates.java @@ -0,0 +1,101 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.action.admin.cluster.snapshots.get; + +import org.elasticsearch.core.Nullable; +import org.elasticsearch.search.sort.SortOrder; +import org.elasticsearch.snapshots.SnapshotInfo; + +import java.util.function.Predicate; +import java.util.function.ToLongFunction; + +/** + * Predicate for the {@code ?after} filter of the get-snapshots action. The {@link #test(SnapshotInfo)} predicate is applied to + * {@link SnapshotInfo} instances to filter out those that sort before the cursor value (i.e. were returned on earlier pages of results). + */ +final class AfterPredicates { + + private static final AfterPredicates MATCH_ALL = new AfterPredicates(null); + + @Nullable // if MATCH_ALL + private final Predicate snapshotPredicate; + + private AfterPredicates(@Nullable Predicate snapshotPredicate) { + this.snapshotPredicate = snapshotPredicate; + } + + /** + * Test using the full {@link SnapshotInfo}. Returns true if the snapshot should be included (sorts after the cursor). + */ + boolean test(SnapshotInfo snapshotInfo) { + return snapshotPredicate == null || snapshotPredicate.test(snapshotInfo); + } + + static AfterPredicates forAfter(@Nullable After after, SnapshotSortKey sortBy, SortOrder order) { + if (after == null) { + return MATCH_ALL; + } + + return new AfterPredicates(switch (sortBy) { + case START_TIME -> longValuePredicate(after, SnapshotInfo::startTime, order); + case NAME -> namePredicate(after, order); + case DURATION -> longValuePredicate(after, info -> info.endTime() - info.startTime(), order); + case INDICES -> longValuePredicate(after, info -> info.indices().size(), order); + case SHARDS -> longValuePredicate(after, SnapshotInfo::totalShards, order); + case FAILED_SHARDS -> longValuePredicate(after, SnapshotInfo::failedShards, order); + case REPOSITORY -> repositoryPredicate(after, order); + }); + } + + private static Predicate longValuePredicate(After after, ToLongFunction extractor, SortOrder sortOrder) { + final long afterVal = Long.parseLong(after.value()); + final String snapshotName = after.snapshotName(); + final String repoName = after.repoName(); + return sortOrder == SortOrder.ASC ? info -> { + final long val = extractor.applyAsLong(info); + return afterVal < val || (afterVal == val && compareName(snapshotName, repoName, info) < 0); + } : info -> { + final long val = extractor.applyAsLong(info); + return afterVal > val || (afterVal == val && compareName(snapshotName, repoName, info) > 0); + }; + } + + private static Predicate namePredicate(After after, SortOrder sortOrder) { + final String snapshotName = after.snapshotName(); + final String repoName = after.repoName(); + return sortOrder == SortOrder.ASC + ? (info -> compareName(snapshotName, repoName, info) < 0) + : (info -> compareName(snapshotName, repoName, info) > 0); + } + + private static Predicate repositoryPredicate(After after, SortOrder sortOrder) { + final String snapshotName = after.snapshotName(); + final String repoName = after.repoName(); + return sortOrder == SortOrder.ASC + ? (info -> compareRepositoryName(snapshotName, repoName, info) < 0) + : (info -> compareRepositoryName(snapshotName, repoName, info) > 0); + } + + private static int compareName(String name, String repoName, SnapshotInfo info) { + final int res = name.compareTo(info.snapshotId().getName()); + if (res != 0) { + return res; + } + return repoName.compareTo(info.repository()); + } + + private static int compareRepositoryName(String name, String repoName, SnapshotInfo info) { + final int res = repoName.compareTo(info.repository()); + if (res != 0) { + return res; + } + return name.compareTo(info.snapshotId().getName()); + } +} diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/FromSortValuePredicates.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/FromSortValuePredicates.java new file mode 100644 index 0000000000000..b7f5d5a78a367 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/FromSortValuePredicates.java @@ -0,0 +1,180 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.action.admin.cluster.snapshots.get; + +import org.elasticsearch.core.Nullable; +import org.elasticsearch.repositories.IndexId; +import org.elasticsearch.repositories.RepositoryData; +import org.elasticsearch.search.sort.SortOrder; +import org.elasticsearch.snapshots.SnapshotId; +import org.elasticsearch.snapshots.SnapshotInfo; + +import java.util.function.Predicate; +import java.util.function.ToLongFunction; + +import static org.elasticsearch.action.admin.cluster.snapshots.get.PreflightFilterResult.EXCLUDE; +import static org.elasticsearch.action.admin.cluster.snapshots.get.PreflightFilterResult.INCLUDE; +import static org.elasticsearch.action.admin.cluster.snapshots.get.PreflightFilterResult.INCONCLUSIVE; + +/** + * A pair of predicates for the {@code ?from_sort_value} filter of the get-snapshots action. The {@link #test(SnapshotId, RepositoryData)} + * predicate is applied to combinations of snapshot id and repository data to determine which snapshots to fully load from the repository + * and rules out all snapshots that do not match the given {@link GetSnapshotsRequest} that can be ruled out through the information in + * {@link RepositoryData}. The predicate returned by {@link #test(SnapshotInfo)} predicate is then applied the instances of + * {@link SnapshotInfo} that were loaded from the repository to filter out those remaining that did not match the request but could not be + * ruled out without loading their {@link SnapshotInfo}. + */ +final class FromSortValuePredicates { + + private static final FromSortValuePredicates MATCH_ALL = new FromSortValuePredicates(null, null); + + @Nullable // non-null for sort keys that can be filtered from repository data (NAME, INDICES, START_TIME, DURATION) + private final PreflightFilterResult.RepositoryDataFilter preflightPredicate; + + @Nullable // null if all snapshots match or the pre-flight filter is guaranteed to be conclusive so no SnapshotInfo filter needed + private final Predicate snapshotPredicate; + + private FromSortValuePredicates( + @Nullable PreflightFilterResult.RepositoryDataFilter preflightPredicate, + @Nullable Predicate snapshotPredicate + ) { + this.snapshotPredicate = snapshotPredicate; + this.preflightPredicate = preflightPredicate; + } + + /** + * Pre-flight test using only snapshot id and repository data, i.e. without needing to load the full {@link SnapshotInfo}. + * + * @return an accurate result for {@link SnapshotSortKey#NAME}, {@link SnapshotSortKey#INDICES} and + * {@link SnapshotSortKey#REPOSITORY} (the latter because {@code GetSnapshotsOperation#skipRepository} handles this case even + * earlier). Also accurate for {@link SnapshotSortKey#START_TIME} and {@link SnapshotSortKey#DURATION} if and only if the + * corresponding values are available in {@link RepositoryData.SnapshotDetails}; returns {@link PreflightFilterResult#INCONCLUSIVE} + * otherwise. + * @see PreflightFilterResult.RepositoryDataFilter + */ + PreflightFilterResult test(SnapshotId snapshotId, RepositoryData repositoryData) { + if (this == MATCH_ALL) { + // no ?from_sort_value parameter, or sorting by REPOSITORY + return INCLUDE; + } + if (preflightPredicate == null) { + // ?from_sort_value specified, and we are sorting by SHARDS or FAILED_SHARDS that requires the full SnapshotInfo + return INCONCLUSIVE; + } + return preflightPredicate.test(snapshotId, repositoryData); + } + + boolean isMatchAll() { + return this == MATCH_ALL; + } + + /** + * Test using the full {@link SnapshotInfo}. + * + * @return an accurate result for {@link SnapshotSortKey#START_TIME} and {@link SnapshotSortKey#DURATION} (in case the pre-flight + * test was inconclusive) and for {@link SnapshotSortKey#SHARDS} and {@link SnapshotSortKey#FAILED_SHARDS} which are not available + * without loading the {@link SnapshotInfo}. Note that {@link SnapshotSortKey#NAME} and {@link SnapshotSortKey#INDICES} can always + * be checked accurately before loading the {@link SnapshotInfo} so they are not checked again here. + */ + boolean test(SnapshotInfo snapshotInfo) { + return snapshotPredicate == null || snapshotPredicate.test(snapshotInfo); + } + + static FromSortValuePredicates forFromSortValue(String fromSortValue, SnapshotSortKey sortBy, SortOrder order) { + if (fromSortValue == null) { + return MATCH_ALL; + } + + return switch (sortBy) { + case START_TIME -> { + final long after = Long.parseLong(fromSortValue); + yield new FromSortValuePredicates((snapshotId, repositoryData) -> { + final long startTime = getStartTime(snapshotId, repositoryData); + if (startTime == -1) { + return INCONCLUSIVE; + } + return order == SortOrder.ASC ? (after <= startTime ? INCLUDE : EXCLUDE) : (after >= startTime ? INCLUDE : EXCLUDE); + }, filterByLongOffset(SnapshotInfo::startTime, after, order)); + } + case NAME -> new FromSortValuePredicates( + order == SortOrder.ASC + ? (snapshotId, ignoredRepositoryData) -> fromSortValue.compareTo(snapshotId.getName()) <= 0 ? INCLUDE : EXCLUDE + : (snapshotId, ignoredRepositoryData) -> fromSortValue.compareTo(snapshotId.getName()) >= 0 ? INCLUDE : EXCLUDE, + null + ); + case DURATION -> { + final long afterDuration = Long.parseLong(fromSortValue); + yield new FromSortValuePredicates((snapshotId, repositoryData) -> { + final long duration = getDuration(snapshotId, repositoryData); + if (duration == -1) { + return INCONCLUSIVE; + } + return order == SortOrder.ASC + ? (afterDuration <= duration ? INCLUDE : EXCLUDE) + : (afterDuration >= duration ? INCLUDE : EXCLUDE); + }, filterByLongOffset(info -> info.endTime() - info.startTime(), afterDuration, order)); + } + case INDICES -> { + final int afterIndexCount = Integer.parseInt(fromSortValue); + yield new FromSortValuePredicates( + order == SortOrder.ASC + ? (snapshotId, repositoryData) -> afterIndexCount <= indexCount(snapshotId, repositoryData) ? INCLUDE : EXCLUDE + : (snapshotId, repositoryData) -> afterIndexCount >= indexCount(snapshotId, repositoryData) ? INCLUDE : EXCLUDE, + null + ); + } + case REPOSITORY -> MATCH_ALL; // already handled in GetSnapshotsOperation#skipRepository + case SHARDS -> new FromSortValuePredicates( + null, + filterByLongOffset(SnapshotInfo::totalShards, Integer.parseInt(fromSortValue), order) + ); + case FAILED_SHARDS -> new FromSortValuePredicates( + null, + filterByLongOffset(SnapshotInfo::failedShards, Integer.parseInt(fromSortValue), order) + ); + }; + } + + private static Predicate filterByLongOffset(ToLongFunction extractor, long after, SortOrder order) { + return order == SortOrder.ASC ? info -> after <= extractor.applyAsLong(info) : info -> after >= extractor.applyAsLong(info); + } + + private static long getDuration(SnapshotId snapshotId, RepositoryData repositoryData) { + final RepositoryData.SnapshotDetails details = repositoryData.getSnapshotDetails(snapshotId); + if (details == null) { + return -1; + } + final long startTime = details.getStartTimeMillis(); + if (startTime == -1) { + return -1; + } + final long endTime = details.getEndTimeMillis(); + if (endTime == -1) { + return -1; + } + return endTime - startTime; + } + + private static long getStartTime(SnapshotId snapshotId, RepositoryData repositoryData) { + final RepositoryData.SnapshotDetails details = repositoryData.getSnapshotDetails(snapshotId); + return details == null ? -1 : details.getStartTimeMillis(); + } + + private static int indexCount(SnapshotId snapshotId, RepositoryData repositoryData) { + // TODO: this could be made more efficient by caching this number in RepositoryData + int indexCount = 0; + for (IndexId idx : repositoryData.getIndices().values()) { + if (repositoryData.getSnapshots(idx).contains(snapshotId)) { + indexCount++; + } + } + return indexCount; + } +} diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsRequest.java index 6cd644d371d8f..f66372a93e4cc 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsRequest.java @@ -58,7 +58,7 @@ public class GetSnapshotsRequest extends MasterNodeRequest * Sort key value at which to start fetching snapshots. Mutually exclusive with {@link #offset} if not {@code null}. */ @Nullable - private SnapshotSortKey.After after; + private After after; @Nullable private String fromSortValue; @@ -112,7 +112,7 @@ public GetSnapshotsRequest(StreamInput in) throws IOException { snapshots = in.readStringArray(); ignoreUnavailable = in.readBoolean(); verbose = in.readBoolean(); - after = in.readOptionalWriteable(SnapshotSortKey.After::new); + after = in.readOptionalWriteable(After::new); sort = in.readEnum(SnapshotSortKey.class); size = in.readVInt(); order = SortOrder.readFromStream(in); @@ -298,7 +298,7 @@ public boolean includeIndexNames() { } @Nullable - public SnapshotSortKey.After after() { + public After after() { return after; } @@ -306,7 +306,7 @@ public SnapshotSortKey sort() { return sort; } - public GetSnapshotsRequest after(@Nullable SnapshotSortKey.After after) { + public GetSnapshotsRequest after(@Nullable After after) { this.after = after; return this; } diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsRequestBuilder.java index 176523a4725f0..4e3708ee57fa2 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsRequestBuilder.java @@ -114,10 +114,10 @@ public GetSnapshotsRequestBuilder setVerbose(boolean verbose) { } public GetSnapshotsRequestBuilder setAfter(String after) { - return setAfter(after == null ? null : SnapshotSortKey.decodeAfterQueryParam(after)); + return setAfter(after == null ? null : After.decodeAfterQueryParam(after)); } - public GetSnapshotsRequestBuilder setAfter(@Nullable SnapshotSortKey.After after) { + public GetSnapshotsRequestBuilder setAfter(@Nullable After after) { request.after(after); return this; } diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/SnapshotSortKey.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/SnapshotSortKey.java index 77f1584bf08d7..ba47bae8a5c2d 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/SnapshotSortKey.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/SnapshotSortKey.java @@ -9,20 +9,10 @@ package org.elasticsearch.action.admin.cluster.snapshots.get; -import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.io.stream.Writeable; -import org.elasticsearch.core.Nullable; -import org.elasticsearch.core.Predicates; import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.snapshots.SnapshotInfo; -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.util.Base64; import java.util.Comparator; -import java.util.function.Predicate; -import java.util.function.ToLongFunction; /** * Sort key for snapshots e.g. returned from the get-snapshots API. All values break ties using {@link SnapshotInfo#snapshotId} (i.e. by @@ -32,126 +22,37 @@ public enum SnapshotSortKey { /** * Sort by snapshot start time. */ - START_TIME("start_time", Comparator.comparingLong(SnapshotInfo::startTime)) { - @Override - protected String getSortKeyValue(SnapshotInfo snapshotInfo) { - return Long.toString(snapshotInfo.startTime()); - } - - @Override - protected Predicate innerGetAfterPredicate(After after, SortOrder sortOrder) { - return after.longValuePredicate(SnapshotInfo::startTime, sortOrder); - } - }, + START_TIME("start_time", Comparator.comparingLong(SnapshotInfo::startTime)), /** * Sort by snapshot name. */ - NAME("name", Comparator.comparing(sni -> sni.snapshotId().getName())) { - @Override - protected String getSortKeyValue(SnapshotInfo snapshotInfo) { - return snapshotInfo.snapshotId().getName(); - } - - @Override - protected Predicate innerGetAfterPredicate(After after, SortOrder sortOrder) { - // TODO: cover via pre-flight predicate - final String snapshotName = after.snapshotName(); - final String repoName = after.repoName(); - return sortOrder == SortOrder.ASC - ? (info -> compareName(snapshotName, repoName, info) < 0) - : (info -> compareName(snapshotName, repoName, info) > 0); - } - }, + NAME("name", Comparator.comparing(sni -> sni.snapshotId().getName())), /** * Sort by snapshot duration (end time minus start time). */ - DURATION("duration", Comparator.comparingLong(sni -> sni.endTime() - sni.startTime())) { - @Override - protected String getSortKeyValue(SnapshotInfo snapshotInfo) { - return Long.toString(snapshotInfo.endTime() - snapshotInfo.startTime()); - } - - @Override - protected Predicate innerGetAfterPredicate(After after, SortOrder sortOrder) { - return after.longValuePredicate(info -> info.endTime() - info.startTime(), sortOrder); - } - }, + DURATION("duration", Comparator.comparingLong(sni -> sni.endTime() - sni.startTime())), /** * Sort by number of indices in the snapshot. */ - INDICES("index_count", Comparator.comparingInt(sni -> sni.indices().size())) { - @Override - protected String getSortKeyValue(SnapshotInfo snapshotInfo) { - return Integer.toString(snapshotInfo.indices().size()); - } - - @Override - protected Predicate innerGetAfterPredicate(After after, SortOrder sortOrder) { - // TODO: cover via pre-flight predicate - return after.longValuePredicate(info -> info.indices().size(), sortOrder); - } - }, + INDICES("index_count", Comparator.comparingInt(sni -> sni.indices().size())), /** * Sort by number of shards in the snapshot. */ - SHARDS("shard_count", Comparator.comparingInt(SnapshotInfo::totalShards)) { - @Override - protected String getSortKeyValue(SnapshotInfo snapshotInfo) { - return Integer.toString(snapshotInfo.totalShards()); - } - - @Override - protected Predicate innerGetAfterPredicate(After after, SortOrder sortOrder) { - return after.longValuePredicate(SnapshotInfo::totalShards, sortOrder); - } - }, + SHARDS("shard_count", Comparator.comparingInt(SnapshotInfo::totalShards)), /** * Sort by number of failed shards in the snapshot. */ - FAILED_SHARDS("failed_shard_count", Comparator.comparingInt(SnapshotInfo::failedShards)) { - @Override - protected String getSortKeyValue(SnapshotInfo snapshotInfo) { - return Integer.toString(snapshotInfo.failedShards()); - } - - @Override - protected Predicate innerGetAfterPredicate(After after, SortOrder sortOrder) { - return after.longValuePredicate(SnapshotInfo::failedShards, sortOrder); - } - }, + FAILED_SHARDS("failed_shard_count", Comparator.comparingInt(SnapshotInfo::failedShards)), /** * Sort by repository name. */ - REPOSITORY("repository", Comparator.comparing(SnapshotInfo::repository)) { - @Override - protected String getSortKeyValue(SnapshotInfo snapshotInfo) { - return snapshotInfo.repository(); - } - - @Override - protected Predicate innerGetAfterPredicate(After after, SortOrder sortOrder) { - // TODO: cover via pre-flight predicate - final String snapshotName = after.snapshotName(); - final String repoName = after.repoName(); - return sortOrder == SortOrder.ASC - ? (info -> compareRepositoryName(snapshotName, repoName, info) < 0) - : (info -> compareRepositoryName(snapshotName, repoName, info) > 0); - } - - private static int compareRepositoryName(String name, String repoName, SnapshotInfo info) { - final int res = repoName.compareTo(info.repository()); - if (res != 0) { - return res; - } - return name.compareTo(info.snapshotId().getName()); - } - }; + REPOSITORY("repository", Comparator.comparing(SnapshotInfo::repository)); private final String name; private final Comparator ascendingSnapshotInfoComparator; @@ -178,59 +79,6 @@ public final Comparator getSnapshotInfoComparator(SortOrder sortOr }; } - /** - * @return an {@link After} which can be included in a {@link GetSnapshotsRequest} (e.g. to be sent to a remote node) and ultimately - * converted into a predicate to filter out {@link SnapshotInfo} items which were returned on earlier pages of results. See also - * {@link #encodeAfterQueryParam} and {@link #getAfterPredicate}. - */ - public static After decodeAfterQueryParam(String param) { - final String[] parts = new String(Base64.getUrlDecoder().decode(param), StandardCharsets.UTF_8).split(","); - if (parts.length != 3) { - throw new IllegalArgumentException("invalid ?after parameter [" + param + "]"); - } - return new After(parts[0], parts[1], parts[2]); - } - - /** - * @return an encoded representation of the value of the sort key for the given {@link SnapshotInfo}, including the values of the - * snapshot name and repo name for tiebreaking purposes, which can be returned to the user so they can pass it back to the - * {@code ?after} param of a subsequent call to the get-snapshots API in order to retrieve the next page of results. - */ - public final String encodeAfterQueryParam(SnapshotInfo snapshotInfo) { - final var rawValue = getSortKeyValue(snapshotInfo) + "," + snapshotInfo.repository() + "," + snapshotInfo.snapshotId().getName(); - return Base64.getUrlEncoder().encodeToString(rawValue.getBytes(StandardCharsets.UTF_8)); - } - - /** - * @return a string representation of the value of the sort key for the given {@link SnapshotInfo}, which should be the last item in the - * response, which is combined with the snapshot and repository names, encoded, and returned to the user so they can pass it back to - * the {@code ?after} param of a subsequent call to the get-snapshots API in order to retrieve the next page of results. - */ - protected abstract String getSortKeyValue(SnapshotInfo snapshotInfo); - - /** - * @return a predicate to filter out {@link SnapshotInfo} items that match the user's query but which sort earlier than the given - * {@link After} value (i.e. they were returned on earlier pages of results). If {@code after} is {@code null} then the returned - * predicate matches all snapshots. - */ - public final Predicate getAfterPredicate(@Nullable After after, SortOrder sortOrder) { - return after == null ? Predicates.always() : innerGetAfterPredicate(after, sortOrder); - } - - /** - * @return a predicate to filter out {@link SnapshotInfo} items that match the user's query but which sort earlier than the given - * {@link After} value (i.e. they were returned on earlier pages of results). The {@code after} parameter is not {@code null}. - */ - protected abstract Predicate innerGetAfterPredicate(After after, SortOrder sortOrder); - - private static int compareName(String name, String repoName, SnapshotInfo info) { - final int res = name.compareTo(info.snapshotId().getName()); - if (res != 0) { - return res; - } - return repoName.compareTo(info.repository()); - } - public static SnapshotSortKey of(String name) { return switch (name) { case "start_time" -> START_TIME; @@ -243,29 +91,4 @@ public static SnapshotSortKey of(String name) { default -> throw new IllegalArgumentException("unknown sort key [" + name + "]"); }; } - - public record After(String value, String repoName, String snapshotName) implements Writeable { - - After(StreamInput in) throws IOException { - this(in.readString(), in.readString(), in.readString()); - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - out.writeString(value); - out.writeString(repoName); - out.writeString(snapshotName); - } - - Predicate longValuePredicate(ToLongFunction extractor, SortOrder sortOrder) { - final var after = Long.parseLong(value); - return sortOrder == SortOrder.ASC ? info -> { - final long val = extractor.applyAsLong(info); - return after < val || (after == val && compareName(snapshotName, repoName, info) < 0); - } : info -> { - final long val = extractor.applyAsLong(info); - return after > val || (after == val && compareName(snapshotName, repoName, info) > 0); - }; - } - } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java index fa0fe750bb223..6fe4b12c5de1d 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java @@ -69,7 +69,6 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.function.BooleanSupplier; import java.util.function.Predicate; -import java.util.function.ToLongFunction; import static org.elasticsearch.action.admin.cluster.snapshots.get.PreflightFilterResult.EXCLUDE; import static org.elasticsearch.action.admin.cluster.snapshots.get.PreflightFilterResult.INCLUDE; @@ -268,7 +267,7 @@ private class GetSnapshotsOperation { // snapshots selection private final SnapshotNamePredicate snapshotNamePredicate; - private final SnapshotPredicates fromSortValuePredicates; + private final FromSortValuePredicates fromSortValuePredicates; private final Predicate slmPolicyPredicate; private final EnumSet states; private final boolean matchAllCompletedStates; @@ -278,7 +277,7 @@ private class GetSnapshotsOperation { private final SortOrder order; @Nullable private final String fromSortValue; - private final Predicate afterPredicate; + private final AfterPredicates afterPredicates; // current state private final SnapshotsInProgress snapshotsInProgress; @@ -310,7 +309,7 @@ private class GetSnapshotsOperation { SortOrder order, String fromSortValue, int offset, - SnapshotSortKey.After after, + After after, int size, SnapshotsInProgress snapshotsInProgress, boolean verbose, @@ -331,9 +330,9 @@ private class GetSnapshotsOperation { this.matchAllCompletedStates = states.containsAll(COMPLETED_STATES); this.snapshotNamePredicate = SnapshotNamePredicate.forSnapshots(ignoreUnavailable, snapshots); - this.fromSortValuePredicates = SnapshotPredicates.forFromSortValue(fromSortValue, sortBy, order); + this.fromSortValuePredicates = FromSortValuePredicates.forFromSortValue(fromSortValue, sortBy, order); this.slmPolicyPredicate = SlmPolicyPredicate.forPolicies(policies); - this.afterPredicate = sortBy.getAfterPredicate(after, order); + this.afterPredicates = AfterPredicates.forAfter(after, sortBy, order); this.getSnapshotInfoExecutor = new GetSnapshotInfoExecutor( threadPool.info(ThreadPool.Names.SNAPSHOT_META).getMax(), @@ -421,7 +420,7 @@ private void populateResults(ActionListener listener) { public void onResponse(SnapshotInfo snapshotInfo) { if (matchesPredicates(snapshotInfo)) { totalCount.incrementAndGet(); - if (afterPredicate.test(snapshotInfo)) { + if (afterPredicates.test(snapshotInfo)) { snapshotInfoCollector.add(snapshotInfo.maybeWithoutIndices(indices)); } } @@ -632,7 +631,7 @@ private GetSnapshotsResponse buildResponse() { final int remaining = snapshotInfoCollector.getRemaining(); return new GetSnapshotsResponse( snapshotInfos, - remaining > 0 ? sortBy.encodeAfterQueryParam(snapshotInfos.getLast()) : null, + remaining > 0 ? After.fromSnapshotInfo(snapshotInfos.getLast(), sortBy).toQueryParam() : null, totalCount.get(), remaining ); @@ -641,7 +640,7 @@ private GetSnapshotsResponse buildResponse() { private boolean assertSatisfiesAllPredicates(List snapshotInfos) { snapshotInfos.forEach(snapshotInfo -> { assert matchesPredicates(snapshotInfo); - assert afterPredicate.test(snapshotInfo); + assert afterPredicates.test(snapshotInfo); assert indices || snapshotInfo.indices().isEmpty(); }); return true; @@ -720,163 +719,6 @@ private boolean matchesPredicates(SnapshotInfo snapshotInfo) { } } - /** - * A pair of predicates for the get snapshots action. The {@link #test(SnapshotId, RepositoryData)} predicate is applied to combinations - * of snapshot id and repository data to determine which snapshots to fully load from the repository and rules out all snapshots that do - * not match the given {@link GetSnapshotsRequest} that can be ruled out through the information in {@link RepositoryData}. - * The predicate returned by {@link #test(SnapshotInfo)} predicate is then applied the instances of {@link SnapshotInfo} that were - * loaded from the repository to filter out those remaining that did not match the request but could not be ruled out without loading - * their {@link SnapshotInfo}. - */ - private static final class SnapshotPredicates { - - private static final SnapshotPredicates MATCH_ALL = new SnapshotPredicates(null, null); - - @Nullable // non-null for sort keys that can be filtered from repository data (NAME, INDICES, START_TIME, DURATION) - private final PreflightFilterResult.RepositoryDataFilter preflightPredicate; - - @Nullable // null if all snapshots match or the pre-flight filter is guaranteed to be conclusive so no SnapshotInfo filter needed - private final Predicate snapshotPredicate; - - private SnapshotPredicates( - @Nullable PreflightFilterResult.RepositoryDataFilter preflightPredicate, - @Nullable Predicate snapshotPredicate - ) { - this.snapshotPredicate = snapshotPredicate; - this.preflightPredicate = preflightPredicate; - } - - /** - * Pre-flight test using only snapshot id and repository data, i.e. without needing to load the full {@link SnapshotInfo}. - * - * @return an accurate result for {@link SnapshotSortKey#NAME}, {@link SnapshotSortKey#INDICES} and - * {@link SnapshotSortKey#REPOSITORY} (the latter because {@link GetSnapshotsOperation#skipRepository} handles this case even - * earlier). Also accurate for {@link SnapshotSortKey#START_TIME} and {@link SnapshotSortKey#DURATION} if and only if the - * corresponding values are available in {@link RepositoryData.SnapshotDetails}; returns {@link PreflightFilterResult#INCONCLUSIVE} - * otherwise. - * - * @see PreflightFilterResult.RepositoryDataFilter - */ - PreflightFilterResult test(SnapshotId snapshotId, RepositoryData repositoryData) { - if (this == MATCH_ALL) { - // no ?from_sort_value parameter, or sorting by REPOSITORY - return INCLUDE; - } - if (preflightPredicate == null) { - // ?from_sort_value specified, and we are sorting by SHARDS or FAILED_SHARDS that requires the full SnapshotInfo - return INCONCLUSIVE; - } - return preflightPredicate.test(snapshotId, repositoryData); - } - - boolean isMatchAll() { - return snapshotPredicate == null; - } - - /** - * Test using the full {@link SnapshotInfo}. - * - * @return an accurate result for {@link SnapshotSortKey#START_TIME} and {@link SnapshotSortKey#DURATION} (in case the pre-flight - * test was inconclusive) and for {@link SnapshotSortKey#SHARDS} and {@link SnapshotSortKey#FAILED_SHARDS} which are not available - * without loading the {@link SnapshotInfo}. Note that {@link SnapshotSortKey#NAME} and {@link SnapshotSortKey#INDICES} can always - * be checked accurately before loading the {@link SnapshotInfo} so they are not checked again here. - */ - boolean test(SnapshotInfo snapshotInfo) { - return snapshotPredicate == null || snapshotPredicate.test(snapshotInfo); - } - - static SnapshotPredicates forFromSortValue(String fromSortValue, SnapshotSortKey sortBy, SortOrder order) { - if (fromSortValue == null) { - return MATCH_ALL; - } - - return switch (sortBy) { - case START_TIME -> { - final long after = Long.parseLong(fromSortValue); - yield new SnapshotPredicates((snapshotId, repositoryData) -> { - final long startTime = getStartTime(snapshotId, repositoryData); - if (startTime == -1) { - return INCONCLUSIVE; - } - return order == SortOrder.ASC ? (after <= startTime ? INCLUDE : EXCLUDE) : (after >= startTime ? INCLUDE : EXCLUDE); - }, filterByLongOffset(SnapshotInfo::startTime, after, order)); - } - case NAME -> new SnapshotPredicates( - order == SortOrder.ASC - ? (snapshotId, ignoredRepositoryData) -> fromSortValue.compareTo(snapshotId.getName()) <= 0 ? INCLUDE : EXCLUDE - : (snapshotId, ignoredRepositoryData) -> fromSortValue.compareTo(snapshotId.getName()) >= 0 ? INCLUDE : EXCLUDE, - null - ); - case DURATION -> { - final long afterDuration = Long.parseLong(fromSortValue); - yield new SnapshotPredicates((snapshotId, repositoryData) -> { - final long duration = getDuration(snapshotId, repositoryData); - if (duration == -1) { - return INCONCLUSIVE; - } - return order == SortOrder.ASC - ? (afterDuration <= duration ? INCLUDE : EXCLUDE) - : (afterDuration >= duration ? INCLUDE : EXCLUDE); - }, filterByLongOffset(info -> info.endTime() - info.startTime(), afterDuration, order)); - } - case INDICES -> { - final int afterIndexCount = Integer.parseInt(fromSortValue); - yield new SnapshotPredicates( - order == SortOrder.ASC - ? (snapshotId, repositoryData) -> afterIndexCount <= indexCount(snapshotId, repositoryData) ? INCLUDE : EXCLUDE - : (snapshotId, repositoryData) -> afterIndexCount >= indexCount(snapshotId, repositoryData) ? INCLUDE : EXCLUDE, - null - ); - } - case REPOSITORY -> MATCH_ALL; // already handled in GetSnapshotsOperation#skipRepository - case SHARDS -> new SnapshotPredicates( - null, - filterByLongOffset(SnapshotInfo::totalShards, Integer.parseInt(fromSortValue), order) - ); - case FAILED_SHARDS -> new SnapshotPredicates( - null, - filterByLongOffset(SnapshotInfo::failedShards, Integer.parseInt(fromSortValue), order) - ); - }; - } - - private static Predicate filterByLongOffset(ToLongFunction extractor, long after, SortOrder order) { - return order == SortOrder.ASC ? info -> after <= extractor.applyAsLong(info) : info -> after >= extractor.applyAsLong(info); - } - - private static long getDuration(SnapshotId snapshotId, RepositoryData repositoryData) { - final RepositoryData.SnapshotDetails details = repositoryData.getSnapshotDetails(snapshotId); - if (details == null) { - return -1; - } - final long startTime = details.getStartTimeMillis(); - if (startTime == -1) { - return -1; - } - final long endTime = details.getEndTimeMillis(); - if (endTime == -1) { - return -1; - } - return endTime - startTime; - } - - private static long getStartTime(SnapshotId snapshotId, RepositoryData repositoryData) { - final RepositoryData.SnapshotDetails details = repositoryData.getSnapshotDetails(snapshotId); - return details == null ? -1 : details.getStartTimeMillis(); - } - - private static int indexCount(SnapshotId snapshotId, RepositoryData repositoryData) { - // TODO: this could be made more efficient by caching this number in RepositoryData - int indexCount = 0; - for (IndexId idx : repositoryData.getIndices().values()) { - if (repositoryData.getSnapshots(idx).contains(snapshotId)) { - indexCount++; - } - } - return indexCount; - } - } - /** * Throttling executor for retrieving {@link SnapshotInfo} instances from the repository without spamming the SNAPSHOT_META threadpool * and starving other users of access to it. Similar to {@link Repository#getSnapshotInfo} but allows for finer-grained control over diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestGetSnapshotsAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestGetSnapshotsAction.java index 4b58fae1741c7..b75de31b2ab47 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestGetSnapshotsAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestGetSnapshotsAction.java @@ -9,6 +9,7 @@ package org.elasticsearch.rest.action.admin.cluster; +import org.elasticsearch.action.admin.cluster.snapshots.get.After; import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsRequest; import org.elasticsearch.action.admin.cluster.snapshots.get.SnapshotSortKey; import org.elasticsearch.client.internal.node.NodeClient; @@ -115,7 +116,7 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC getSnapshotsRequest.offset(offset); final String afterString = request.param("after"); if (afterString != null) { - getSnapshotsRequest.after(SnapshotSortKey.decodeAfterQueryParam(afterString)); + getSnapshotsRequest.after(After.decodeAfterQueryParam(afterString)); } final String fromSortValue = request.param("from_sort_value"); if (fromSortValue != null) { diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/get/AfterPredicatesTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/get/AfterPredicatesTests.java new file mode 100644 index 0000000000000..1c83695cb4547 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/get/AfterPredicatesTests.java @@ -0,0 +1,115 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.action.admin.cluster.snapshots.get; + +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.elasticsearch.search.sort.SortOrder; +import org.elasticsearch.snapshots.SnapshotInfo; +import org.elasticsearch.snapshots.SnapshotInfoTestUtils; +import org.elasticsearch.test.ESTestCase; +import org.junit.Before; + +import java.util.Arrays; + +import static org.elasticsearch.action.admin.cluster.snapshots.get.After.fromSnapshotInfo; +import static org.elasticsearch.search.sort.SortOrder.DESC; +import static org.elasticsearch.snapshots.SnapshotInfoTestUtils.createRandomSnapshotInfo; + +public class AfterPredicatesTests extends ESTestCase { + + private final SnapshotSortKey sortBy; + private final SortOrder order; + + private SnapshotInfo info1; + private SnapshotInfo info2; + + private After after1; + private After after2; + + public AfterPredicatesTests(SnapshotSortKey sortBy, SortOrder order) { + this.sortBy = sortBy; + this.order = order; + } + + @ParametersFactory(argumentFormatting = "sortBy=%s order=%s") + public static Iterable parameters() { + return Arrays.stream(SnapshotSortKey.values()) + .flatMap(k -> Arrays.stream(SortOrder.values()).map(o -> new Object[] { k, o })) + .toList(); + } + + @Before + public void setUpSnapshotInfos() { + final var infoA = createRandomSnapshotInfo(); + final var infoB = randomValueOtherThanMany( + info -> infoA.startTime() == info.startTime() + || infoA.snapshotId().getName().equals(info.snapshotId().getName()) + || infoA.endTime() - infoA.startTime() == info.endTime() - info.startTime() + || infoA.indices().size() == info.indices().size() + || infoA.totalShards() == info.totalShards() + || infoA.failedShards() == info.failedShards() + || infoA.repository().equals(info.repository()), + SnapshotInfoTestUtils::createRandomSnapshotInfo + ); + final int comparison = sortBy.getSnapshotInfoComparator(order).compare(infoA, infoB); + assertNotEquals(0, comparison); + info1 = comparison < 0 ? infoA : infoB; + info2 = comparison < 0 ? infoB : infoA; + after1 = fromSnapshotInfo(info1, sortBy); + after2 = fromSnapshotInfo(info2, sortBy); + } + + private void verifyPredicates(After after, boolean expectInfo1Included, boolean expectInfo2Included) { + final var predicates = AfterPredicates.forAfter(after, sortBy, order); + assertEquals(expectInfo1Included, predicates.test(info1)); + assertEquals(expectInfo2Included, predicates.test(info2)); + } + + public void testMatchAll() { + verifyPredicates(null, true, true); + } + + public void testAfterFirst() { + verifyPredicates(after1, false, true); + } + + public void testAfterSecond() { + verifyPredicates(after2, false, false); + } + + public void testSnapshotNameTiebreak() { + if (sortBy == SnapshotSortKey.NAME) { + return; + } + + verifyPredicates( + new After( + after1.value(), + sortBy == SnapshotSortKey.REPOSITORY ? after1.repoName() : randomRepoName(), + stringBefore(after1.snapshotName()) + ), + true, + true + ); + } + + public void testRepositoryNameTiebreak() { + if (sortBy == SnapshotSortKey.REPOSITORY) { + return; + } + + verifyPredicates(new After(after1.value(), stringBefore(after1.repoName()), after1.snapshotName()), true, true); + } + + private String stringBefore(String candidate) { + return order == DESC ? candidate + "z" : ""; + } +} diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/get/AfterTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/get/AfterTests.java new file mode 100644 index 0000000000000..db0fa1c38481b --- /dev/null +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/get/AfterTests.java @@ -0,0 +1,82 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.action.admin.cluster.snapshots.get; + +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractWireSerializingTestCase; +import org.elasticsearch.test.ESTestCase; + +import java.nio.charset.StandardCharsets; +import java.util.Base64; + +import static org.elasticsearch.action.admin.cluster.snapshots.get.After.fromSnapshotInfo; +import static org.elasticsearch.snapshots.SnapshotInfoTestUtils.createRandomSnapshotInfo; +import static org.hamcrest.Matchers.equalTo; + +public class AfterTests extends AbstractWireSerializingTestCase { + + @Override + protected Writeable.Reader instanceReader() { + return After::new; + } + + @Override + protected After createTestInstance() { + return new After(randomAlphaOfLengthBetween(1, 20), randomRepoName(), randomSnapshotName()); + } + + @Override + protected After mutateInstance(final After instance) { + return switch (between(0, 2)) { + case 0 -> new After( + randomValueOtherThan(instance.value(), () -> randomAlphaOfLengthBetween(1, 20)), + instance.repoName(), + instance.snapshotName() + ); + case 1 -> new After( + instance.value(), + randomValueOtherThan(instance.repoName(), ESTestCase::randomRepoName), + instance.snapshotName() + ); + case 2 -> new After( + instance.value(), + instance.repoName(), + randomValueOtherThan(instance.snapshotName(), ESTestCase::randomSnapshotName) + ); + default -> throw new AssertionError("impossible"); + }; + } + + public void testRoundTripToQueryParam() { + final var after = createTestInstance(); + assertThat(After.decodeAfterQueryParam(after.toQueryParam()), equalTo(after)); + } + + public void testDecodeAfterQueryParamInvalidFormatThrows() { + expectThrows(IllegalArgumentException.class, () -> After.decodeAfterQueryParam("not-valid-base64!!!")); + expectThrows( + IllegalArgumentException.class, + () -> After.decodeAfterQueryParam(Base64.getUrlEncoder().encodeToString("only,two".getBytes(StandardCharsets.UTF_8))) + ); + } + + public void testFromSnapshotInfo() { + final var info = createRandomSnapshotInfo(); + assertThat(fromSnapshotInfo(info, randomFrom(SnapshotSortKey.values())).repoName(), equalTo(info.repository())); + assertThat(fromSnapshotInfo(info, randomFrom(SnapshotSortKey.values())).snapshotName(), equalTo(info.snapshotId().getName())); + assertThat(fromSnapshotInfo(info, SnapshotSortKey.START_TIME).value(), equalTo(Long.toString(info.startTime()))); + assertThat(fromSnapshotInfo(info, SnapshotSortKey.NAME).value(), equalTo(info.snapshotId().getName())); + assertThat(fromSnapshotInfo(info, SnapshotSortKey.DURATION).value(), equalTo(Long.toString(info.endTime() - info.startTime()))); + assertThat(fromSnapshotInfo(info, SnapshotSortKey.INDICES).value(), equalTo(Integer.toString(info.indices().size()))); + assertThat(fromSnapshotInfo(info, SnapshotSortKey.SHARDS).value(), equalTo(Integer.toString(info.totalShards()))); + assertThat(fromSnapshotInfo(info, SnapshotSortKey.FAILED_SHARDS).value(), equalTo(Integer.toString(info.failedShards()))); + assertThat(fromSnapshotInfo(info, SnapshotSortKey.REPOSITORY).value(), equalTo(info.repository())); + } +} diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/get/FromSortValuePredicatesTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/get/FromSortValuePredicatesTests.java new file mode 100644 index 0000000000000..367029a71500b --- /dev/null +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/get/FromSortValuePredicatesTests.java @@ -0,0 +1,213 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.action.admin.cluster.snapshots.get; + +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.elasticsearch.common.UUIDs; +import org.elasticsearch.repositories.IndexId; +import org.elasticsearch.repositories.IndexMetaDataGenerations; +import org.elasticsearch.repositories.RepositoryData; +import org.elasticsearch.repositories.ShardGenerations; +import org.elasticsearch.search.sort.SortOrder; +import org.elasticsearch.snapshots.SnapshotId; +import org.elasticsearch.snapshots.SnapshotInfo; +import org.elasticsearch.snapshots.SnapshotInfoTestUtils; +import org.elasticsearch.test.ESTestCase; +import org.junit.Before; + +import java.util.Arrays; +import java.util.EnumSet; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +import static org.elasticsearch.action.admin.cluster.snapshots.get.After.fromSnapshotInfo; +import static org.elasticsearch.action.admin.cluster.snapshots.get.PreflightFilterResult.EXCLUDE; +import static org.elasticsearch.action.admin.cluster.snapshots.get.PreflightFilterResult.INCLUDE; +import static org.elasticsearch.action.admin.cluster.snapshots.get.PreflightFilterResult.INCONCLUSIVE; +import static org.elasticsearch.snapshots.SnapshotInfoTestUtils.createRandomSnapshotInfo; + +public class FromSortValuePredicatesTests extends ESTestCase { + + // exclude SHARDS and FAILED_SHARDS in the preflight cases since these require SnapshotInfo to be loaded + private static final EnumSet PREFLIGHT_TEST_KEYS = EnumSet.of( + SnapshotSortKey.NAME, + SnapshotSortKey.START_TIME, + SnapshotSortKey.INDICES, + SnapshotSortKey.DURATION + ); + + // preflight cases which might or might not work depending on whether SnapshotDetails is present + private static final EnumSet PREFLIGHT_UNRELIABLE_TEST_KEYS = EnumSet.of( + SnapshotSortKey.START_TIME, + SnapshotSortKey.DURATION + ); + + // SHARDS and FAILED_SHARDS always require SnapshotInfo to be loaded + private static final EnumSet PREFLIGHT_INCONCLUSIVE_TEST_KEYS = EnumSet.of( + SnapshotSortKey.SHARDS, + SnapshotSortKey.FAILED_SHARDS + ); + + // exclude NAME/REPOSITORY/INDICES in the SnapshotInfo cases since these are reliably handled by the pre-flight test + private static final EnumSet SNAPSHOT_INFO_TEST_KEYS = EnumSet.of( + SnapshotSortKey.START_TIME, + SnapshotSortKey.DURATION, + SnapshotSortKey.SHARDS, + SnapshotSortKey.FAILED_SHARDS + ); + + private final SnapshotSortKey sortBy; + private final SortOrder order; + + private SnapshotInfo info1; + private SnapshotInfo info2; + + private String sortValue1; + private String sortValue2; + + private RepositoryData repositoryData1NoDetails; + private RepositoryData repositoryData2NoDetails; + + private RepositoryData repositoryData1; + private RepositoryData repositoryData2; + private FromSortValuePredicates predicates1; + private FromSortValuePredicates predicates2; + + public FromSortValuePredicatesTests(SnapshotSortKey sortBy, SortOrder order) { + this.sortBy = sortBy; + this.order = order; + } + + @ParametersFactory(argumentFormatting = "sortBy=%s order=%s") + public static Iterable parameters() { + return Arrays.stream(SnapshotSortKey.values()) + .flatMap(k -> Arrays.stream(SortOrder.values()).map(o -> new Object[] { k, o })) + .toList(); + } + + @Before + public void setUpSnapshotInfos() { + final var infoA = createRandomSnapshotInfo(); + final var infoB = randomValueOtherThanMany( + info -> infoA.startTime() == info.startTime() + || infoA.snapshotId().getName().equals(info.snapshotId().getName()) + || infoA.endTime() - infoA.startTime() == info.endTime() - info.startTime() + || infoA.indices().size() == info.indices().size() + || infoA.totalShards() == info.totalShards() + || infoA.failedShards() == info.failedShards() + || infoA.repository().equals(info.repository()), + SnapshotInfoTestUtils::createRandomSnapshotInfo + ); + final int comparison = sortBy.getSnapshotInfoComparator(order).compare(infoA, infoB); + assertNotEquals(0, comparison); + info1 = comparison < 0 ? infoA : infoB; + info2 = comparison < 0 ? infoB : infoA; + + sortValue1 = fromSnapshotInfo(info1, sortBy).value(); + sortValue2 = fromSnapshotInfo(info2, sortBy).value(); + + predicates1 = getPredicates(sortValue1); + predicates2 = getPredicates(sortValue2); + + repositoryData1NoDetails = buildRepositoryDataNoDetails(info1); + repositoryData2NoDetails = buildRepositoryDataNoDetails(info2); + + repositoryData1 = repositoryData1NoDetails.withExtraDetails(extraDetailsMap(info1)); + repositoryData2 = repositoryData2NoDetails.withExtraDetails(extraDetailsMap(info2)); + } + + private Map extraDetailsMap(SnapshotInfo info1) { + return Map.of(info1.snapshotId(), RepositoryData.SnapshotDetails.fromSnapshotInfo(info1)); + } + + private RepositoryData buildRepositoryDataNoDetails(SnapshotInfo snapshotInfo) { + return new RepositoryData( + randomUUID(), + randomNonNegativeLong(), + Map.of(snapshotInfo.snapshotId().getUUID(), snapshotInfo.snapshotId()), + randomFrom(Map.of(), Map.of(snapshotInfo.snapshotId().getUUID(), RepositoryData.SnapshotDetails.EMPTY)), + IntStream.range(0, snapshotInfo.indices().size() + 1) + .boxed() + .collect( + Collectors.toMap( + i -> new IndexId("idx-" + i, UUIDs.randomBase64UUID()), + i -> i < snapshotInfo.indices().size() + ? List.of(snapshotInfo.snapshotId()) + : List.of(new SnapshotId(randomSnapshotName(), randomUUID())) + ) + ), + ShardGenerations.EMPTY, + IndexMetaDataGenerations.EMPTY, + randomUUID() + ); + } + + private FromSortValuePredicates getPredicates(String fromSortValue) { + return FromSortValuePredicates.forFromSortValue(fromSortValue, sortBy, order); + } + + public void testSnapshotInfoPredicate() { + assertTrue(predicates1.test(info1)); + assertTrue(predicates1.test(info2)); + assertEquals(SNAPSHOT_INFO_TEST_KEYS.contains(sortBy) == false, predicates2.test(info1)); + assertTrue(predicates2.test(info2)); + } + + public void testPreflightConclusive() { + if (PREFLIGHT_TEST_KEYS.contains(sortBy)) { + assertFalse(predicates1.isMatchAll()); + assertFalse(predicates2.isMatchAll()); + + assertEquals(INCLUDE, predicates1.test(info1.snapshotId(), repositoryData1)); + assertEquals(INCLUDE, predicates1.test(info2.snapshotId(), repositoryData2)); + assertEquals(EXCLUDE, predicates2.test(info1.snapshotId(), repositoryData1)); + assertEquals(INCLUDE, predicates2.test(info2.snapshotId(), repositoryData2)); + + if (PREFLIGHT_UNRELIABLE_TEST_KEYS.contains(sortBy) == false) { + assertEquals(INCLUDE, predicates1.test(info1.snapshotId(), repositoryData1NoDetails)); + assertEquals(INCLUDE, predicates1.test(info2.snapshotId(), repositoryData2NoDetails)); + assertEquals(EXCLUDE, predicates2.test(info1.snapshotId(), repositoryData1NoDetails)); + assertEquals(INCLUDE, predicates2.test(info2.snapshotId(), repositoryData2NoDetails)); + } + } + } + + public void testPreflightInconclusive() { + if (PREFLIGHT_UNRELIABLE_TEST_KEYS.contains(sortBy) || PREFLIGHT_INCONCLUSIVE_TEST_KEYS.contains(sortBy)) { + assertFalse(predicates1.isMatchAll()); + assertFalse(predicates2.isMatchAll()); + assertEquals(INCONCLUSIVE, predicates1.test(info1.snapshotId(), repositoryData1NoDetails)); + assertEquals(INCONCLUSIVE, predicates1.test(info2.snapshotId(), repositoryData2NoDetails)); + assertEquals(INCONCLUSIVE, predicates2.test(info1.snapshotId(), repositoryData1NoDetails)); + assertEquals(INCONCLUSIVE, predicates2.test(info2.snapshotId(), repositoryData2NoDetails)); + } + + if (PREFLIGHT_INCONCLUSIVE_TEST_KEYS.contains(sortBy)) { + assertEquals(INCONCLUSIVE, predicates1.test(info1.snapshotId(), repositoryData1)); + assertEquals(INCONCLUSIVE, predicates1.test(info2.snapshotId(), repositoryData2)); + assertEquals(INCONCLUSIVE, predicates2.test(info1.snapshotId(), repositoryData1)); + assertEquals(INCONCLUSIVE, predicates2.test(info2.snapshotId(), repositoryData2)); + } + } + + public void testPreflightMatchAllWhenNullOrRepositorySort() { + final var predicates = getPredicates( + sortBy == SnapshotSortKey.REPOSITORY ? randomFrom(sortValue1, sortValue2, randomRepoName()) : null + ); + assertTrue(predicates.isMatchAll()); + assertEquals(INCLUDE, predicates.test(info1.snapshotId(), randomFrom(repositoryData1, repositoryData1NoDetails))); + assertEquals(INCLUDE, predicates.test(info2.snapshotId(), randomFrom(repositoryData2, repositoryData2NoDetails))); + assertTrue(predicates.test(info1)); + assertTrue(predicates.test(info2)); + } +} diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsRequestTests.java index abeb70e76349e..aec6f9da459d1 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsRequestTests.java @@ -62,7 +62,7 @@ public void testValidateParameters() { } { final GetSnapshotsRequest request = new GetSnapshotsRequest(TEST_REQUEST_TIMEOUT, "repo", "snapshot").verbose(false) - .after(new SnapshotSortKey.After("foo", "repo", "bar")); + .after(new After("foo", "repo", "bar")); final ActionRequestValidationException e = request.validate(); assertThat(e.getMessage(), containsString("can't use after with verbose=false")); } @@ -74,14 +74,14 @@ public void testValidateParameters() { } { final GetSnapshotsRequest request = new GetSnapshotsRequest(TEST_REQUEST_TIMEOUT, "repo", "snapshot").after( - new SnapshotSortKey.After("foo", "repo", "bar") + new After("foo", "repo", "bar") ).offset(randomIntBetween(1, 500)); final ActionRequestValidationException e = request.validate(); assertThat(e.getMessage(), containsString("can't use after and offset simultaneously")); } { final GetSnapshotsRequest request = new GetSnapshotsRequest(TEST_REQUEST_TIMEOUT, "repo", "snapshot").fromSortValue("foo") - .after(new SnapshotSortKey.After("foo", "repo", "bar")); + .after(new After("foo", "repo", "bar")); final ActionRequestValidationException e = request.validate(); assertThat(e.getMessage(), containsString("can't use after and from_sort_value simultaneously")); }