Skip to content

Remove legacy MetaDataStateFormat #31603

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 6 commits into from
Jun 27, 2018
Merged

Remove legacy MetaDataStateFormat #31603

Changes from 1 commit
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
c7011a6
Remove legacy MetaDataStateFormat
ywelsch Jun 27, 2018
ec76043
fix tests
ywelsch Jun 27, 2018
88eac6f
Remove more legacy stuff
ywelsch Jun 27, 2018
649f2f2
expectThrows
ywelsch Jun 27, 2018
da6350b
fix test that failed because it used legacy format
ywelsch Jun 27, 2018
ba97446
remove legacy indices
ywelsch Jun 27, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
76 changes: 14 additions & 62 deletions server/src/main/java/org/elasticsearch/gateway/MetaDataStateFormat.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,19 +29,17 @@
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.OutputStreamIndexOutput;
import org.apache.lucene.store.SimpleFSDirectory;
import org.elasticsearch.common.logging.Loggers;
import org.elasticsearch.core.internal.io.IOUtils;
import org.elasticsearch.ExceptionsHelper;
import org.elasticsearch.common.bytes.BytesArray;
import org.elasticsearch.common.logging.Loggers;
import org.elasticsearch.common.lucene.store.IndexOutputOutputStream;
import org.elasticsearch.common.lucene.store.InputStreamIndexInput;
import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.common.xcontent.XContentHelper;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.core.internal.io.IOUtils;

import java.io.FileNotFoundException;
import java.io.IOException;
Expand All @@ -54,7 +52,6 @@
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.function.Predicate;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
Expand Down Expand Up @@ -263,10 +260,9 @@ long findMaxStateId(final String prefix, Path... locations) throws IOException {
* @param dataLocations the data-locations to try.
* @return the latest state or <code>null</code> if no state was found.
*/
public T loadLatestState(Logger logger, NamedXContentRegistry namedXContentRegistry, Path... dataLocations) throws IOException {
public T loadLatestState(Logger logger, NamedXContentRegistry namedXContentRegistry, Path... dataLocations) throws IOException {
List<PathAndStateId> files = new ArrayList<>();
long maxStateId = -1;
boolean maxStateIdIsLegacy = true;
if (dataLocations != null) { // select all eligible files first
for (Path dataLocation : dataLocations) {
final Path stateDir = dataLocation.resolve(STATE_DIR_NAME);
Expand All @@ -280,9 +276,7 @@ public T loadLatestState(Logger logger, NamedXContentRegistry namedXContentRegi
if (matcher.matches()) {
final long stateId = Long.parseLong(matcher.group(1));
maxStateId = Math.max(maxStateId, stateId);
final boolean legacy = MetaDataStateFormat.STATE_FILE_EXTENSION.equals(matcher.group(2)) == false;
maxStateIdIsLegacy &= legacy; // on purpose, see NOTE below
PathAndStateId pav = new PathAndStateId(stateFile, stateId, legacy);
PathAndStateId pav = new PathAndStateId(stateFile, stateId);
logger.trace("found state file: {}", pav);
files.add(pav);
}
Expand All @@ -292,39 +286,19 @@ public T loadLatestState(Logger logger, NamedXContentRegistry namedXContentRegi
}
}
}
final List<Throwable> exceptions = new ArrayList<>();
T state = null;
// NOTE: we might have multiple version of the latest state if there are multiple data dirs.. for this case
// we iterate only over the ones with the max version. If we have at least one state file that uses the
// new format (ie. legacy == false) then we know that the latest version state ought to use this new format.
// In case the state file with the latest version does not use the new format while older state files do,
// the list below will be empty and loading the state will fail
// we iterate only over the ones with the max version.
long finalMaxStateId = maxStateId;
Collection<PathAndStateId> pathAndStateIds = files
.stream()
.filter(new StateIdAndLegacyPredicate(maxStateId, maxStateIdIsLegacy))
.filter(pathAndStateId -> pathAndStateId.id == finalMaxStateId)
.collect(Collectors.toCollection(ArrayList::new));

final List<Throwable> exceptions = new ArrayList<>();
for (PathAndStateId pathAndStateId : pathAndStateIds) {
try {
final Path stateFile = pathAndStateId.file;
final long id = pathAndStateId.id;
if (pathAndStateId.legacy) { // read the legacy format -- plain XContent
final byte[] data = Files.readAllBytes(stateFile);
if (data.length == 0) {
logger.debug("{}: no data for [{}], ignoring...", prefix, stateFile.toAbsolutePath());
continue;
}
try (XContentParser parser = XContentHelper
.createParser(namedXContentRegistry, LoggingDeprecationHandler.INSTANCE, new BytesArray(data))) {
state = fromXContent(parser);
}
if (state == null) {
logger.debug("{}: no data for [{}], ignoring...", prefix, stateFile.toAbsolutePath());
Copy link
Contributor

@DaveCTurner DaveCTurner Jun 27, 2018

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Except we didn't ignore it, we returned null straight away...

Copy link
Contributor Author

@ywelsch ywelsch Jun 27, 2018

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

details ;)

}
} else {
state = read(namedXContentRegistry, stateFile);
logger.trace("state id [{}] read from [{}]", id, stateFile.getFileName());
}
T state = read(namedXContentRegistry, pathAndStateId.file);
logger.trace("state id [{}] read from [{}]", pathAndStateId.id, pathAndStateId.file.getFileName());
return state;
} catch (Exception e) {
exceptions.add(new IOException("failed to read " + pathAndStateId.toString(), e));
Expand All @@ -338,46 +312,24 @@ public T loadLatestState(Logger logger, NamedXContentRegistry namedXContentRegi
// We have some state files but none of them gave us a usable state
throw new IllegalStateException("Could not find a state file to recover from among " + files);
}
return state;
}

/**
* Filters out all {@link org.elasticsearch.gateway.MetaDataStateFormat.PathAndStateId} instances with a different id than
* the given one.
*/
private static final class StateIdAndLegacyPredicate implements Predicate<PathAndStateId> {
private final long id;
private final boolean legacy;

StateIdAndLegacyPredicate(long id, boolean legacy) {
this.id = id;
this.legacy = legacy;
}

@Override
public boolean test(PathAndStateId input) {
return input.id == id && input.legacy == legacy;
}
return null;
}

/**
* Internal struct-like class that holds the parsed state id, the file
* and a flag if the file is a legacy state ie. pre 1.5
* Internal struct-like class that holds the parsed state id and the file
*/
private static class PathAndStateId {
final Path file;
final long id;
final boolean legacy;

private PathAndStateId(Path file, long id, boolean legacy) {
private PathAndStateId(Path file, long id) {
this.file = file;
this.id = id;
this.legacy = legacy;
}

@Override
public String toString() {
return "[id:" + id + ", legacy:" + legacy + ", file:" + file.toAbsolutePath() + "]";
return "[id:" + id + ", file:" + file.toAbsolutePath() + "]";
}
}

Expand Down