Skip to content

[CCR] Added auto_follow_exception.timestamp field to auto follow stats #36947

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
martijnvg merged 3 commits into from
Dec 24, 2018
Merged

[CCR] Added auto_follow_exception.timestamp field to auto follow stats #36947

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
a759b33
[CCR] Added `auto_follow_exception.timestamp` field to auto follow st…
martijnvg Dec 21, 2018
f79f31b
use absolte time!
martijnvg Dec 22, 2018
613941d
fixed checkstyle violation
martijnvg Dec 22, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 9 additions & 6 deletions client/rest-high-level/src/main/java/org/elasticsearch/client/ccr/AutoFollowStats.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@

import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.common.ParseField;
import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.common.xcontent.ConstructingObjectParser;

import java.util.AbstractMap;
Expand All @@ -38,6 +39,7 @@ public final class AutoFollowStats {
new ParseField("number_of_failed_remote_cluster_state_requests");
static final ParseField RECENT_AUTO_FOLLOW_ERRORS = new ParseField("recent_auto_follow_errors");
static final ParseField LEADER_INDEX = new ParseField("leader_index");
static final ParseField TIMESTAMP = new ParseField("timestamp");
static final ParseField AUTO_FOLLOW_EXCEPTION = new ParseField("auto_follow_exception");
static final ParseField AUTO_FOLLOWED_CLUSTERS = new ParseField("auto_followed_clusters");
static final ParseField CLUSTER_NAME = new ParseField("cluster_name");
Expand All @@ -51,7 +53,7 @@ public final class AutoFollowStats {
(Long) args[1],
(Long) args[2],
new TreeMap<>(
((List<Map.Entry<String, ElasticsearchException>>) args[3])
((List<Map.Entry<String, Tuple<Long, ElasticsearchException>>>) args[3])
.stream()
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue))),
new TreeMap<>(
Expand All @@ -60,10 +62,10 @@ public final class AutoFollowStats {
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)))
));

private static final ConstructingObjectParser<Map.Entry<String, ElasticsearchException>, Void> AUTO_FOLLOW_EXCEPTIONS_PARSER =
static final ConstructingObjectParser<Map.Entry<String, Tuple<Long, ElasticsearchException>>, Void> AUTO_FOLLOW_EXCEPTIONS_PARSER =
new ConstructingObjectParser<>(
"auto_follow_stats_errors",
args -> new AbstractMap.SimpleEntry<>((String) args[0], (ElasticsearchException) args[1]));
args -> new AbstractMap.SimpleEntry<>((String) args[0], Tuple.tuple((Long) args[1], (ElasticsearchException) args[2])));

private static final ConstructingObjectParser<Map.Entry<String, AutoFollowedCluster>, Void> AUTO_FOLLOWED_CLUSTERS_PARSER =
new ConstructingObjectParser<>(
Expand All @@ -72,6 +74,7 @@ public final class AutoFollowStats {

static {
AUTO_FOLLOW_EXCEPTIONS_PARSER.declareString(ConstructingObjectParser.constructorArg(), LEADER_INDEX);
AUTO_FOLLOW_EXCEPTIONS_PARSER.declareLong(ConstructingObjectParser.constructorArg(), TIMESTAMP);
AUTO_FOLLOW_EXCEPTIONS_PARSER.declareObject(
ConstructingObjectParser.constructorArg(),
(p, c) -> ElasticsearchException.fromXContent(p),
Expand All @@ -93,13 +96,13 @@ public final class AutoFollowStats {
private final long numberOfFailedFollowIndices;
private final long numberOfFailedRemoteClusterStateRequests;
private final long numberOfSuccessfulFollowIndices;
private final NavigableMap<String, ElasticsearchException> recentAutoFollowErrors;
private final NavigableMap<String, Tuple<Long, ElasticsearchException>> recentAutoFollowErrors;
private final NavigableMap<String, AutoFollowedCluster> autoFollowedClusters;

AutoFollowStats(long numberOfFailedFollowIndices,
long numberOfFailedRemoteClusterStateRequests,
long numberOfSuccessfulFollowIndices,
NavigableMap<String, ElasticsearchException> recentAutoFollowErrors,
NavigableMap<String, Tuple<Long, ElasticsearchException>> recentAutoFollowErrors,
NavigableMap<String, AutoFollowedCluster> autoFollowedClusters) {
this.numberOfFailedFollowIndices = numberOfFailedFollowIndices;
this.numberOfFailedRemoteClusterStateRequests = numberOfFailedRemoteClusterStateRequests;
Expand All @@ -120,7 +123,7 @@ public long getNumberOfSuccessfulFollowIndices() {
return numberOfSuccessfulFollowIndices;
}

public NavigableMap<String, ElasticsearchException> getRecentAutoFollowErrors() {
public NavigableMap<String, Tuple<Long, ElasticsearchException>> getRecentAutoFollowErrors() {
return recentAutoFollowErrors;
}

Expand Down
26 changes: 16 additions & 10 deletions client/rest-high-level/src/test/java/org/elasticsearch/client/ccr/CcrStatsResponseTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,15 +74,18 @@ private static void assertEqualInstances(CcrStatsResponse expectedInstance, CcrS
equalTo(expectedAutoFollowStats.getRecentAutoFollowErrors().size()));
assertThat(newAutoFollowStats.getRecentAutoFollowErrors().keySet(),
equalTo(expectedAutoFollowStats.getRecentAutoFollowErrors().keySet()));
for (final Map.Entry<String, ElasticsearchException> entry : newAutoFollowStats.getRecentAutoFollowErrors().entrySet()) {
for (final Map.Entry<String, Tuple<Long, ElasticsearchException>> entry :
newAutoFollowStats.getRecentAutoFollowErrors().entrySet()) {
// x-content loses the exception
final ElasticsearchException expected = expectedAutoFollowStats.getRecentAutoFollowErrors().get(entry.getKey());
assertThat(entry.getValue().getMessage(), containsString(expected.getMessage()));
assertNotNull(entry.getValue().getCause());
final Tuple<Long, ElasticsearchException> expected =
expectedAutoFollowStats.getRecentAutoFollowErrors().get(entry.getKey());
assertThat(entry.getValue().v2().getMessage(), containsString(expected.v2().getMessage()));
assertThat(entry.getValue().v1(), equalTo(expected.v1()));
assertNotNull(entry.getValue().v2().getCause());
assertThat(
entry.getValue().getCause(),
entry.getValue().v2().getCause(),
anyOf(instanceOf(ElasticsearchException.class), instanceOf(IllegalStateException.class)));
assertThat(entry.getValue().getCause().getMessage(), containsString(expected.getCause().getMessage()));
assertThat(entry.getValue().v2().getCause().getMessage(), containsString(expected.v2().getCause().getMessage()));
}
}
{
Expand Down Expand Up @@ -172,14 +175,16 @@ private static void toXContent(CcrStatsResponse response, XContentBuilder builde
builder.field(AutoFollowStats.NUMBER_OF_FAILED_INDICES_AUTO_FOLLOWED.getPreferredName(),
autoFollowStats.getNumberOfFailedFollowIndices());
builder.startArray(AutoFollowStats.RECENT_AUTO_FOLLOW_ERRORS.getPreferredName());
for (Map.Entry<String, ElasticsearchException> entry : autoFollowStats.getRecentAutoFollowErrors().entrySet()) {
for (Map.Entry<String, Tuple<Long, ElasticsearchException>> entry :
autoFollowStats.getRecentAutoFollowErrors().entrySet()) {
builder.startObject();
{
builder.field(AutoFollowStats.LEADER_INDEX.getPreferredName(), entry.getKey());
builder.field(AutoFollowStats.TIMESTAMP.getPreferredName(), entry.getValue().v1());
builder.field(AutoFollowStats.AUTO_FOLLOW_EXCEPTION.getPreferredName());
builder.startObject();
{
ElasticsearchException.generateThrowableXContent(builder, ToXContent.EMPTY_PARAMS, entry.getValue());
ElasticsearchException.generateThrowableXContent(builder, ToXContent.EMPTY_PARAMS, entry.getValue().v2());
}
builder.endObject();
}
Expand Down Expand Up @@ -325,9 +330,10 @@ private static CcrStatsResponse createTestInstance() {

private static AutoFollowStats randomAutoFollowStats() {
final int count = randomIntBetween(0, 16);
final NavigableMap<String, ElasticsearchException> readExceptions = new TreeMap<>();
final NavigableMap<String, Tuple<Long, ElasticsearchException>> readExceptions = new TreeMap<>();
for (int i = 0; i < count; i++) {
readExceptions.put("" + i, new ElasticsearchException(new IllegalStateException("index [" + i + "]")));
readExceptions.put("" + i, Tuple.tuple(randomNonNegativeLong(),
new ElasticsearchException(new IllegalStateException("index [" + i + "]"))));
}
final NavigableMap<String, AutoFollowedCluster> autoFollowClusters = new TreeMap<>();
for (int i = 0; i < count; i++) {
Expand Down
11 changes: 6 additions & 5 deletions ...ck/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/AutoFollowCoordinator.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,7 @@ public class AutoFollowCoordinator implements ClusterStateListener {
private long numberOfSuccessfulIndicesAutoFollowed = 0;
private long numberOfFailedIndicesAutoFollowed = 0;
private long numberOfFailedRemoteClusterStateRequests = 0;
private final LinkedHashMap<String, ElasticsearchException> recentAutoFollowErrors;
private final LinkedHashMap<String, Tuple<Long, ElasticsearchException>> recentAutoFollowErrors;

public AutoFollowCoordinator(
Settings settings,
Expand All @@ -96,9 +96,9 @@ public AutoFollowCoordinator(
this.ccrLicenseChecker = Objects.requireNonNull(ccrLicenseChecker, "ccrLicenseChecker");
this.relativeMillisTimeProvider = relativeMillisTimeProvider;
clusterService.addListener(this);
this.recentAutoFollowErrors = new LinkedHashMap<String, ElasticsearchException>() {
this.recentAutoFollowErrors = new LinkedHashMap<String, Tuple<Long, ElasticsearchException>>() {
@Override
protected boolean removeEldestEntry(final Map.Entry<String, ElasticsearchException> eldest) {
protected boolean removeEldestEntry(final Map.Entry<String, Tuple<Long, ElasticsearchException>> eldest) {
return size() > MAX_AUTO_FOLLOW_ERRORS;
}
};
Expand Down Expand Up @@ -138,10 +138,11 @@ public synchronized AutoFollowStats getStats() {
}

synchronized void updateStats(List<AutoFollowResult> results) {
long newStatsReceivedTimeStamp = relativeMillisTimeProvider.getAsLong();

@jasontedor jasontedor Dec 21, 2018

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We want the absolute time provider here!

@martijnvg martijnvg Dec 22, 2018

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

whoops, you're absolutely right. I guess I made it this change too quickly...

for (AutoFollowResult result : results) {
if (result.clusterStateFetchException != null) {
recentAutoFollowErrors.put(result.autoFollowPatternName,
new ElasticsearchException(result.clusterStateFetchException));
Tuple.tuple(newStatsReceivedTimeStamp, new ElasticsearchException(result.clusterStateFetchException)));
numberOfFailedRemoteClusterStateRequests++;
LOGGER.warn(new ParameterizedMessage("failure occurred while fetching cluster state for auto follow pattern [{}]",
result.autoFollowPatternName), result.clusterStateFetchException);
Expand All @@ -150,7 +151,7 @@ synchronized void updateStats(List<AutoFollowResult> results) {
if (entry.getValue() != null) {
numberOfFailedIndicesAutoFollowed++;
recentAutoFollowErrors.put(result.autoFollowPatternName + ":" + entry.getKey().getName(),
ExceptionsHelper.convertToElastic(entry.getValue()));
Tuple.tuple(newStatsReceivedTimeStamp, ExceptionsHelper.convertToElastic(entry.getValue())));
LOGGER.warn(new ParameterizedMessage("failure occurred while auto following index [{}] for auto follow " +
"pattern [{}]", entry.getKey(), result.autoFollowPatternName), entry.getValue());
} else {
Expand Down
6 changes: 3 additions & 3 deletions x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/AutoFollowIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -281,12 +281,12 @@ public void testConflictingPatterns() throws Exception {
assertThat(autoFollowStats.getNumberOfFailedRemoteClusterStateRequests(), equalTo(0L));

assertThat(autoFollowStats.getRecentAutoFollowErrors().size(), equalTo(2));
ElasticsearchException autoFollowError1 = autoFollowStats.getRecentAutoFollowErrors().get("my-pattern1:logs-201801");
ElasticsearchException autoFollowError1 = autoFollowStats.getRecentAutoFollowErrors().get("my-pattern1:logs-201801").v2();
assertThat(autoFollowError1, notNullValue());
assertThat(autoFollowError1.getRootCause().getMessage(), equalTo("index to follow [logs-201801] for pattern [my-pattern1] " +
"matches with other patterns [my-pattern2]"));

ElasticsearchException autoFollowError2 = autoFollowStats.getRecentAutoFollowErrors().get("my-pattern2:logs-201801");
ElasticsearchException autoFollowError2 = autoFollowStats.getRecentAutoFollowErrors().get("my-pattern2:logs-201801").v2();
assertThat(autoFollowError2, notNullValue());
assertThat(autoFollowError2.getRootCause().getMessage(), equalTo("index to follow [logs-201801] for pattern [my-pattern2] " +
"matches with other patterns [my-pattern1]"));
Expand All @@ -311,7 +311,7 @@ public void testAutoFollowSoftDeletesDisabled() throws Exception {
assertThat(autoFollowStats.getNumberOfSuccessfulFollowIndices(), equalTo(0L));
assertThat(autoFollowStats.getNumberOfFailedFollowIndices(), equalTo(1L));
assertThat(autoFollowStats.getRecentAutoFollowErrors().size(), equalTo(1));
ElasticsearchException failure = autoFollowStats.getRecentAutoFollowErrors().firstEntry().getValue();
ElasticsearchException failure = autoFollowStats.getRecentAutoFollowErrors().firstEntry().getValue().v2();
assertThat(failure.getMessage(), equalTo("index [logs-20200101] cannot be followed, " +
"because soft deletes are not enabled"));
IndicesExistsRequest request = new IndicesExistsRequest("copy-logs-20200101");
Expand Down
14 changes: 7 additions & 7 deletions ...ugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/AutoFollowCoordinatorTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -558,7 +558,7 @@ public void testStats() {
assertThat(autoFollowStats.getNumberOfFailedRemoteClusterStateRequests(), equalTo(1L));
assertThat(autoFollowStats.getNumberOfSuccessfulFollowIndices(), equalTo(0L));
assertThat(autoFollowStats.getRecentAutoFollowErrors().size(), equalTo(1));
assertThat(autoFollowStats.getRecentAutoFollowErrors().get("_alias1").getCause().getMessage(), equalTo("error"));
assertThat(autoFollowStats.getRecentAutoFollowErrors().get("_alias1").v2().getCause().getMessage(), equalTo("error"));

autoFollowCoordinator.updateStats(Arrays.asList(
new AutoFollowCoordinator.AutoFollowResult("_alias1",
Expand All @@ -571,9 +571,9 @@ public void testStats() {
assertThat(autoFollowStats.getNumberOfFailedRemoteClusterStateRequests(), equalTo(1L));
assertThat(autoFollowStats.getNumberOfSuccessfulFollowIndices(), equalTo(0L));
assertThat(autoFollowStats.getRecentAutoFollowErrors().size(), equalTo(3));
assertThat(autoFollowStats.getRecentAutoFollowErrors().get("_alias1").getCause().getMessage(), equalTo("error"));
assertThat(autoFollowStats.getRecentAutoFollowErrors().get("_alias1:index1").getCause().getMessage(), equalTo("error"));
assertThat(autoFollowStats.getRecentAutoFollowErrors().get("_alias2:index2").getCause().getMessage(), equalTo("error"));
assertThat(autoFollowStats.getRecentAutoFollowErrors().get("_alias1").v2().getCause().getMessage(), equalTo("error"));
assertThat(autoFollowStats.getRecentAutoFollowErrors().get("_alias1:index1").v2().getCause().getMessage(), equalTo("error"));
assertThat(autoFollowStats.getRecentAutoFollowErrors().get("_alias2:index2").v2().getCause().getMessage(), equalTo("error"));

autoFollowCoordinator.updateStats(Arrays.asList(
new AutoFollowCoordinator.AutoFollowResult("_alias1",
Expand All @@ -586,9 +586,9 @@ public void testStats() {
assertThat(autoFollowStats.getNumberOfFailedRemoteClusterStateRequests(), equalTo(1L));
assertThat(autoFollowStats.getNumberOfSuccessfulFollowIndices(), equalTo(2L));
assertThat(autoFollowStats.getRecentAutoFollowErrors().size(), equalTo(3));
assertThat(autoFollowStats.getRecentAutoFollowErrors().get("_alias1").getCause().getMessage(), equalTo("error"));
assertThat(autoFollowStats.getRecentAutoFollowErrors().get("_alias1:index1").getCause().getMessage(), equalTo("error"));
assertThat(autoFollowStats.getRecentAutoFollowErrors().get("_alias2:index2").getCause().getMessage(), equalTo("error"));
assertThat(autoFollowStats.getRecentAutoFollowErrors().get("_alias1").v2().getCause().getMessage(), equalTo("error"));
assertThat(autoFollowStats.getRecentAutoFollowErrors().get("_alias1:index1").v2().getCause().getMessage(), equalTo("error"));
assertThat(autoFollowStats.getRecentAutoFollowErrors().get("_alias2:index2").v2().getCause().getMessage(), equalTo("error"));
}

public void testUpdateAutoFollowers() {
Expand Down
21 changes: 12 additions & 9 deletions x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/AutoFollowStatsTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
package org.elasticsearch.xpack.ccr.action;

import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.common.io.stream.Writeable;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.test.AbstractSerializingTestCase;
Expand Down Expand Up @@ -40,11 +41,12 @@ protected AutoFollowStats createTestInstance() {
);
}

static NavigableMap<String, ElasticsearchException> randomReadExceptions() {
static NavigableMap<String, Tuple<Long, ElasticsearchException>> randomReadExceptions() {
final int count = randomIntBetween(0, 16);
final NavigableMap<String, ElasticsearchException> readExceptions = new TreeMap<>();
final NavigableMap<String, Tuple<Long, ElasticsearchException>> readExceptions = new TreeMap<>();
for (int i = 0; i < count; i++) {
readExceptions.put("" + i, new ElasticsearchException(new IllegalStateException("index [" + i + "]")));
readExceptions.put("" + i, Tuple.tuple(randomNonNegativeLong(),
new ElasticsearchException(new IllegalStateException("index [" + i + "]"))));
}
return readExceptions;
}
Expand Down Expand Up @@ -74,15 +76,16 @@ protected void assertEqualInstances(AutoFollowStats expectedInstance, AutoFollow

assertThat(newInstance.getRecentAutoFollowErrors().size(), equalTo(expectedInstance.getRecentAutoFollowErrors().size()));
assertThat(newInstance.getRecentAutoFollowErrors().keySet(), equalTo(expectedInstance.getRecentAutoFollowErrors().keySet()));
for (final Map.Entry<String, ElasticsearchException> entry : newInstance.getRecentAutoFollowErrors().entrySet()) {
for (final Map.Entry<String, Tuple<Long, ElasticsearchException>> entry : newInstance.getRecentAutoFollowErrors().entrySet()) {
// x-content loses the exception
final ElasticsearchException expected = expectedInstance.getRecentAutoFollowErrors().get(entry.getKey());
assertThat(entry.getValue().getMessage(), containsString(expected.getMessage()));
assertNotNull(entry.getValue().getCause());
final Tuple<Long, ElasticsearchException> expected = expectedInstance.getRecentAutoFollowErrors().get(entry.getKey());
assertThat(entry.getValue().v1(), equalTo(expected.v1()));
assertThat(entry.getValue().v2().getMessage(), containsString(expected.v2().getMessage()));
assertNotNull(entry.getValue().v2().getCause());
assertThat(
entry.getValue().getCause(),
entry.getValue().v2().getCause(),
anyOf(instanceOf(ElasticsearchException.class), instanceOf(IllegalStateException.class)));
assertThat(entry.getValue().getCause().getMessage(), containsString(expected.getCause().getMessage()));
assertThat(entry.getValue().v2().getCause().getMessage(), containsString(expected.v2().getCause().getMessage()));
}

assertThat(newInstance.getAutoFollowedClusters(), equalTo(expectedInstance.getAutoFollowedClusters()));
Expand Down
Loading