Skip to content

[RollupV2] Move rollup metadata to field mappings #69921

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 14 commits into from
Mar 11, 2021
Merged

[RollupV2] Move rollup metadata to field mappings #69921

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
a7e04ab
Moved field specific metadata to field mappings
csoulios Mar 3, 2021
c0cb760
Added test for rolling up data stream indices
csoulios Mar 3, 2021
2bdb9f9
Addressed reviewer comments
csoulios Mar 4, 2021
2a2cb11
Moved rollup source idx name/uuid to IndexMetadata
csoulios Mar 4, 2021
78b3ade
Merge branch 'master' into rollupv2-meta-refactor
csoulios Mar 5, 2021
66a4ccb
Merge branch 'master' into rollupv2-meta-refactor
csoulios Mar 8, 2021
170c12e
Moved rollup source idx name/uuid to IndexMetadata
csoulios Mar 8, 2021
b265f07
Added detailed tests for metrics
csoulios Mar 9, 2021
ed5fa3c
Merge branch 'master' into rollupv2-meta-refactor
csoulios Mar 9, 2021
541de2b
use private settings
talevy Mar 9, 2021
d92496e
Code cleanup
csoulios Mar 10, 2021
ed1698c
Merge branch 'master' into rollupv2-meta-refactor
csoulios Mar 10, 2021
9191148
Addressed reviewer comments
csoulios Mar 10, 2021
c1a26ce
Merge branch 'master' into rollupv2-meta-refactor
csoulios Mar 10, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 0 additions & 9 deletions server/src/main/java/org/elasticsearch/cluster/ClusterModule.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,6 @@
import org.elasticsearch.cluster.metadata.MetadataMappingService;
import org.elasticsearch.cluster.metadata.MetadataUpdateSettingsService;
import org.elasticsearch.cluster.metadata.RepositoriesMetadata;
import org.elasticsearch.cluster.metadata.RollupMetadata;
import org.elasticsearch.cluster.routing.DelayedAllocationService;
import org.elasticsearch.cluster.routing.allocation.AllocationService;
import org.elasticsearch.cluster.routing.allocation.ExistingShardsAllocator;
Expand Down Expand Up @@ -65,7 +64,6 @@
import org.elasticsearch.persistent.PersistentTasksCustomMetadata;
import org.elasticsearch.persistent.PersistentTasksNodeService;
import org.elasticsearch.plugins.ClusterPlugin;
import org.elasticsearch.rollup.RollupV2;
import org.elasticsearch.script.ScriptMetadata;
import org.elasticsearch.snapshots.SnapshotsInfoService;
import org.elasticsearch.tasks.Task;
Expand Down Expand Up @@ -135,9 +133,6 @@ public static List<Entry> getNamedWriteables() {
ComposableIndexTemplateMetadata::readDiffFrom);
registerMetadataCustom(entries, DataStreamMetadata.TYPE, DataStreamMetadata::new, DataStreamMetadata::readDiffFrom);

if (RollupV2.isEnabled()) {
registerMetadataCustom(entries, RollupMetadata.TYPE, RollupMetadata::new, RollupMetadata::readDiffFrom);
}
// Task Status (not Diffable)
entries.add(new Entry(Task.Status.class, PersistentTasksNodeService.Status.NAME, PersistentTasksNodeService.Status::new));
return entries;
Expand All @@ -162,10 +157,6 @@ public static List<NamedXContentRegistry.Entry> getNamedXWriteables() {
ComposableIndexTemplateMetadata::fromXContent));
entries.add(new NamedXContentRegistry.Entry(Metadata.Custom.class, new ParseField(DataStreamMetadata.TYPE),
DataStreamMetadata::fromXContent));
if (RollupV2.isEnabled()) {
entries.add(new NamedXContentRegistry.Entry(Metadata.Custom.class, new ParseField(RollupMetadata.TYPE),
RollupMetadata::fromXContent));
}
return entries;
}

Expand Down
178 changes: 0 additions & 178 deletions server/src/main/java/org/elasticsearch/cluster/metadata/RollupGroup.java
View file
Open in desktop

This file was deleted.

168 changes: 2 additions & 166 deletions server/src/main/java/org/elasticsearch/cluster/metadata/RollupMetadata.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,174 +8,10 @@

package org.elasticsearch.cluster.metadata;

import org.elasticsearch.Version;
import org.elasticsearch.cluster.Diff;
import org.elasticsearch.cluster.DiffableUtils;
import org.elasticsearch.cluster.NamedDiff;
import org.elasticsearch.common.ParseField;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.xcontent.ConstructingObjectParser;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentParser;
public class RollupMetadata {

import java.io.IOException;
import java.util.EnumSet;
import java.util.HashMap;
import java.util.Map;
import java.util.Objects;

/**
* Custom {@link Metadata} implementation for storing a map of {@link RollupGroup}s and their names.
*/
public class RollupMetadata implements Metadata.Custom {
public static final String TYPE = "rollup";
public static final String SOURCE_INDEX_NAME_META_FIELD = "source_index";
private static final ParseField ROLLUP = new ParseField("rollup");

@SuppressWarnings("unchecked")
private static final ConstructingObjectParser<RollupMetadata, Void> PARSER = new ConstructingObjectParser<>(TYPE, false,
a -> new RollupMetadata((Map<String, RollupGroup>) a[0]));

static {
PARSER.declareObject(ConstructingObjectParser.constructorArg(), (p, c) -> {
Map<String, RollupGroup> rollupGroups = new HashMap<>();
while (p.nextToken() != XContentParser.Token.END_OBJECT) {
String name = p.currentName();
rollupGroups.put(name, RollupGroup.parse(p));
}
return rollupGroups;
}, ROLLUP);
}

private final Map<String, RollupGroup> rollupIndices;

public RollupMetadata(Map<String, RollupGroup> rollupIndices) {
this.rollupIndices = rollupIndices;
}

public RollupMetadata(StreamInput in) throws IOException {
this.rollupIndices = in.readMap(StreamInput::readString, RollupGroup::new);
}

public Map<String, RollupGroup> rollupGroups() {
return this.rollupIndices;
}

public boolean contains(String index) {
return this.rollupIndices.containsKey(index);
}

@Override
public Diff<Metadata.Custom> diff(Metadata.Custom before) {
return new RollupMetadata.RollupMetadataDiff((RollupMetadata) before, this);
}

public static NamedDiff<Metadata.Custom> readDiffFrom(StreamInput in) throws IOException {
return new RollupMetadataDiff(in);
}

@Override
public EnumSet<Metadata.XContentContext> context() {
return Metadata.ALL_CONTEXTS;
}

@Override
public String getWriteableName() {
return TYPE;
}

@Override
public Version getMinimalSupportedVersion() {
return Version.V_7_11_0;
}

@Override
public void writeTo(StreamOutput out) throws IOException {
out.writeMap(this.rollupIndices, StreamOutput::writeString, (stream, val) -> val.writeTo(stream));
}

public static RollupMetadata fromXContent(XContentParser parser) throws IOException {
return PARSER.parse(parser, null);
}

@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
builder.startObject(ROLLUP.getPreferredName());
for (Map.Entry<String, RollupGroup> rollup : rollupIndices.entrySet()) {
builder.field(rollup.getKey(), rollup.getValue());
}
builder.endObject();
return builder;
}

public static Builder builder() {
return new Builder();
}

@Override
public int hashCode() {
return Objects.hash(this.rollupIndices);
}

@Override
public boolean equals(Object obj) {
if (obj == null) {
return false;
}
if (obj.getClass() != getClass()) {
return false;
}
RollupMetadata other = (RollupMetadata) obj;
return Objects.equals(this.rollupIndices, other.rollupIndices);
}

@Override
public String toString() {
return Strings.toString(this);
}

public static class Builder {

private final Map<String, RollupGroup> rollupIndices = new HashMap<>();

public Builder putRollupGroup(String name, RollupGroup group) {
rollupIndices.put(name, group);
return this;
}

public RollupMetadata build() {
return new RollupMetadata(rollupIndices);
}
}

static class RollupMetadataDiff implements NamedDiff<Metadata.Custom> {

final Diff<Map<String, RollupGroup>> rollupIndicesDiff;

RollupMetadataDiff(RollupMetadata before, RollupMetadata after) {
this.rollupIndicesDiff = DiffableUtils.diff(before.rollupIndices, after.rollupIndices, DiffableUtils.getStringKeySerializer());
}

RollupMetadataDiff(StreamInput in) throws IOException {
this.rollupIndicesDiff = DiffableUtils.readJdkMapDiff(in, DiffableUtils.getStringKeySerializer(),
RollupGroup::new, RollupGroup::readDiffFrom);
}

@Override
public Metadata.Custom apply(Metadata.Custom part) {
return new RollupMetadata(rollupIndicesDiff.apply(((RollupMetadata) part).rollupIndices));
}

@Override
public void writeTo(StreamOutput out) throws IOException {
rollupIndicesDiff.writeTo(out);
}
public static final String SOURCE_INDEX_ID_META_FIELD = "source_index_uuid";

@Override
public String getWriteableName() {
return TYPE;
}
}
}
Loading