Skip to content

Add optional description parameter to ingest processors. #57906

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 10 commits into from
Jun 15, 2020
Merged

Add optional description parameter to ingest processors. #57906

Show file tree
Hide file tree
Changes from 6 commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
5c020ad
Add optional description parameter to ingest processors.
talevy Jun 9, 2020
bf0305f
fix ingest-common test refactoring
talevy Jun 10, 2020
882d17c
more precommit fixes
talevy Jun 10, 2020
590e2cd
misordering in dotexpander
talevy Jun 10, 2020
6316c2b
more fixes
talevy Jun 10, 2020
8e315c0
Merge remote-tracking branch 'elastic/master' into ingest-description
talevy Jun 10, 2020
02d642d
Merge remote-tracking branch 'elastic/master' into ingest-description
talevy Jun 12, 2020
22e0e2f
add more yml tests
talevy Jun 12, 2020
90ad387
Merge remote-tracking branch 'elastic/master' into ingest-description
talevy Jun 15, 2020
32063d6
Merge remote-tracking branch 'elastic/master' into ingest-description
talevy Jun 15, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 6 additions & 5 deletions .../ingest-common/src/main/java/org/elasticsearch/ingest/common/AbstractStringProcessor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,8 +39,8 @@ abstract class AbstractStringProcessor<T> extends AbstractProcessor {
private final boolean ignoreMissing;
private final String targetField;

AbstractStringProcessor(String tag, String field, boolean ignoreMissing, String targetField) {
super(tag);
AbstractStringProcessor(String tag, String description, boolean ignoreMissing, String targetField, String field) {
super(tag, description);
this.field = field;
this.ignoreMissing = ignoreMissing;
this.targetField = targetField;
Expand Down Expand Up @@ -106,15 +106,16 @@ protected Factory(String processorType) {

@Override
public AbstractStringProcessor<?> create(Map<String, Processor.Factory> registry, String tag,
Map<String, Object> config) throws Exception {
String description, Map<String, Object> config) throws Exception {
String field = ConfigurationUtils.readStringProperty(processorType, tag, config, "field");
boolean ignoreMissing = ConfigurationUtils.readBooleanProperty(processorType, tag, config, "ignore_missing", false);
String targetField = ConfigurationUtils.readStringProperty(processorType, tag, config, "target_field", field);

return newProcessor(tag, config, field, ignoreMissing, targetField);
return newProcessor(tag, description, config, field, ignoreMissing, targetField);
}

protected abstract AbstractStringProcessor<?> newProcessor(String processorTag, Map<String, Object> config, String field,
protected abstract AbstractStringProcessor<?> newProcessor(String processorTag, String description,
Map<String, Object> config, String field,
boolean ignoreMissing, String targetField);
}
}
8 changes: 4 additions & 4 deletions modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/AppendProcessor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,8 +41,8 @@ public final class AppendProcessor extends AbstractProcessor {
private final TemplateScript.Factory field;
private final ValueSource value;

AppendProcessor(String tag, TemplateScript.Factory field, ValueSource value) {
super(tag);
AppendProcessor(String tag, String description, TemplateScript.Factory field, ValueSource value) {
super(tag, description);
this.field = field;
this.value = value;
}
Expand Down Expand Up @@ -76,12 +76,12 @@ public Factory(ScriptService scriptService) {

@Override
public AppendProcessor create(Map<String, Processor.Factory> registry, String processorTag,
Map<String, Object> config) throws Exception {
String description, Map<String, Object> config) throws Exception {
String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field");
Object value = ConfigurationUtils.readObject(TYPE, processorTag, config, "value");
TemplateScript.Factory compiledTemplate = ConfigurationUtils.compileTemplate(TYPE, processorTag,
"field", field, scriptService);
return new AppendProcessor(processorTag, compiledTemplate, ValueSource.wrap(value, scriptService));
return new AppendProcessor(processorTag, description, compiledTemplate, ValueSource.wrap(value, scriptService));
}
}
}
8 changes: 4 additions & 4 deletions modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/BytesProcessor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,8 +31,8 @@ public final class BytesProcessor extends AbstractStringProcessor<Long> {

public static final String TYPE = "bytes";

BytesProcessor(String processorTag, String field, boolean ignoreMissing, String targetField) {
super(processorTag, field, ignoreMissing, targetField);
BytesProcessor(String processorTag, String description, String field, boolean ignoreMissing, String targetField) {
super(processorTag, description, ignoreMissing, targetField, field);
}

public static long apply(String value) {
Expand All @@ -56,9 +56,9 @@ public Factory() {
}

@Override
protected BytesProcessor newProcessor(String tag, Map<String, Object> config, String field,
protected BytesProcessor newProcessor(String tag, String description, Map<String, Object> config, String field,
boolean ignoreMissing, String targetField) {
return new BytesProcessor(tag, field, ignoreMissing, targetField);
return new BytesProcessor(tag, description, field, ignoreMissing, targetField);
}
}
}
8 changes: 4 additions & 4 deletions modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ConvertProcessor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -148,8 +148,8 @@ public static Type fromString(String processorTag, String propertyName, String t
private final Type convertType;
private final boolean ignoreMissing;

ConvertProcessor(String tag, String field, String targetField, Type convertType, boolean ignoreMissing) {
super(tag);
ConvertProcessor(String tag, String description, String field, String targetField, Type convertType, boolean ignoreMissing) {
super(tag, description);
this.field = field;
this.targetField = targetField;
this.convertType = convertType;
Expand Down Expand Up @@ -205,13 +205,13 @@ public String getType() {
public static final class Factory implements Processor.Factory {
@Override
public ConvertProcessor create(Map<String, Processor.Factory> registry, String processorTag,
Map<String, Object> config) throws Exception {
String description, Map<String, Object> config) throws Exception {
String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field");
String typeProperty = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "type");
String targetField = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "target_field", field);
Type convertType = Type.fromString(processorTag, "type", typeProperty);
boolean ignoreMissing = ConfigurationUtils.readBooleanProperty(TYPE, processorTag, config, "ignore_missing", false);
return new ConvertProcessor(processorTag, field, targetField, convertType, ignoreMissing);
return new ConvertProcessor(processorTag, description, field, targetField, convertType, ignoreMissing);
}
}
}
15 changes: 8 additions & 7 deletions modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/CsvProcessor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
import org.elasticsearch.ingest.AbstractProcessor;
import org.elasticsearch.ingest.ConfigurationUtils;
import org.elasticsearch.ingest.IngestDocument;
import org.elasticsearch.ingest.Processor;

import java.util.List;
import java.util.Map;
Expand Down Expand Up @@ -54,9 +55,9 @@ public final class CsvProcessor extends AbstractProcessor {
final boolean ignoreMissing;
final Object emptyValue;

CsvProcessor(String tag, String field, String[] headers, boolean trim, char separator, char quote, boolean ignoreMissing,
Object emptyValue) {
super(tag);
CsvProcessor(String tag, String description, String field, String[] headers, boolean trim, char separator, char quote,
boolean ignoreMissing, Object emptyValue) {
super(tag, description);
this.field = field;
this.headers = headers;
this.trim = trim;
Expand Down Expand Up @@ -89,8 +90,8 @@ public String getType() {

public static final class Factory implements org.elasticsearch.ingest.Processor.Factory {
@Override
public CsvProcessor create(Map<String, org.elasticsearch.ingest.Processor.Factory> registry, String processorTag,
Map<String, Object> config) {
public CsvProcessor create(Map<String, Processor.Factory> registry, String processorTag,
String description, Map<String, Object> config) {
String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field");
String quote = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "quote", "\"");
if (quote.length() != 1) {
Expand All @@ -110,8 +111,8 @@ public CsvProcessor create(Map<String, org.elasticsearch.ingest.Processor.Factor
if (targetFields.isEmpty()) {
throw newConfigurationException(TYPE, processorTag, "target_fields", "target fields list can't be empty");
}
return new CsvProcessor(processorTag, field, targetFields.toArray(String[]::new), trim, separator.charAt(0), quote.charAt(0),
ignoreMissing, emptyValue);
return new CsvProcessor(processorTag, description, field, targetFields.toArray(String[]::new), trim, separator.charAt(0),
quote.charAt(0), ignoreMissing, emptyValue);
}
}
}
10 changes: 5 additions & 5 deletions ...s/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateIndexNameProcessor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,10 +50,10 @@ public final class DateIndexNameProcessor extends AbstractProcessor {
private final ZoneId timezone;
private final List<Function<String, ZonedDateTime>> dateFormats;

DateIndexNameProcessor(String tag, String field, List<Function<String, ZonedDateTime>> dateFormats, ZoneId timezone,
TemplateScript.Factory indexNamePrefixTemplate, TemplateScript.Factory dateRoundingTemplate,
DateIndexNameProcessor(String tag, String description, String field, List<Function<String, ZonedDateTime>> dateFormats,
ZoneId timezone, TemplateScript.Factory indexNamePrefixTemplate, TemplateScript.Factory dateRoundingTemplate,
TemplateScript.Factory indexNameFormatTemplate) {
super(tag);
super(tag, description);
this.field = field;
this.timezone = timezone;
this.dateFormats = dateFormats;
Expand Down Expand Up @@ -145,7 +145,7 @@ public Factory(ScriptService scriptService) {

@Override
public DateIndexNameProcessor create(Map<String, Processor.Factory> registry, String tag,
Map<String, Object> config) throws Exception {
String description, Map<String, Object> config) throws Exception {
String localeString = ConfigurationUtils.readOptionalStringProperty(TYPE, tag, config, "locale");
String timezoneString = ConfigurationUtils.readOptionalStringProperty(TYPE, tag, config, "timezone");
ZoneId timezone = timezoneString == null ? ZoneOffset.UTC : ZoneId.of(timezoneString);
Expand Down Expand Up @@ -177,7 +177,7 @@ public DateIndexNameProcessor create(Map<String, Processor.Factory> registry, St
String indexNameFormat = ConfigurationUtils.readStringProperty(TYPE, tag, config, "index_name_format", "yyyy-MM-dd");
TemplateScript.Factory indexNameFormatTemplate =
ConfigurationUtils.compileTemplate(TYPE, tag, "index_name_format", indexNameFormat, scriptService);
return new DateIndexNameProcessor(tag, field, dateFormats, timezone, indexNamePrefixTemplate,
return new DateIndexNameProcessor(tag, description, field, dateFormats, timezone, indexNamePrefixTemplate,
dateRoundingTemplate, indexNameFormatTemplate);
}
}
Expand Down
9 changes: 5 additions & 4 deletions modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateProcessor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,9 +52,9 @@ public final class DateProcessor extends AbstractProcessor {
private final List<String> formats;
private final List<Function<Map<String, Object>, Function<String, ZonedDateTime>>> dateParsers;

DateProcessor(String tag, @Nullable TemplateScript.Factory timezone, @Nullable TemplateScript.Factory locale,
DateProcessor(String tag, String description, @Nullable TemplateScript.Factory timezone, @Nullable TemplateScript.Factory locale,
String field, List<String> formats, String targetField) {
super(tag);
super(tag, description);
this.timezone = timezone;
this.locale = locale;
this.field = field;
Expand Down Expand Up @@ -137,7 +137,7 @@ public Factory(ScriptService scriptService) {
}

public DateProcessor create(Map<String, Processor.Factory> registry, String processorTag,
Map<String, Object> config) throws Exception {
String description, Map<String, Object> config) throws Exception {
String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field");
String targetField = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "target_field", DEFAULT_TARGET_FIELD);
String timezoneString = ConfigurationUtils.readOptionalStringProperty(TYPE, processorTag, config, "timezone");
Expand All @@ -153,7 +153,8 @@ public DateProcessor create(Map<String, Processor.Factory> registry, String proc
"locale", localeString, scriptService);
}
List<String> formats = ConfigurationUtils.readList(TYPE, processorTag, config, "formats");
return new DateProcessor(processorTag, compiledTimezoneTemplate, compiledLocaleTemplate, field, formats, targetField);
return new DateProcessor(processorTag, description, compiledTimezoneTemplate, compiledLocaleTemplate, field, formats,
targetField);
}
}
}
9 changes: 5 additions & 4 deletions modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DissectProcessor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,8 +37,8 @@ public final class DissectProcessor extends AbstractProcessor {
final String appendSeparator;
final DissectParser dissectParser;

DissectProcessor(String tag, String field, String pattern, String appendSeparator, boolean ignoreMissing) {
super(tag);
DissectProcessor(String tag, String description, String field, String pattern, String appendSeparator, boolean ignoreMissing) {
super(tag, description);
this.field = field;
this.ignoreMissing = ignoreMissing;
this.pattern = pattern;
Expand Down Expand Up @@ -66,12 +66,13 @@ public String getType() {
public static final class Factory implements Processor.Factory {

@Override
public DissectProcessor create(Map<String, Processor.Factory> registry, String processorTag, Map<String, Object> config) {
public DissectProcessor create(Map<String, Processor.Factory> registry, String processorTag, String description,
Map<String, Object> config) {
String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field");
String pattern = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "pattern");
String appendSeparator = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "append_separator", "");
boolean ignoreMissing = ConfigurationUtils.readBooleanProperty(TYPE, processorTag, config, "ignore_missing", false);
return new DissectProcessor(processorTag, field, pattern, appendSeparator, ignoreMissing);
return new DissectProcessor(processorTag, description, field, pattern, appendSeparator, ignoreMissing);
}
}
}
8 changes: 4 additions & 4 deletions ...les/ingest-common/src/main/java/org/elasticsearch/ingest/common/DotExpanderProcessor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,8 +33,8 @@ public final class DotExpanderProcessor extends AbstractProcessor {
private final String path;
private final String field;

DotExpanderProcessor(String tag, String path, String field) {
super(tag);
DotExpanderProcessor(String tag, String description, String path, String field) {
super(tag, description);
this.path = path;
this.field = field;
}
Expand Down Expand Up @@ -96,7 +96,7 @@ String getField() {
public static final class Factory implements Processor.Factory {

@Override
public Processor create(Map<String, Processor.Factory> processorFactories, String tag,
public Processor create(Map<String, Processor.Factory> processorFactories, String tag, String description,
Map<String, Object> config) throws Exception {
String field = ConfigurationUtils.readStringProperty(TYPE, tag, config, "field");
if (field.contains(".") == false) {
Expand All @@ -117,7 +117,7 @@ public Processor create(Map<String, Processor.Factory> processorFactories, Strin
}

String path = ConfigurationUtils.readOptionalStringProperty(TYPE, tag, config, "path");
return new DotExpanderProcessor(tag, path, field);
return new DotExpanderProcessor(tag, null, path, field);
}
}
}
8 changes: 4 additions & 4 deletions modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/FailProcessor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,8 +38,8 @@ public final class FailProcessor extends AbstractProcessor {

private final TemplateScript.Factory message;

FailProcessor(String tag, TemplateScript.Factory message) {
super(tag);
FailProcessor(String tag, String description, TemplateScript.Factory message) {
super(tag, description);
this.message = message;
}

Expand Down Expand Up @@ -67,11 +67,11 @@ public Factory(ScriptService scriptService) {

@Override
public FailProcessor create(Map<String, Processor.Factory> registry, String processorTag,
Map<String, Object> config) throws Exception {
String description, Map<String, Object> config) throws Exception {
String message = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "message");
TemplateScript.Factory compiledTemplate = ConfigurationUtils.compileTemplate(TYPE, processorTag,
"message", message, scriptService);
return new FailProcessor(processorTag, compiledTemplate);
return new FailProcessor(processorTag, description, compiledTemplate);
}
}
}
Expand Down
8 changes: 4 additions & 4 deletions modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ForEachProcessor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,8 +54,8 @@ public final class ForEachProcessor extends AbstractProcessor implements Wrappin
private final Processor processor;
private final boolean ignoreMissing;

ForEachProcessor(String tag, String field, Processor processor, boolean ignoreMissing) {
super(tag);
ForEachProcessor(String tag, String description, String field, Processor processor, boolean ignoreMissing) {
super(tag, description);
this.field = field;
this.processor = processor;
this.ignoreMissing = ignoreMissing;
Expand Down Expand Up @@ -134,7 +134,7 @@ public static final class Factory implements Processor.Factory {

@Override
public ForEachProcessor create(Map<String, Processor.Factory> factories, String tag,
Map<String, Object> config) throws Exception {
String description, Map<String, Object> config) throws Exception {
String field = readStringProperty(TYPE, tag, config, "field");
boolean ignoreMissing = readBooleanProperty(TYPE, tag, config, "ignore_missing", false);
Map<String, Map<String, Object>> processorConfig = readMap(TYPE, tag, config, "processor");
Expand All @@ -145,7 +145,7 @@ public ForEachProcessor create(Map<String, Processor.Factory> factories, String
Map.Entry<String, Map<String, Object>> entry = entries.iterator().next();
Processor processor =
ConfigurationUtils.readProcessor(factories, scriptService, entry.getKey(), entry.getValue());
return new ForEachProcessor(tag, field, processor, ignoreMissing);
return new ForEachProcessor(tag, description, field, processor, ignoreMissing);
}
}
}
Loading