Skip to content

Commit

Permalink
Add support for properties, source_indexed_field, format, ignore_malf…
Browse files Browse the repository at this point in the history 
…ormed

Signed-off-by: Rishabh Maurya <[email protected]>
  • Loading branch information
@rishabhmaurya
rishabhmaurya committed May 9, 2024
1 parent 8830138 commit d7b3d1d
Show file tree
Hide file tree
Showing 13 changed files with 823 additions and 145 deletions.
58 changes: 46 additions & 12 deletions server/src/main/java/org/opensearch/index/mapper/DefaultDerivedFieldResolver.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;

import static org.opensearch.index.mapper.FieldMapper.IGNORE_MALFORMED_SETTING;

/**
* Accepts definition of DerivedField from search request in both forms: map parsed from SearchRequest and {@link DerivedField} defined using client.
* The object is initialized per search request and is responsible to resolve {@link DerivedFieldType} given a field name.
Expand Down Expand Up @@ -106,18 +108,37 @@ private DerivedFieldType resolveNestedField(String fieldName) {
if (parentDerivedField != null) {
try {
Script script = parentDerivedField.derivedField.getScript();
Mapper inferredFieldMapper = typeInference.infer(getValueFetcher(fieldName, script));
if (inferredFieldMapper != null) {
return getDerivedFieldType(
new DerivedField(
fieldName,
inferredFieldMapper.typeName(),
script,
parentDerivedField.derivedField.getSourceIndexedField()
)
);
String nestedType = explicitTypeFromParent(
parentDerivedField.derivedField,
fieldName.substring(fieldName.indexOf(".") + 1)
);
if (nestedType == null) {
Mapper inferredFieldMapper = typeInference.infer(getValueFetcher(fieldName, script));
if (inferredFieldMapper != null) {
nestedType = inferredFieldMapper.typeName();
}
}
if (nestedType != null) {
DerivedField derivedField = new DerivedField(fieldName, nestedType, script);
if (parentDerivedField.derivedField.getProperties() != null) {
derivedField.setProperties(parentDerivedField.derivedField.getProperties());
}
if (parentDerivedField.derivedField.getSourceIndexedField() != null) {
derivedField.setSourceIndexedField(parentDerivedField.derivedField.getSourceIndexedField());
}
if (parentDerivedField.derivedField.getFormat() != null) {
derivedField.setFormat(parentDerivedField.derivedField.getFormat());
}
if (parentDerivedField.derivedField.getIgnoreMalformed()) {
derivedField.setIgnoreMalformed(parentDerivedField.derivedField.getIgnoreMalformed());
}
return getDerivedFieldType(derivedField);
} else {
logger.warn("Field type cannot be inferred. Ensure the field {} is not rare across entire index", fieldName);
logger.warn(
"Field type cannot be inferred. Ensure the field {} is not rare across entire index or provide explicit mapping using [properties] under parent object [{}] ",
fieldName,
parentDerivedField.derivedField.getName()
);
}
} catch (IOException e) {
logger.warn(e);
Expand All @@ -133,6 +154,17 @@ private MappedFieldType getParentDerivedField(String fieldName) {
return null;
}

private static String explicitTypeFromParent(DerivedField parentDerivedField, String subField) {
if (parentDerivedField == null
|| parentDerivedField.getProperties() == null
|| parentDerivedField.getProperties().isEmpty()
|| subField == null
|| subField.isEmpty()) {
return null;
}
return parentDerivedField.getProperties().get(subField);
}

ValueFetcher getValueFetcher(String fieldName, Script script) {
String subFieldName = fieldName.substring(fieldName.indexOf(".") + 1);
return new ObjectDerivedFieldType.ObjectDerivedFieldValueFetcher(
Expand Down Expand Up @@ -178,7 +210,9 @@ private DerivedFieldType getDerivedFieldType(DerivedField derivedField) {
);
DerivedFieldMapper.Builder builder = new DerivedFieldMapper.Builder(
derivedField,
queryShardContext.getMapperService().getIndexAnalyzers()
queryShardContext.getMapperService().getIndexAnalyzers(),
null,
IGNORE_MALFORMED_SETTING.getDefault(queryShardContext.getIndexSettings().getSettings())
);
return builder.build(builderContext).fieldType();
}
Expand Down
77 changes: 57 additions & 20 deletions server/src/main/java/org/opensearch/index/mapper/DerivedField.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,64 +18,71 @@
import org.opensearch.script.Script;

import java.io.IOException;
import java.util.Map;
import java.util.Objects;

/**
* DerivedField representation: expects a name, type and script.
*/
@PublicApi(since = "2.14.0")
public class DerivedField implements Writeable, ToXContentFragment {

private final String name;
private final String type;

private final String sourceIndexedField;

private final Script script;
private String sourceIndexedField;
private Map<String, String> properties;
private boolean ignoreMalformed;
private String format;

public DerivedField(String name, String type, Script script, String sourceIndexedField) {
public DerivedField(String name, String type, Script script) {
this.name = name;
this.type = type;
this.script = script;
this.sourceIndexedField = sourceIndexedField;
}

public DerivedField(String name, String type, Script script) {
this(name, type, script, null);
}

public DerivedField(StreamInput in) throws IOException {
name = in.readString();
type = in.readString();
script = new Script(in);
if (in.readBoolean()) {
sourceIndexedField = in.readString();
} else {
sourceIndexedField = null;
properties = in.readMap(StreamInput::readString, StreamInput::readString);
}
sourceIndexedField = in.readOptionalString();
format = in.readOptionalString();
ignoreMalformed = Boolean.TRUE.equals(in.readOptionalBoolean());
}

@Override
public void writeTo(StreamOutput out) throws IOException {
out.writeString(name);
out.writeString(type);
script.writeTo(out);
if (sourceIndexedField != null) {
out.writeBoolean(true);
out.writeString(sourceIndexedField);
} else {
if (properties == null) {
out.writeBoolean(false);
} else {
out.writeBoolean(true);
out.writeMap(properties, StreamOutput::writeString, StreamOutput::writeString);
}
out.writeOptionalString(sourceIndexedField);
out.writeOptionalString(format);
out.writeOptionalBoolean(ignoreMalformed);
}

@Override
public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException {
builder.startObject(name);
builder.field("type", type);
builder.field("script", script);
if (properties != null) {
builder.field("properties", properties);
}
if (sourceIndexedField != null) {
builder.field("source_indexed_field", sourceIndexedField);
}
if (format != null) {
builder.field("format", format);
}
builder.field("ignore_malformed", ignoreMalformed);
builder.endObject();
return builder;
}
Expand All @@ -92,13 +99,41 @@ public Script getScript() {
return script;
}

public Map<String, String> getProperties() {
return properties;
}

public String getSourceIndexedField() {
return sourceIndexedField;
}

public String getFormat() {
return format;
}

public boolean getIgnoreMalformed() {
return ignoreMalformed;
}

public void setProperties(Map<String, String> properties) {
this.properties = properties;
}

public void setSourceIndexedField(String sourceIndexedField) {
this.sourceIndexedField = sourceIndexedField;
}

public void setFormat(String format) {
this.format = format;
}

public void setIgnoreMalformed(boolean ignoreMalformed) {
this.ignoreMalformed = ignoreMalformed;
}

@Override
public int hashCode() {
return Objects.hash(name, type, script, sourceIndexedField);
return Objects.hash(name, type, script, sourceIndexedField, properties, ignoreMalformed, format);
}

@Override
Expand All @@ -113,7 +148,9 @@ public boolean equals(Object obj) {
return Objects.equals(name, other.name)
&& Objects.equals(type, other.type)
&& Objects.equals(script, other.script)
&& Objects.equals(sourceIndexedField, other.sourceIndexedField);
&& Objects.equals(sourceIndexedField, other.sourceIndexedField)
&& Objects.equals(properties, other.properties)
&& ignoreMalformed == other.ignoreMalformed
&& Objects.equals(format, other.format);
}

}
Loading

0 comments on commit d7b3d1d

Please sign in to comment.