From d82978c733849460d56d56e89691ca86d3533113 Mon Sep 17 00:00:00 2001 From: Felix Barnsteiner Date: Tue, 25 Mar 2025 09:00:44 +0100 Subject: [PATCH 1/3] Bracket syntax for accessing dotted field names in ingest processors --- .../rest-api-spec/test/ingest/280_rename.yml | 27 +++++++++ .../elasticsearch/ingest/IngestDocument.java | 55 ++++++++++++++++- .../ingest/IngestDocumentTests.java | 60 +++++++++++++++++++ 3 files changed, 140 insertions(+), 2 deletions(-) diff --git a/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/280_rename.yml b/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/280_rename.yml index 26a0d5eef50ae..9fcdef2704217 100644 --- a/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/280_rename.yml +++ b/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/280_rename.yml @@ -108,3 +108,30 @@ teardown: index: test id: "1" - match: { _source.event.original: "overridden original message" } + +--- +"Test Rename dotted field access": + - do: + ingest.put_pipeline: + id: "1" + body: + processors: + - rename: + field: "attributes['foo.bar']" + target_field: "attributes['bar.foo']" + - match: { acknowledged: true } + + - do: + index: + index: test + id: "1" + pipeline: "1" + body: + attributes: + "foo.bar": "test" + + - do: + get: + index: test + id: "1" + - match: { _source.attributes.bar\.foo: "test" } diff --git a/server/src/main/java/org/elasticsearch/ingest/IngestDocument.java b/server/src/main/java/org/elasticsearch/ingest/IngestDocument.java index 11c725bacc510..b769dc02259ed 100644 --- a/server/src/main/java/org/elasticsearch/ingest/IngestDocument.java +++ b/server/src/main/java/org/elasticsearch/ingest/IngestDocument.java @@ -1041,12 +1041,63 @@ private FieldPath(String path) { newPath = path; } } - this.pathElements = newPath.split("\\."); - if (pathElements.length == 1 && pathElements[0].isEmpty()) { + this.pathElements = parsePath(newPath); + if (pathElements.length == 0 || (pathElements.length == 1 && pathElements[0].isEmpty())) { throw new IllegalArgumentException("path [" + path + "] is not valid"); } } + private String[] parsePath(String path) { + List elements = new ArrayList<>(); + StringBuilder current = new StringBuilder(); + boolean inBrackets = false; + boolean doubleQuote = false; + for (int i = 0; i < path.length(); i++) { + char c = path.charAt(i); + char next = 0; + if (i < path.length() - 1) { + next = path.charAt(i + 1); + } + + if (inBrackets == false && c == '[' && (next == '\'' || next == '"')) { + if (current.isEmpty() == false) { + elements.add(current.toString()); + current.setLength(0); + } + inBrackets = true; + doubleQuote = next == '"'; + i++; + } else if (inBrackets && (c == '\'' || c == '"') && next == ']') { + char expected = doubleQuote ? '"' : '\''; + if (expected != c) { + throw new IllegalArgumentException("path [" + path + "] is not valid"); + } + String element = current.toString(); + elements.add(element); + current.setLength(0); + inBrackets = false; + i++; + } else if (inBrackets == false && c == '.') { + if (current.isEmpty() == false) { + elements.add(current.toString()); + current.setLength(0); + } + } else { + current.append(c); + } + } + + if (inBrackets) { + throw new IllegalArgumentException("path [" + path + "] is not valid"); + } + + if (current.isEmpty() == false) { + elements.add(current.toString()); + } + + return elements.toArray(new String[0]); + } + public Object initialContext(IngestDocument document) { return useIngestContext ? document.getIngestMetadata() : document.getCtxMap(); } diff --git a/server/src/test/java/org/elasticsearch/ingest/IngestDocumentTests.java b/server/src/test/java/org/elasticsearch/ingest/IngestDocumentTests.java index b7120eec3d252..ded7e8f0a6820 100644 --- a/server/src/test/java/org/elasticsearch/ingest/IngestDocumentTests.java +++ b/server/src/test/java/org/elasticsearch/ingest/IngestDocumentTests.java @@ -1227,4 +1227,64 @@ public void testSourceHashMapIsNotCopied() { assertThat(document2.getCtxMap().getMetadata(), not(sameInstance(document1.getCtxMap().getMetadata()))); } } + + public void testPathWithBracketNotation() { + // Test basic bracket notation + IngestDocument document = RandomDocumentPicks.randomIngestDocument(random()); + + // Test bracket notation with single quotes + document.setFieldValue("foo", Map.of("dotted.field", "value1")); + assertThat(document.getFieldValue("foo['dotted.field']", String.class), equalTo("value1")); + assertThat(document.getFieldValue("foo.['dotted.field']", String.class), equalTo("value1")); + + // Test bracket notation with double quotes + assertThat(document.getFieldValue("foo[\"dotted.field\"]", String.class), equalTo("value1")); + + // Test multiple bracket notations mixed with dots + document.setFieldValue("foo", Map.of( + "nested.field", Map.of( + "bar", Map.of( + "another.field", "value2" + ) + ) + )); + assertThat( + document.getFieldValue("foo['nested.field'].bar[\"another.field\"]", String.class), + equalTo("value2") + ); + } + + public void testInvalidBracketNotation() { + IngestDocument document = RandomDocumentPicks.randomIngestDocument(random()); + document.setFieldValue("foo", Map.of("dotted.field", "value")); + + // Missing closing bracket + IllegalArgumentException e1 = expectThrows( + IllegalArgumentException.class, + () -> document.getFieldValue("foo['dotted.field", String.class) + ); + assertThat(e1.getMessage(), containsString("path [foo['dotted.field] is not valid")); + + // Missing quotes + document.setFieldValue("foo[dotted", Map.of("field]", "value")); + + assertThat( + document.getFieldValue("foo[dotted.field]", String.class), + equalTo("value") + ); + + // Mixed quotes + IllegalArgumentException e3 = expectThrows( + IllegalArgumentException.class, + () -> document.getFieldValue("foo['dotted.field\"]", String.class) + ); + assertThat(e3.getMessage(), containsString("path [foo['dotted.field\"]] is not valid")); + + // Empty brackets + IllegalArgumentException e4 = expectThrows( + IllegalArgumentException.class, + () -> document.getFieldValue("['']", String.class) + ); + assertThat(e4.getMessage(), containsString("path [['']] is not valid")); + } } From e8f1a1ffe297aa8581237393aac61f6d7bb6ce92 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Tue, 25 Mar 2025 09:16:38 +0000 Subject: [PATCH 2/3] [CI] Auto commit changes from spotless --- .../ingest/IngestDocumentTests.java | 23 ++++--------------- 1 file changed, 4 insertions(+), 19 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/ingest/IngestDocumentTests.java b/server/src/test/java/org/elasticsearch/ingest/IngestDocumentTests.java index ded7e8f0a6820..c160b5bc72239 100644 --- a/server/src/test/java/org/elasticsearch/ingest/IngestDocumentTests.java +++ b/server/src/test/java/org/elasticsearch/ingest/IngestDocumentTests.java @@ -1241,17 +1241,8 @@ public void testPathWithBracketNotation() { assertThat(document.getFieldValue("foo[\"dotted.field\"]", String.class), equalTo("value1")); // Test multiple bracket notations mixed with dots - document.setFieldValue("foo", Map.of( - "nested.field", Map.of( - "bar", Map.of( - "another.field", "value2" - ) - ) - )); - assertThat( - document.getFieldValue("foo['nested.field'].bar[\"another.field\"]", String.class), - equalTo("value2") - ); + document.setFieldValue("foo", Map.of("nested.field", Map.of("bar", Map.of("another.field", "value2")))); + assertThat(document.getFieldValue("foo['nested.field'].bar[\"another.field\"]", String.class), equalTo("value2")); } public void testInvalidBracketNotation() { @@ -1268,10 +1259,7 @@ public void testInvalidBracketNotation() { // Missing quotes document.setFieldValue("foo[dotted", Map.of("field]", "value")); - assertThat( - document.getFieldValue("foo[dotted.field]", String.class), - equalTo("value") - ); + assertThat(document.getFieldValue("foo[dotted.field]", String.class), equalTo("value")); // Mixed quotes IllegalArgumentException e3 = expectThrows( @@ -1281,10 +1269,7 @@ public void testInvalidBracketNotation() { assertThat(e3.getMessage(), containsString("path [foo['dotted.field\"]] is not valid")); // Empty brackets - IllegalArgumentException e4 = expectThrows( - IllegalArgumentException.class, - () -> document.getFieldValue("['']", String.class) - ); + IllegalArgumentException e4 = expectThrows(IllegalArgumentException.class, () -> document.getFieldValue("['']", String.class)); assertThat(e4.getMessage(), containsString("path [['']] is not valid")); } } From 4985a6af15e2fcc286309ec281e73284e6a1b070 Mon Sep 17 00:00:00 2001 From: Felix Barnsteiner Date: Tue, 25 Mar 2025 11:48:31 +0100 Subject: [PATCH 3/3] Update docs/changelog/125566.yaml --- docs/changelog/125566.yaml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 docs/changelog/125566.yaml diff --git a/docs/changelog/125566.yaml b/docs/changelog/125566.yaml new file mode 100644 index 0000000000000..3088c650c28c7 --- /dev/null +++ b/docs/changelog/125566.yaml @@ -0,0 +1,5 @@ +pr: 125566 +summary: Bracket syntax for accessing dotted field names in ingest processors +area: Ingest Node +type: enhancement +issues: []