From 4602602fc977cf07bfcf6995eed521bcdd30ab04 Mon Sep 17 00:00:00 2001 From: Isaiah Inuwa Date: Fri, 31 May 2019 10:15:20 -0500 Subject: [PATCH 1/2] Allow dots in field names. --- .../inputs/extractors/JsonExtractor.java | 2 +- .../java/org/graylog2/plugin/Message.java | 27 +------------------ .../inputs/extractors/JsonExtractorTest.java | 12 ++++----- .../java/org/graylog2/plugin/MessageTest.java | 6 ++--- 4 files changed, 11 insertions(+), 36 deletions(-) diff --git a/graylog2-server/src/main/java/org/graylog2/inputs/extractors/JsonExtractor.java b/graylog2-server/src/main/java/org/graylog2/inputs/extractors/JsonExtractor.java index 1c719bc4dc1c..563d8b25d6b7 100644 --- a/graylog2-server/src/main/java/org/graylog2/inputs/extractors/JsonExtractor.java +++ b/graylog2-server/src/main/java/org/graylog2/inputs/extractors/JsonExtractor.java @@ -85,7 +85,7 @@ public JsonExtractor(final MetricRegistry metricRegistry, this.flatten = firstNonNull((Boolean) extractorConfig.get(CK_FLATTEN), false); this.listSeparator = firstNonNull((String) extractorConfig.get(CK_LIST_SEPARATOR), ", "); - this.keySeparator = firstNonNull((String) extractorConfig.get(CK_KEY_SEPARATOR), "_"); + this.keySeparator = firstNonNull((String) extractorConfig.get(CK_KEY_SEPARATOR), "."); this.kvSeparator = firstNonNull((String) extractorConfig.get(CK_KV_SEPARATOR), "="); this.replaceKeyWhitespace = firstNonNull((Boolean) extractorConfig.get(CK_REPLACE_KEY_WHITESPACE), false); this.keyWhitespaceReplacement = firstNonNull((String) extractorConfig.get(CK_KEY_WHITESPACE_REPLACEMENT), "_"); diff --git a/graylog2-server/src/main/java/org/graylog2/plugin/Message.java b/graylog2-server/src/main/java/org/graylog2/plugin/Message.java index 7e39f781b5f7..b08f59becc8b 100644 --- a/graylog2-server/src/main/java/org/graylog2/plugin/Message.java +++ b/graylog2-server/src/main/java/org/graylog2/plugin/Message.java @@ -247,32 +247,7 @@ public Map toElasticSearchObject(@Nonnull final Meter invalidTim } final Object value = entry.getValue(); - // Elasticsearch does not allow "." characters in keys since version 2.0. - // See: https://www.elastic.co/guide/en/elasticsearch/reference/2.0/breaking_20_mapping_changes.html#_field_names_may_not_contain_dots - if (key.contains(".")) { - final String newKey = key.replace('.', KEY_REPLACEMENT_CHAR); - - // If the message already contains the transformed key, we skip the field and emit a warning. - // This is still not optimal but better than implementing expensive logic with multiple replacement - // character options. Conflicts should be rare... - if (!obj.containsKey(newKey)) { - obj.put(newKey, value); - } else { - LOG.warn("Keys must not contain a \".\" character! Ignoring field \"{}\"=\"{}\" in message [{}] - Unable to replace \".\" with a \"{}\" because of key conflict: \"{}\"=\"{}\"", - key, value, getId(), KEY_REPLACEMENT_CHAR, newKey, obj.get(newKey)); - LOG.debug("Full message with \".\" in message key: {}", this); - } - } else { - if (obj.containsKey(key)) { - final String newKey = key.replace(KEY_REPLACEMENT_CHAR, '.'); - // Deliberate warning duplicates because the key with the "." might be transformed before reaching - // the duplicate original key with a "_". Otherwise we would silently overwrite the transformed key. - LOG.warn("Keys must not contain a \".\" character! Ignoring field \"{}\"=\"{}\" in message [{}] - Unable to replace \".\" with a \"{}\" because of key conflict: \"{}\"=\"{}\"", - newKey, fields.get(newKey), getId(), KEY_REPLACEMENT_CHAR, key, value); - LOG.debug("Full message with \".\" in message key: {}", this); - } - obj.put(key, value); - } + obj.put(key, value); } obj.put(FIELD_MESSAGE, getMessage()); diff --git a/graylog2-server/src/test/java/org/graylog2/inputs/extractors/JsonExtractorTest.java b/graylog2-server/src/test/java/org/graylog2/inputs/extractors/JsonExtractorTest.java index 40976047a8d0..1f561038389e 100644 --- a/graylog2-server/src/test/java/org/graylog2/inputs/extractors/JsonExtractorTest.java +++ b/graylog2-server/src/test/java/org/graylog2/inputs/extractors/JsonExtractorTest.java @@ -94,10 +94,10 @@ public void testRunWithObject() throws Exception { final Extractor.Result[] results = jsonExtractor.run(value); assertThat(results).contains( - new Extractor.Result("foobar", "object_text", -1, -1), - new Extractor.Result(1234.5678, "object_number", -1, -1), - new Extractor.Result(true, "object_bool", -1, -1), - new Extractor.Result("foobar", "object_nested_text", -1, -1) + new Extractor.Result("foobar", "object.text", -1, -1), + new Extractor.Result(1234.5678, "object.number", -1, -1), + new Extractor.Result(true, "object.bool", -1, -1), + new Extractor.Result("foobar", "object.nested.text", -1, -1) ); } @@ -158,7 +158,7 @@ public void issue2375() throws Exception { assertThat(results).contains( new Extractor.Result("Myserver#DS", "Source", -1, -1), - new Extractor.Result("Purge Vector", "Target_Data_Attribute Name", -1, -1) + new Extractor.Result("Purge Vector", "Target.Data.Attribute Name", -1, -1) ); } @@ -213,7 +213,7 @@ public void testRunWithWhitespaceInNestedKey() throws Exception { ""); assertThat(jsonExtractor.run(value)).containsOnly( - new Extractor.Result(42, "foo_b-a-r_b-a-z", -1, -1) + new Extractor.Result(42, "foo.b-a-r.b-a-z", -1, -1) ); } diff --git a/graylog2-server/src/test/java/org/graylog2/plugin/MessageTest.java b/graylog2-server/src/test/java/org/graylog2/plugin/MessageTest.java index a5b0b10a5149..6289d9fc0e6b 100644 --- a/graylog2-server/src/test/java/org/graylog2/plugin/MessageTest.java +++ b/graylog2-server/src/test/java/org/graylog2/plugin/MessageTest.java @@ -409,9 +409,9 @@ public void testToElasticSearchObjectWithInvalidKey() throws Exception { final Map object = message.toElasticSearchObject(invalidTimestampMeter); - // Elasticsearch >=2.0 does not allow "." in keys. Make sure we replace them before writing the message. - assertEquals("#toElasticsearchObject() should replace \".\" in keys with a \"_\"", - "dot", object.get("field_3")); + // Elasticsearch >=2.0 and < 5.0 does not allow "." in keys. Make sure we replace them before writing the message. + assertEquals("#toElasticsearchObject() should not replace \".\" in keys with a \"_\"", + "dot", object.get("field.3")); assertEquals("foo", object.get("message")); assertEquals("bar", object.get("source")); From 548b69e082ee875b37ebe182055d93aa8fd0ed8e Mon Sep 17 00:00:00 2001 From: Isaiah Inuwa Date: Fri, 31 May 2019 16:12:22 -0500 Subject: [PATCH 2/2] Allow parsing of arrays one level deep as strings. --- .../org/graylog2/inputs/codecs/GelfCodec.java | 13 ++++++++++++- .../graylog2/inputs/codecs/GelfCodecTest.java | 17 +++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/graylog2-server/src/main/java/org/graylog2/inputs/codecs/GelfCodec.java b/graylog2-server/src/main/java/org/graylog2/inputs/codecs/GelfCodec.java index af2c52856733..33a431185335 100644 --- a/graylog2-server/src/main/java/org/graylog2/inputs/codecs/GelfCodec.java +++ b/graylog2-server/src/main/java/org/graylog2/inputs/codecs/GelfCodec.java @@ -45,7 +45,10 @@ import javax.annotation.Nonnull; import javax.annotation.Nullable; import javax.inject.Inject; + +import java.util.ArrayList; import java.util.Iterator; +import java.util.List; import java.util.Map; @Codec(name = "gelf", displayName = "GELF") @@ -205,7 +208,15 @@ public Message decode(@Nonnull final RawMessage rawMessage) { final JsonNode value = entry.getValue(); final Object fieldValue; - if (value.isContainerNode()) { + if (value.isArray()) { + List fieldArray = new ArrayList<>(); + + for (final JsonNode objNode : value) { + fieldArray.add(objNode.asText()); + } + fieldValue = fieldArray; + } + else if (value.isContainerNode()) { fieldValue = value.toString(); } else if (value.isFloatingPointNumber()) { fieldValue = value.asDouble(); diff --git a/graylog2-server/src/test/java/org/graylog2/inputs/codecs/GelfCodecTest.java b/graylog2-server/src/test/java/org/graylog2/inputs/codecs/GelfCodecTest.java index 710fd697032b..54cf396c4e3e 100644 --- a/graylog2-server/src/test/java/org/graylog2/inputs/codecs/GelfCodecTest.java +++ b/graylog2-server/src/test/java/org/graylog2/inputs/codecs/GelfCodecTest.java @@ -34,6 +34,7 @@ import java.nio.charset.StandardCharsets; import java.time.DateTimeException; import java.util.Collections; +import java.util.List; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatIllegalArgumentException; @@ -408,4 +409,20 @@ public void decodeSucceedsWithValidTimestampAsStringIssue4027() throws Exception assertThat(message).isNotNull(); assertThat(message.getTimestamp()).isEqualTo(DateTime.parse("2017-07-21T14:23:00.661Z")); } + + @Test + public void decodeSucceedsWithCustomFieldAsArray() throws Exception { + final String json = "{" + + "\"version\": \"1.1\"," + + "\"short_message\": \"A short message that helps you identify what is going on\"," + + "\"host\": \"example.org\"," + + "\"timestamp\": \"1500646980.661\"," + + "\"tags\": [\"tag1\",\"tag2\"]" + + "}"; + final RawMessage rawMessage = new RawMessage(json.getBytes(StandardCharsets.UTF_8)); + + final Message message = codec.decode(rawMessage); + assertThat("tag2").isEqualTo((message.getFieldAs(List.class, "tags")).get(1)); + + } }