From 291108b4b712734cd756745ad4264e0b787b09c3 Mon Sep 17 00:00:00 2001 From: Peter Ansell Date: Sat, 29 Aug 2020 13:22:09 +1000 Subject: [PATCH 1/3] issue #60942 : Add network from MaxMind Geo ASN database Signed-off-by: Peter Ansell --- .../elasticsearch/ingest/geoip/GeoIpProcessor.java | 14 +++++++++++--- .../ingest/geoip/GeoIpProcessorFactoryTests.java | 2 +- .../ingest/geoip/GeoIpProcessorTests.java | 3 ++- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java index 2fe9c76451bc0..7422eacb84a6b 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java @@ -19,6 +19,7 @@ package org.elasticsearch.ingest.geoip; +import com.maxmind.db.Network; import com.maxmind.geoip2.exception.AddressNotFoundException; import com.maxmind.geoip2.model.AsnResponse; import com.maxmind.geoip2.model.CityResponse; @@ -345,6 +346,7 @@ private Map retrieveAsnGeoData(InetAddress ipAddress) { Integer asn = response.getAutonomousSystemNumber(); String organization_name = response.getAutonomousSystemOrganization(); + Network network = response.getNetwork(); Map geoData = new HashMap<>(); for (Property property : this.properties) { @@ -362,6 +364,11 @@ private Map retrieveAsnGeoData(InetAddress ipAddress) { geoData.put("organization_name", organization_name); } break; + case NETWORK: + if (network != null) { + geoData.put("network", network.toString()); + } + break; } } return geoData; @@ -376,7 +383,7 @@ public static final class Factory implements Processor.Factory { Property.CONTINENT_NAME, Property.COUNTRY_ISO_CODE )); static final Set DEFAULT_ASN_PROPERTIES = Collections.unmodifiableSet(EnumSet.of( - Property.IP, Property.ASN, Property.ORGANIZATION_NAME + Property.IP, Property.ASN, Property.ORGANIZATION_NAME, Property.NETWORK )); private final Map databaseReaders; @@ -464,7 +471,8 @@ enum Property { TIMEZONE, LOCATION, ASN, - ORGANIZATION_NAME; + ORGANIZATION_NAME, + NETWORK; static final EnumSet ALL_CITY_PROPERTIES = EnumSet.of( Property.IP, Property.COUNTRY_ISO_CODE, Property.COUNTRY_NAME, Property.CONTINENT_NAME, @@ -475,7 +483,7 @@ enum Property { Property.IP, Property.CONTINENT_NAME, Property.COUNTRY_NAME, Property.COUNTRY_ISO_CODE ); static final EnumSet ALL_ASN_PROPERTIES = EnumSet.of( - Property.IP, Property.ASN, Property.ORGANIZATION_NAME + Property.IP, Property.ASN, Property.ORGANIZATION_NAME, Property.NETWORK ); public static Property parseProperty(String databaseType, String value) { diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorFactoryTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorFactoryTests.java index 86f14c5940313..3f356d512fc67 100644 --- a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorFactoryTests.java +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorFactoryTests.java @@ -189,7 +189,7 @@ public void testBuildWithAsnDbAndCityFields() throws Exception { config.put("properties", Collections.singletonList(cityProperty)); Exception e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, null, config)); assertThat(e.getMessage(), equalTo("[properties] illegal property value [" + cityProperty + - "]. valid values are [IP, ASN, ORGANIZATION_NAME]")); + "]. valid values are [IP, ASN, ORGANIZATION_NAME, NETWORK]")); } public void testBuildNonExistingDbFile() throws Exception { diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorTests.java index c33080891644f..e580b1eb4f823 100644 --- a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorTests.java +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorTests.java @@ -209,10 +209,11 @@ public void testAsn() throws Exception { assertThat(ingestDocument.getSourceAndMetadata().get("source_field"), equalTo(ip)); @SuppressWarnings("unchecked") Map geoData = (Map) ingestDocument.getSourceAndMetadata().get("target_field"); - assertThat(geoData.size(), equalTo(3)); + assertThat(geoData.size(), equalTo(4)); assertThat(geoData.get("ip"), equalTo(ip)); assertThat(geoData.get("asn"), equalTo(1136)); assertThat(geoData.get("organization_name"), equalTo("KPN B.V.")); + assertThat(geoData.get("network"), equalTo("82.168.0.0/14")); } public void testAddressIsNotInTheDatabase() throws Exception { From f2aca418e4d272264ff7f68090b5f7ac6f526229 Mon Sep 17 00:00:00 2001 From: Peter Ansell Date: Mon, 31 Aug 2020 20:51:10 +1000 Subject: [PATCH 2/3] issue #60942 : Add new property to YAML test Signed-off-by: Peter Ansell --- .../rest-api-spec/test/ingest_geoip/20_geoip_processor.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/ingest-geoip/src/yamlRestTest/resources/rest-api-spec/test/ingest_geoip/20_geoip_processor.yml b/modules/ingest-geoip/src/yamlRestTest/resources/rest-api-spec/test/ingest_geoip/20_geoip_processor.yml index f6bdce0532ace..84a942a37cce3 100644 --- a/modules/ingest-geoip/src/yamlRestTest/resources/rest-api-spec/test/ingest_geoip/20_geoip_processor.yml +++ b/modules/ingest-geoip/src/yamlRestTest/resources/rest-api-spec/test/ingest_geoip/20_geoip_processor.yml @@ -299,7 +299,8 @@ index: test id: 1 - match: { _source.field1: "82.171.64.0" } - - length: { _source.geoip: 3 } + - length: { _source.geoip: 4 } - match: { _source.geoip.ip: "82.171.64.0" } - match: { _source.geoip.asn: 1136 } - match: { _source.geoip.organization_name: "KPN B.V." } + - match: { _source.geoip.network: "82.168.0.0/14" } From 9cd54ba90aabe722e7140d03597942f70b7e92e1 Mon Sep 17 00:00:00 2001 From: Peter Ansell Date: Sat, 5 Sep 2020 19:18:54 +1000 Subject: [PATCH 3/3] issue #60942 : Add network field to geoip processor docs Signed-off-by: Peter Ansell --- docs/reference/ingest/processors/geoip.asciidoc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/reference/ingest/processors/geoip.asciidoc b/docs/reference/ingest/processors/geoip.asciidoc index debdbf4ed6f72..4ce75e5f2b462 100644 --- a/docs/reference/ingest/processors/geoip.asciidoc +++ b/docs/reference/ingest/processors/geoip.asciidoc @@ -40,9 +40,10 @@ and `location`. The fields actually added depend on what has been found and whic `country_iso_code`, `country_name` and `continent_name`. The fields actually added depend on what has been found and which properties were configured in `properties`. * If the GeoLite2 ASN database is used, then the following fields may be added under the `target_field`: `ip`, -`asn`, and `organization_name`. The fields actually added depend on what has been found and which properties were configured +`asn`, `organization_name` and `network`. The fields actually added depend on what has been found and which properties were configured in `properties`. + Here is an example that uses the default city database and adds the geographical information to the `geoip` field based on the `ip` field: [source,console]