Add region ISO code to GeoIP Ingest plugin (#31669)

Author: nickpeihl

docs/plugins/ingest-geoip.asciidoc

@@ -26,14 +26,14 @@ include::install_remove.asciidoc[]
 | `field`                | yes       | -                                                                                  | The field to get the ip address from for the geographical lookup.
 | `target_field`         | no        | geoip                                                                              | The field that will hold the geographical information looked up from the Maxmind database.
 | `database_file`        | no        | GeoLite2-City.mmdb                                                                 | The database filename in the geoip config directory. The ingest-geoip plugin ships with the GeoLite2-City.mmdb, GeoLite2-Country.mmdb and GeoLite2-ASN.mmdb files.
-| `properties`           | no        | [`continent_name`, `country_iso_code`, `region_name`, `city_name`, `location`] *   | Controls what properties are added to the `target_field` based on the geoip lookup.
+| `properties`           | no        | [`continent_name`, `country_iso_code`, `region_iso_code`, `region_name`, `city_name`, `location`] *   | Controls what properties are added to the `target_field` based on the geoip lookup.
 | `ignore_missing`       | no        | `false`                                                                            | If `true` and `field` does not exist, the processor quietly exits without modifying the document
 |======
 
 *Depends on what is available in `database_field`:
 
 * If the GeoLite2 City database is used, then the following fields may be added under the `target_field`: `ip`,
-`country_iso_code`, `country_name`, `continent_name`, `region_name`, `city_name`, `timezone`, `latitude`, `longitude`
+`country_iso_code`, `country_name`, `continent_name`, `region_iso_code`, `region_name`, `city_name`, `timezone`, `latitude`, `longitude`
 and `location`. The fields actually added depend on what has been found and which properties were configured in `properties`.
 * If the GeoLite2 Country database is used, then the following fields may be added under the `target_field`: `ip`,
 `country_iso_code`, `country_name` and `continent_name`. The fields actually added depend on what has been found and which properties

modules/mapper-extras/src/test/resources/org/elasticsearch/index/mapper/filebeat-6.0.template.json

@@ -1,7 +1,5 @@
 {
-  "index_patterns": [
-    "filebeat-6.0.0-*"
-  ],
+  "index_patterns": ["filebeat-6.0.0-*"],
   "mappings": {
     "doc": {
       "_meta": {
@@ -67,12 +65,14 @@
                       "type": "keyword"
                     },
                     "country_iso_code": {
-                      "ignore_above": 1024,
                       "type": "keyword"
                     },
                     "location": {
                       "type": "geo_point"
                     },
+                    "region_iso_code": {
+                      "type": "keyword"
+                    },
                     "region_name": {
                       "ignore_above": 1024,
                       "type": "keyword"

plugins/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java

@@ -185,6 +185,16 @@ private Map<String, Object> retrieveCityGeoData(InetAddress ipAddress) {
                         geoData.put("continent_name", continentName);
                     }
                     break;
+                case REGION_ISO_CODE:
+                    // ISO 3166-2 code for country subdivisions.
+                    // See iso.org/iso-3166-country-codes.html
+                    String countryIso = country.getIsoCode();
+                    String subdivisionIso = subdivision.getIsoCode();
+                    if (countryIso != null && subdivisionIso != null) {
+                        String regionIsoCode = countryIso + "-" + subdivisionIso;
+                        geoData.put("region_iso_code", regionIsoCode);
+                    }
+                    break;
                 case REGION_NAME:
                     String subdivisionName = subdivision.getName();
                     if (subdivisionName != null) {
@@ -300,8 +310,8 @@ private Map<String, Object> retrieveAsnGeoData(InetAddress ipAddress) {
 
     public static final class Factory implements Processor.Factory {
         static final Set<Property> DEFAULT_CITY_PROPERTIES = EnumSet.of(
-            Property.CONTINENT_NAME, Property.COUNTRY_ISO_CODE, Property.REGION_NAME,
-            Property.CITY_NAME, Property.LOCATION
+            Property.CONTINENT_NAME, Property.COUNTRY_ISO_CODE, Property.REGION_ISO_CODE,
+            Property.REGION_NAME, Property.CITY_NAME, Property.LOCATION
         );
         static final Set<Property> DEFAULT_COUNTRY_PROPERTIES = EnumSet.of(
             Property.CONTINENT_NAME, Property.COUNTRY_ISO_CODE
@@ -377,6 +387,7 @@ enum Property {
         COUNTRY_ISO_CODE,
         COUNTRY_NAME,
         CONTINENT_NAME,
+        REGION_ISO_CODE,
         REGION_NAME,
         CITY_NAME,
         TIMEZONE,
@@ -386,7 +397,8 @@ enum Property {
 
         static final EnumSet<Property> ALL_CITY_PROPERTIES = EnumSet.of(
             Property.IP, Property.COUNTRY_ISO_CODE, Property.COUNTRY_NAME, Property.CONTINENT_NAME,
-            Property.REGION_NAME, Property.CITY_NAME, Property.TIMEZONE, Property.LOCATION
+            Property.REGION_ISO_CODE, Property.REGION_NAME, Property.CITY_NAME, Property.TIMEZONE,
+            Property.LOCATION
         );
         static final EnumSet<Property> ALL_COUNTRY_PROPERTIES = EnumSet.of(
             Property.IP, Property.CONTINENT_NAME, Property.COUNTRY_NAME, Property.COUNTRY_ISO_CODE

plugins/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorFactoryTests.java

@@ -284,7 +284,7 @@ public void testBuildIllegalFieldOption() throws Exception {
         config1.put("properties", Collections.singletonList("invalid"));
         Exception e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, config1));
         assertThat(e.getMessage(), equalTo("[properties] illegal property value [invalid]. valid values are [IP, COUNTRY_ISO_CODE, " +
-            "COUNTRY_NAME, CONTINENT_NAME, REGION_NAME, CITY_NAME, TIMEZONE, LOCATION]"));
+            "COUNTRY_NAME, CONTINENT_NAME, REGION_ISO_CODE, REGION_NAME, CITY_NAME, TIMEZONE, LOCATION]"));
 
         Map<String, Object> config2 = new HashMap<>();
         config2.put("field", "_field");

plugins/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorTests.java

@@ -117,11 +117,12 @@ public void testCity_withIpV6() throws Exception {
         assertThat(ingestDocument.getSourceAndMetadata().get("source_field"), equalTo(address));
         @SuppressWarnings("unchecked")
         Map<String, Object> geoData = (Map<String, Object>) ingestDocument.getSourceAndMetadata().get("target_field");
-        assertThat(geoData.size(), equalTo(8));
+        assertThat(geoData.size(), equalTo(9));
         assertThat(geoData.get("ip"), equalTo(address));
         assertThat(geoData.get("country_iso_code"), equalTo("US"));
         assertThat(geoData.get("country_name"), equalTo("United States"));
         assertThat(geoData.get("continent_name"), equalTo("North America"));
+        assertThat(geoData.get("region_iso_code"), equalTo("US-FL"));
         assertThat(geoData.get("region_name"), equalTo("Florida"));
         assertThat(geoData.get("city_name"), equalTo("Hollywood"));
         assertThat(geoData.get("timezone"), equalTo("America/New_York"));

plugins/ingest-geoip/src/test/resources/rest-api-spec/test/ingest_geoip/20_geoip_processor.yml

@@ -30,11 +30,12 @@
         type: test
         id: 1
   - match: { _source.field1: "128.101.101.101" }
-  - length: { _source.geoip: 5 }
+  - length: { _source.geoip: 6 }
   - match: { _source.geoip.city_name: "Minneapolis" }
   - match: { _source.geoip.country_iso_code: "US" }
   - match: { _source.geoip.location.lon: -93.2166 }
   - match: { _source.geoip.location.lat: 44.9759 }
+  - match: { _source.geoip.region_iso_code: "US-MN" }
   - match: { _source.geoip.region_name: "Minnesota" }
   - match: { _source.geoip.continent_name: "North America" }
 
@@ -54,7 +55,7 @@
               {
                 "geoip" : {
                   "field" : "field1",
-                  "properties" : ["city_name", "country_iso_code", "ip", "location", "timezone", "country_name", "region_name", "continent_name"]
+                  "properties" : ["city_name", "country_iso_code", "ip", "location", "timezone", "country_name", "region_iso_code", "region_name", "continent_name"]
                 }
               }
             ]
@@ -75,14 +76,15 @@
         type: test
         id: 1
   - match: { _source.field1: "128.101.101.101" }
-  - length: { _source.geoip: 8 }
+  - length: { _source.geoip: 9 }
   - match: { _source.geoip.city_name: "Minneapolis" }
   - match: { _source.geoip.country_iso_code: "US" }
   - match: { _source.geoip.ip: "128.101.101.101" }
   - match: { _source.geoip.location.lon: -93.2166 }
   - match: { _source.geoip.location.lat: 44.9759 }
   - match: { _source.geoip.timezone: "America/Chicago" }
   - match: { _source.geoip.country_name: "United States" }
+  - match: { _source.geoip.region_iso_code: "US-MN" }
   - match: { _source.geoip.region_name: "Minnesota" }
   - match: { _source.geoip.continent_name: "North America" }
 
@@ -188,11 +190,12 @@
         type: test
         id: 2
   - match: { _source.field1: "128.101.101.101" }
-  - length: { _source.geoip: 5 }
+  - length: { _source.geoip: 6 }
   - match: { _source.geoip.city_name: "Minneapolis" }
   - match: { _source.geoip.country_iso_code: "US" }
   - match: { _source.geoip.location.lon: -93.2166 }
   - match: { _source.geoip.location.lat: 44.9759 }
+  - match: { _source.geoip.region_iso_code: "US-MN" }
   - match: { _source.geoip.region_name: "Minnesota" }
   - match: { _source.geoip.continent_name: "North America" }