Merge pull request #303 from mfdz/issue_302

Fix #302: Read address attributes from nominatim

es/mappings.json

@@ -483,7 +483,85 @@
 						]
 					}
 				}
-			}
+			},
+			"district": {
+				"properties": {
+					"de": {
+						"type": "text",
+						"index": false,
+						"copy_to": [
+							"collector.de"
+						]
+					},
+					"default": {
+						"type": "text",
+						"index": false,
+						"copy_to": [
+							"collector.default"
+						]
+					},
+					"en": {
+						"type": "text",
+						"index": false,
+						"copy_to": [
+							"collector.en"
+						]
+					},
+					"fr": {
+						"type": "text",
+						"index": false,
+						"copy_to": [
+							"collector.fr"
+						]
+					},
+					"it": {
+						"type": "text",
+						"index": false,
+						"copy_to": [
+							"collector.it"
+						]
+					}
+				}
+			},
+	  		"locality": {
+				"properties": {
+					"de": {
+						"type": "text",
+						"index": false,
+						"copy_to": [
+							"collector.de"
+						]
+					},
+					"default": {
+						"type": "text",
+						"index": false,
+						"copy_to": [
+							"collector.default"
+						]
+					},
+					"en": {
+						"type": "text",
+						"index": false,
+						"copy_to": [
+							"collector.en"
+						]
+					},
+					"fr": {
+						"type": "text",
+						"index": false,
+						"copy_to": [
+							"collector.fr"
+						]
+					},
+					"it": {
+						"type": "text",
+						"index": false,
+						"copy_to": [
+							"collector.it"
+						]
+					}
+				}
+			}		
 		}
 	}
 }

src/main/java/de/komoot/photon/Constants.java

@@ -12,6 +12,8 @@ public class Constants {
     public static final String COUNTRY = "country";
     public static final String COUNTRYCODE = "countrycode";
     public static final String CITY = "city";
+    public static final String DISTRICT = "district";
+    public static final String LOCALITY = "locality";
     public static final String STREET = "street";
     public static final String STATE = "state";
     public static final String TYPE = "type";

src/main/java/de/komoot/photon/PhotonDoc.java

@@ -1,14 +1,14 @@
 package de.komoot.photon;
 
+import com.google.common.collect.ImmutableMap;
 import com.neovisionaries.i18n.CountryCode;
 import com.vividsolutions.jts.geom.Envelope;
 import com.vividsolutions.jts.geom.Point;
 import lombok.Getter;
 import lombok.Setter;
+import lombok.extern.slf4j.Slf4j;
 
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
+import java.util.*;
 
 /**
  * denormalized doc with all information needed be dumped to elasticsearch
@@ -17,6 +17,7 @@
  */
 @Getter
 @Setter
+@Slf4j
 public class PhotonDoc {
     final private long placeId;
     final private String osmType;
@@ -25,6 +26,7 @@ public class PhotonDoc {
     final private String tagValue;
     final private Map<String, String> name;
     private String postcode;
+    final private Map<String, String> address;
     final private Map<String, String> extratags;
     final private Envelope bbox;
     final private long parentPlaceId; // 0 if unset
@@ -34,14 +36,16 @@ public class PhotonDoc {
     final private int rankSearch;
 
     private Map<String, String> street;
+    private Map<String, String> locality;
+    private Map<String, String> district;
     private Map<String, String> city;
     private Set<Map<String, String>> context = new HashSet<Map<String, String>>();
     private Map<String, String> country;
     private Map<String, String> state;
     private String houseNumber;
     private Point centroid;
 
-    public PhotonDoc(long placeId, String osmType, long osmId, String tagKey, String tagValue, Map<String, String> name, String houseNumber, Map<String, String> extratags, Envelope bbox, long parentPlaceId, double importance, CountryCode countryCode, Point centroid, long linkedPlaceId, int rankSearch) {
+    public PhotonDoc(long placeId, String osmType, long osmId, String tagKey, String tagValue, Map<String, String> name, String houseNumber, Map<String, String> address, Map<String, String> extratags, Envelope bbox, long parentPlaceId, double importance, CountryCode countryCode, Point centroid, long linkedPlaceId, int rankSearch) {
         String place = extratags != null ? extratags.get("place") : null;
         if (place != null) {
             // take more specific extra tag information
@@ -56,6 +60,7 @@ public PhotonDoc(long placeId, String osmType, long osmId, String tagKey, String
         this.tagValue = tagValue;
         this.name = name;
         this.houseNumber = houseNumber;
+        this.address = address;
         this.extratags = extratags;
         this.bbox = bbox;
         this.parentPlaceId = parentPlaceId;
@@ -75,6 +80,7 @@ public PhotonDoc(PhotonDoc other) {
         this.name = other.name;
         this.houseNumber = other.houseNumber;
         this.postcode = other.postcode;
+        this.address = other.address;
         this.extratags = other.extratags;
         this.bbox = other.bbox;
         this.parentPlaceId = other.parentPlaceId;
@@ -84,6 +90,8 @@ public PhotonDoc(PhotonDoc other) {
         this.linkedPlaceId = other.linkedPlaceId;
         this.rankSearch = other.rankSearch;
         this.street = other.street;
+        this.locality = other.locality;
+        this.district = other.district;
         this.city = other.city;
         this.context = other.context;
         this.country = other.country;
@@ -102,7 +110,7 @@ public String getUid() {
      */
     public static PhotonDoc create(long placeId, String osmType, long osmId, Map<String, String> nameMap) {
         return new PhotonDoc(placeId, osmType, osmId, "", "", nameMap,
-                "", null, null, 0, 0, null, null, 0, 0);
+                "", null, null, null, 0, 0, null, null, 0, 0);
     }
 
     public boolean isUsefulForIndex() {
@@ -116,4 +124,63 @@ public boolean isUsefulForIndex() {
 
         return true;
     }
+
+    /**
+     * Complete doc from nominatim address information.
+     */
+    public void completeFromAddress() {
+        if (address == null) return;
+
+        String addressStreet = address.get("street");
+        if (addressStreet != null) {
+            this.street = nullToEmptyMap(this.street);
+            setOrReplace(addressStreet, this.street, "street");
+        }
+
+        String addressCity = address.get("city");
+        if (addressCity != null) {
+            this.city = nullToEmptyMap(this.city);
+            setOrReplace(addressCity, this.city, "city");
+        }
+
+        String addressDistrict = address.get("suburb");
+        if (addressDistrict != null) {
+            this.district = nullToEmptyMap(this.district);
+            setOrReplace(addressDistrict, this.district, "suburb");
+        }
+
+        String addressLocality = address.get("neighbourhood");
+        if (addressLocality != null) {
+            this.locality = nullToEmptyMap(this.locality);
+            setOrReplace(addressLocality, this.locality, "neighbourhood");
+        }
+
+        String addressPostCode = address.get("postcode");
+        if (addressPostCode != null && !addressPostCode.equals(this.postcode)) {
+            if (log.isDebugEnabled()) {
+                log.debug("Replacing postcode "+this.postcode+" with "+ addressPostCode+ " for osmId #" + osmId);
+            }
+            this.postcode = addressPostCode;
+        }
+    }
+
+    private static Map<String, String> nullToEmptyMap(Map<String, String> map) {
+        if (map == null) {
+            return new HashMap<>();
+        } else return map;
+    }
+
+    private void setOrReplace(String name, Map<String, String> namesMap, String field) {
+        String existingName = namesMap.get("name");
+        if (!name.equals(existingName)) {
+            if (log.isDebugEnabled()) {
+                log.debug("Replacing "+ field +" name '"+existingName+"' with '"+ name+ "' for osmId #" + osmId);
+            }
+            // we keep the former name in the context as it might be helpful when looking up typos
+            if(!Objects.isNull(existingName)) {
+                context.add(ImmutableMap.of("formerName", existingName));
+            }
+            namesMap.put("name", name);
+        }
+    }
 }

src/main/java/de/komoot/photon/Utils.java

@@ -53,6 +53,8 @@ public static XContentBuilder convert(PhotonDoc doc, String[] languages) throws
             builder.field(Constants.COUNTRYCODE, countryCode.getAlpha2());
         writeIntlNames(builder, doc.getState(), "state", languages);
         writeIntlNames(builder, doc.getStreet(), "street", languages);
+        writeIntlNames(builder, doc.getLocality(), "locality", languages);
+        writeIntlNames(builder, doc.getDistrict(), "district", languages);
         writeContext(builder, doc.getContext(), languages);
         writeExtent(builder, doc.getBbox());
 

src/main/java/de/komoot/photon/elasticsearch/Server.java

@@ -233,6 +233,8 @@ private JSONObject addLangsToMapping(JSONObject mappingsObject) {
                 propertiesObject = addToCollector("country", propertiesObject, copyToCollectorObject, lang);
                 propertiesObject = addToCollector("state", propertiesObject, copyToCollectorObject, lang);
                 propertiesObject = addToCollector("street", propertiesObject, copyToCollectorObject, lang);
+                propertiesObject = addToCollector("district", propertiesObject, copyToCollectorObject, lang);
+                propertiesObject = addToCollector("locality", propertiesObject, copyToCollectorObject, lang);
                 propertiesObject = addToCollector("name", propertiesObject, nameToCollectorObject, lang);
 
                 // add language specific collector to default for name

src/main/java/de/komoot/photon/nominatim/NominatimConnector.java

@@ -143,6 +143,7 @@ public NominatimResult mapRow(ResultSet rs, int rownum) throws SQLException {
                     "house_number",
                     Collections.<String, String>emptyMap(), // no name
                     (String) null,
+                    Collections.<String, String>emptyMap(), // no address
                     Collections.<String, String>emptyMap(), // no extratags
                     (Envelope) null,
                     rs.getLong("parent_place_id"),
@@ -186,6 +187,7 @@ public NominatimResult mapRow(ResultSet rs, int rowNum) throws SQLException {
                     rs.getString("type"),
                     DBUtils.getMap(rs, "name"),
                     (String) null,
+                    DBUtils.getMap(rs, "address"),
                     DBUtils.getMap(rs, "extratags"),
                     envelope,
                     rs.getLong("parent_place_id"),
@@ -205,7 +207,7 @@ public NominatimResult mapRow(ResultSet rs, int rowNum) throws SQLException {
             return result;
         }
     };
-    private final String selectColsPlaceX = "place_id, osm_type, osm_id, class, type, name, housenumber, postcode, extratags, ST_Envelope(geometry) AS bbox, parent_place_id, linked_place_id, rank_search, importance, country_code, centroid";
+    private final String selectColsPlaceX = "place_id, osm_type, osm_id, class, type, name, housenumber, postcode, address, extratags, ST_Envelope(geometry) AS bbox, parent_place_id, linked_place_id, rank_search, importance, country_code, centroid";
     private final String selectColsOsmline = "place_id, osm_id, parent_place_id, startnumber, endnumber, interpolationtype, postcode, country_code, linegeo";
     private final String selectColsAddress = "p.place_id, p.name, p.class, p.type, p.rank_address";
     private Importer importer;
@@ -293,7 +295,7 @@ public AddressRow mapRow(ResultSet rs, int rowNum) throws SQLException {
         return terms;
     }
 
-    private static final PhotonDoc FINAL_DOCUMENT = new PhotonDoc(0, null, 0, null, null, null, null, null, null, 0, 0, null, null, 0, 0);
+    private static final PhotonDoc FINAL_DOCUMENT = new PhotonDoc(0, null, 0, null, null, null, null, null, null, null, 0, 0, null, null, 0, 0);
 
     private class ImportThread implements Runnable {
         private final BlockingQueue<PhotonDoc> documents;
@@ -471,6 +473,16 @@ private void completePlace(PhotonDoc doc) {
                 continue;
             }
 
+            if (address.isLocality() && doc.getLocality() == null) {
+                doc.setLocality(address.getName());
+                continue;
+            }
+
+            if (address.isDistrict() && doc.getDistrict() == null) {
+                doc.setDistrict(address.getName());
+                continue;
+            }
+
             if (address.isState() && doc.getState() == null) {
                 doc.setState(address.getName());
                 continue;
@@ -481,5 +493,8 @@ private void completePlace(PhotonDoc doc) {
                 doc.getContext().add(address.getName());
             }
         }
+        // finally, overwrite gathered information with higher prio
+        // address info from nominatim which should have precedence
+        doc.completeFromAddress();
     }
 }

src/main/java/de/komoot/photon/nominatim/model/AddressRow.java

@@ -25,6 +25,19 @@ public boolean isStreet() {
         return 26 <= rankAddress && rankAddress < 28;
     }
 
+    /**
+     * @return whether nominatim thinks this place is a locality (=neighbourhood)
+     */
+    public boolean isLocality() {
+        return 22 <= rankAddress && rankAddress < 26;
+    }
+
+    /**
+     * @return whether nominatim thinks this place is a district (=suburb)
+     */
+    public boolean isDistrict() {
+        return 17 <= rankAddress && rankAddress < 22;
+    }
     public boolean isCity() {
         return 13 <= rankAddress && rankAddress <= 16;
     }

src/main/java/de/komoot/photon/utils/ConvertToJson.java

@@ -19,15 +19,16 @@
 @Slf4j
 public class ConvertToJson {
     private final static String[] KEYS_LANG_UNSPEC = {Constants.OSM_ID, Constants.OSM_VALUE, Constants.OSM_KEY, Constants.POSTCODE, Constants.HOUSENUMBER, Constants.COUNTRYCODE, Constants.OSM_TYPE};
-    private final static String[] KEYS_LANG_SPEC = {Constants.NAME, Constants.COUNTRY, Constants.CITY, Constants.STREET, Constants.STATE};
+    private final static String[] KEYS_LANG_SPEC = {Constants.NAME, Constants.COUNTRY, Constants.CITY, Constants.DISTRICT, Constants.LOCALITY, Constants.STREET, Constants.STATE};
+
     private final String lang;
 
     public ConvertToJson(String lang) {
         this.lang = lang;
     }
 
     public List<JSONObject> convert(SearchResponse searchResponse) {
-        SearchHit[] hits = searchResponse.getHits().hits();
+        SearchHit[] hits = searchResponse.getHits().getHits();
         final List<JSONObject> list = Lists.newArrayListWithExpectedSize(hits.length);
         for (SearchHit hit : hits) {
             final Map<String, Object> source = hit.getSource();

src/test/java/de/komoot/photon/ESBaseTester.java

@@ -37,7 +37,7 @@ public class ESBaseTester {
     private PhotonDoc createDoc(double lon, double lat, int id, int osmId, String key, String value) {
         ImmutableMap<String, String> nameMap = ImmutableMap.of("name", "berlin");
         Point location = FACTORY.createPoint(new Coordinate(lon, lat));
-        return new PhotonDoc(id, "way", osmId, key, value, nameMap, null, null, null, 0, 0.5, null, location, 0, 0);
+        return new PhotonDoc(id, "way", osmId, key, value, nameMap, null, null, null, null, 0, 0.5, null, location, 0, 0);
     }
 
     @Before