Add neighbourhood and suburb

Signed-off-by: Holger Bruch <[email protected]>

es/mappings.json

@@ -477,7 +477,85 @@
 						]
 					}
 				}
-			}
+			},
+			"suburb": {
+				"properties": {
+					"de": {
+						"type": "text",
+						"index": false,
+						"copy_to": [
+							"collector.de"
+						]
+					},
+					"default": {
+						"type": "text",
+						"index": false,
+						"copy_to": [
+							"collector.default"
+						]
+					},
+					"en": {
+						"type": "text",
+						"index": false,
+						"copy_to": [
+							"collector.en"
+						]
+					},
+					"fr": {
+						"type": "text",
+						"index": false,
+						"copy_to": [
+							"collector.fr"
+						]
+					},
+					"it": {
+						"type": "text",
+						"index": false,
+						"copy_to": [
+							"collector.it"
+						]
+					}
+				}
+			},
+	  	"neighbourhood": {
+				"properties": {
+					"de": {
+						"type": "text",
+						"index": false,
+						"copy_to": [
+							"collector.de"
+						]
+					},
+					"default": {
+						"type": "text",
+						"index": false,
+						"copy_to": [
+							"collector.default"
+						]
+					},
+					"en": {
+						"type": "text",
+						"index": false,
+						"copy_to": [
+							"collector.en"
+						]
+					},
+					"fr": {
+						"type": "text",
+						"index": false,
+						"copy_to": [
+							"collector.fr"
+						]
+					},
+					"it": {
+						"type": "text",
+						"index": false,
+						"copy_to": [
+							"collector.it"
+						]
+					}
+				}
+			}		
 		}
 	}
 }

src/main/java/de/komoot/photon/Constants.java

@@ -11,6 +11,8 @@ public class Constants {
     public static final String NAME = "name";
     public static final String COUNTRY = "country";
     public static final String CITY = "city";
+    public static final String SUBURB = "suburb";
+    public static final String NEIGHBOURHOOD = "neighbourhood";
     public static final String STREET = "street";
     public static final String STATE = "state";
     public static final String TYPE = "type";

src/main/java/de/komoot/photon/PhotonDoc.java

@@ -5,6 +5,7 @@
 import com.vividsolutions.jts.geom.Point;
 import lombok.Getter;
 import lombok.Setter;
+import lombok.extern.slf4j.Slf4j;
 
 import java.util.HashMap;
 import java.util.HashSet;
@@ -18,6 +19,7 @@
  */
 @Getter
 @Setter
+@Slf4j
 public class PhotonDoc {
     final private long placeId;
     final private String osmType;
@@ -36,6 +38,8 @@ public class PhotonDoc {
     final private int rankSearch;
 
     private Map<String, String> street;
+    private Map<String, String> neighbourhood;
+    private Map<String, String> suburb;
     private Map<String, String> city;
     private Set<Map<String, String>> context = new HashSet<Map<String, String>>();
     private Map<String, String> country;
@@ -88,6 +92,8 @@ public PhotonDoc(PhotonDoc other) {
         this.linkedPlaceId = other.linkedPlaceId;
         this.rankSearch = other.rankSearch;
         this.street = other.street;
+        this.neighbourhood = other.neighbourhood;
+        this.suburb = other.suburb;
         this.city = other.city;
         this.context = other.context;
         this.country = other.country;
@@ -127,10 +133,53 @@ public boolean isUsefulForIndex() {
     public void completeFromAddress() {
         String addressStreet = address != null ? address.get("street") : null;
         if (addressStreet != null) {
-            if (street == null) {
-                street = new HashMap<>();
+            if (this.street == null) {
+                this.street = new HashMap<>();
             }
-            street.put("name", addressStreet);
+            setOrReplace(addressStreet, this.street, "street");
+        }
+
+        String addressCity = address != null ? address.get("city") : null;
+        if (addressCity != null) {
+            if (this.city == null) {
+                this.city = new HashMap<>();
+            }
+            setOrReplace(addressCity, this.city, "city");
+        }
+
+        String addressSuburb = address != null ? address.get("suburb") : null;
+        if (addressSuburb != null) {
+            if (this.suburb == null) {
+                this.suburb = new HashMap<>();
+            }
+            setOrReplace(addressSuburb, this.suburb, "suburb");
+        }
+
+        String addressNeighbourhood = address != null ? address.get("neighbourhood") : null;
+        if (addressNeighbourhood != null) {
+            if (this.neighbourhood == null) {
+                this.neighbourhood = new HashMap<>();
+            }
+            setOrReplace(addressNeighbourhood, this.neighbourhood, "neighbourhood");
+        }
+
+        String addressPostCode = address != null ? address.get("postcode") : null;
+        if (addressPostCode != null && !addressPostCode.equals(this.postcode)) {
+            if (log.isDebugEnabled()) {
+                log.debug("Replacing postcode "+this.postcode+" with "+ addressPostCode+ " for osmId #" + osmId);
+            }
+            this.postcode = addressPostCode;
+        }
+    }
+
+    private void setOrReplace(String name, Map<String, String> namesMap, String field) {
+        String existingName = namesMap.get("name");
+        if (!name.equals(existingName)) {
+            if (log.isDebugEnabled()) {
+                log.debug("Replacing "+ field +" name '"+existingName+"' with '"+ name+ "' for osmId #" + osmId);
+            }
+            // TODO: do we need to add former name to context or better not, as it might have been wrong?
+            namesMap.put("name", name);
         }
     }
 }

src/main/java/de/komoot/photon/Utils.java

@@ -49,6 +49,8 @@ public static XContentBuilder convert(PhotonDoc doc, String[] languages) throws
         writeIntlNames(builder, doc.getCountry(), "country", languages);
         writeIntlNames(builder, doc.getState(), "state", languages);
         writeIntlNames(builder, doc.getStreet(), "street", languages);
+        writeIntlNames(builder, doc.getNeighbourhood(), "neighbourhood", languages);
+        writeIntlNames(builder, doc.getSuburb(), "suburb", languages);
         writeContext(builder, doc.getContext(), languages);
         writeExtent(builder, doc.getBbox());
 

src/main/java/de/komoot/photon/elasticsearch/Server.java

@@ -227,6 +227,8 @@ private JSONObject addLangsToMapping(JSONObject mappingsObject) {
                 propertiesObject = addToCollector("country", propertiesObject, copyToCollectorObject, lang);
                 propertiesObject = addToCollector("state", propertiesObject, copyToCollectorObject, lang);
                 propertiesObject = addToCollector("street", propertiesObject, copyToCollectorObject, lang);
+                propertiesObject = addToCollector("suburb", propertiesObject, copyToCollectorObject, lang);
+                propertiesObject = addToCollector("neighbourhood", propertiesObject, copyToCollectorObject, lang);
                 propertiesObject = addToCollector("name", propertiesObject, nameToCollectorObject, lang);
 
                 // add language specific collector to default for name

src/main/java/de/komoot/photon/nominatim/NominatimConnector.java

@@ -319,7 +319,7 @@ public void readEntireDatabase(String... countryCodes) {
         Thread importThread = new Thread(new ImportThread(documents));
         importThread.start();
         String andCountryCodeStr = "", whereCountryCodeStr = "";
-        if (countryCodes.length > 0) {
+        if (countryCodes.length > 0 && !"".equals(countryCodes[0])) {
             String countryCodeStr = "";
             for (String cc : countryCodes) {
                 if (cc.length() != 2)
@@ -334,6 +334,7 @@ public void readEntireDatabase(String... countryCodes) {
 
         template.query("SELECT " + selectColsPlaceX +
                 " FROM placex " +
+                //" WHERE linked_place_id IS NULL AND centroid IS NOT NULL AND OSM_ID=142486682 " + andCountryCodeStr +
                 " WHERE linked_place_id IS NULL AND centroid IS NOT NULL " + andCountryCodeStr +
                 " ORDER BY geometry_sector; ", new RowCallbackHandler() {
             @Override
@@ -471,6 +472,16 @@ private void completePlace(PhotonDoc doc) {
                 continue;
             }
 
+            if (address.isNeighbourhood() && doc.getNeighbourhood() == null) {
+                doc.setNeighbourhood(address.getName());
+                continue;
+            }
+
+            if (address.isSuburb() && doc.getSuburb() == null) {
+                doc.setSuburb(address.getName());
+                continue;
+            }
+
             if (address.isState() && doc.getState() == null) {
                 doc.setState(address.getName());
                 continue;

src/main/java/de/komoot/photon/nominatim/model/AddressRow.java

@@ -43,6 +43,21 @@ public boolean isStreet() {
         return 26 <= rankAddress && rankAddress < 28;
     }
 
+    public boolean isNeighbourhood() {
+        if ("place".equals(osmKey) && "neighbourhood".equals(osmValue)) {
+            return true;
+        }
+        // TODO admin?
+        return false;
+    }
+
+    public boolean isSuburb() {
+        if ("place".equals(osmKey) && "suburb".equals(osmValue)) {
+            return true;
+        }
+        // TODO admin?
+        return false;
+    }
     public boolean isCity() {
         if ("place".equals(osmKey) && Arrays.binarySearch(CITY_PLACE_VALUES, osmValue) >= 0) {
             return true;

src/main/java/de/komoot/photon/utils/ConvertToJson.java

@@ -19,7 +19,7 @@
 @Slf4j
 public class ConvertToJson implements OneWayConverter<SearchResponse, List<JSONObject>> {
     private final static String[] KEYS_LANG_UNSPEC = {Constants.OSM_ID, Constants.OSM_VALUE, Constants.OSM_KEY, Constants.POSTCODE, Constants.HOUSENUMBER, Constants.OSM_TYPE};
-    private final static String[] KEYS_LANG_SPEC = {Constants.NAME, Constants.COUNTRY, Constants.CITY, Constants.STREET, Constants.STATE};
+    private final static String[] KEYS_LANG_SPEC = {Constants.NAME, Constants.COUNTRY, Constants.CITY, Constants.SUBURB, Constants.NEIGHBOURHOOD, Constants.STREET, Constants.STATE};
     private final String lang;
 
     public ConvertToJson(String lang) {
@@ -28,7 +28,7 @@ public ConvertToJson(String lang) {
 
     @Override
     public List<JSONObject> convert(SearchResponse searchResponse) {
-        SearchHit[] hits = searchResponse.getHits().hits();
+        SearchHit[] hits = searchResponse.getHits().getHits();
         final List<JSONObject> list = Lists.newArrayListWithExpectedSize(hits.length);
         for (SearchHit hit : hits) {
             final Map<String, Object> source = hit.getSource();