From 00933ac0d3b8c41e91cb5981dcc92360d4215141 Mon Sep 17 00:00:00 2001 From: Sander Date: Mon, 8 Dec 2014 09:55:24 +0100 Subject: [PATCH 01/24] Added dynamic languages via command - Use -nominatim-import-languages command with comma separated languages to load those languages during import - Use -nominatim-import-languages command with comma seperated languages to use certain languages during run Example: -nominatim-import-languages en,de,nl --- .../java/de/komoot/photon/importer/App.java | 13 ++++---- .../photon/importer/CommandLineArgs.java | 3 ++ .../photon/importer/RequestHandler.java | 10 ++++--- .../java/de/komoot/photon/importer/Utils.java | 30 +++++++++---------- .../importer/elasticsearch/Importer.java | 6 ++-- .../importer/elasticsearch/Updater.java | 8 +++-- .../photon/importer/json/JsonDumper.java | 6 ++-- 7 files changed, 44 insertions(+), 32 deletions(-) diff --git a/src/main/java/de/komoot/photon/importer/App.java b/src/main/java/de/komoot/photon/importer/App.java index 6b59ddf3f..a52d9affc 100644 --- a/src/main/java/de/komoot/photon/importer/App.java +++ b/src/main/java/de/komoot/photon/importer/App.java @@ -41,7 +41,7 @@ public static void main(String[] rawArgs) { if(args.getJsonDump() != null) { try { final String filename = args.getJsonDump(); - final JsonDumper jsonDumper = new JsonDumper(filename); + final JsonDumper jsonDumper = new JsonDumper(filename, args.getNominatimImportLanguages()); NominatimConnector nominatimConnector = new NominatimConnector(args.getHost(), args.getPort(), args.getDatabase(), args.getUser(), args.getPassword()); nominatimConnector.setImporter(jsonDumper); nominatimConnector.readEntireDatabase(); @@ -65,16 +65,17 @@ public static void main(String[] rawArgs) { if(args.isNominatimImport()) { esServer.recreateIndex(); // dump previous data - Importer importer = new Importer(esNodeClient); + log.info("starting import from nominatim to photon with languages: " + args.getNominatimImportLanguages()); + Importer importer = new Importer(esNodeClient, args.getNominatimImportLanguages()); NominatimConnector nominatimConnector = new NominatimConnector(args.getHost(), args.getPort(), args.getDatabase(), args.getUser(), args.getPassword()); nominatimConnector.setImporter(importer); nominatimConnector.readEntireDatabase(); - log.info("imported data from nominatim to photon."); + log.info("imported data from nominatim to photon with languages: " + args.getNominatimImportLanguages()); return; } final NominatimUpdater nominatimUpdater = new NominatimUpdater(args.getHost(), args.getPort(), args.getDatabase(), args.getUser(), args.getPassword()); - de.komoot.photon.importer.Updater updater = new de.komoot.photon.importer.elasticsearch.Updater(esNodeClient); + de.komoot.photon.importer.Updater updater = new de.komoot.photon.importer.elasticsearch.Updater(esNodeClient, args.getNominatimImportLanguages()); nominatimUpdater.setUpdater(updater); startApi(args, esNodeClient, nominatimUpdater); @@ -99,7 +100,7 @@ public void run() { }); final Searcher searcher = new Searcher(esNodeClient); - get(new RequestHandler("api", searcher)); - get(new RequestHandler("api/", searcher)); + get(new RequestHandler("api", searcher, args.getNominatimImportLanguages())); + get(new RequestHandler("api/", searcher, args.getNominatimImportLanguages())); } } diff --git a/src/main/java/de/komoot/photon/importer/CommandLineArgs.java b/src/main/java/de/komoot/photon/importer/CommandLineArgs.java index 7ea80a9b4..b81c37c5f 100644 --- a/src/main/java/de/komoot/photon/importer/CommandLineArgs.java +++ b/src/main/java/de/komoot/photon/importer/CommandLineArgs.java @@ -16,6 +16,9 @@ public class CommandLineArgs { @Parameter(names = "-nominatim-import", description = "import nominatim database into photon (this will delete previous index)") private boolean nominatimImport = false; + + @Parameter(names = "-nominatim-import-languages", description = "languages nominatim importer should import, comma seperated (default: 'en,fr,de,it,nl')") + private String nominatimImportLanguages = "en,fr,de,it,nl"; @Parameter(names = "-json", description = "import nominatim database and dump it to a json like files in (useful for developing)") private String jsonDump = null; diff --git a/src/main/java/de/komoot/photon/importer/RequestHandler.java b/src/main/java/de/komoot/photon/importer/RequestHandler.java index 0fa9f5419..4a8c3f303 100644 --- a/src/main/java/de/komoot/photon/importer/RequestHandler.java +++ b/src/main/java/de/komoot/photon/importer/RequestHandler.java @@ -2,7 +2,8 @@ import com.google.common.base.Joiner; import de.komoot.photon.importer.elasticsearch.Searcher; -import org.elasticsearch.common.collect.ImmutableSet; +import java.util.Arrays; +import java.util.HashSet; import org.json.JSONArray; import org.json.JSONObject; import spark.Request; @@ -19,11 +20,12 @@ */ public class RequestHandler extends Route { private final Searcher searcher; - private static final Set supportedLanguages = ImmutableSet.of("de", "en", "fr", "it"); - - protected RequestHandler(String path, Searcher searcher) { + private final Set supportedLanguages; + + protected RequestHandler(String path, Searcher searcher, String languages) { super(path); this.searcher = searcher; + this.supportedLanguages = new HashSet(Arrays.asList(languages.split(","))); } @Override diff --git a/src/main/java/de/komoot/photon/importer/Utils.java b/src/main/java/de/komoot/photon/importer/Utils.java index e98fdc5cb..705a29747 100644 --- a/src/main/java/de/komoot/photon/importer/Utils.java +++ b/src/main/java/de/komoot/photon/importer/Utils.java @@ -22,9 +22,9 @@ public class Utils { static final Joiner commaJoiner = Joiner.on(", ").skipNulls(); - final static String[] languages = new String[]{"de", "en", "fr", "it"}; + //final static String[] languages = new String[]{"de", "en", "fr", "it"}; - public static XContentBuilder convert(PhotonDoc doc) throws IOException { + public static XContentBuilder convert(PhotonDoc doc, String[] languages) throws IOException { XContentBuilder builder = XContentFactory.jsonBuilder().startObject() .field(Tags.KEY_OSM_ID, doc.getOsmId()) .field(Tags.KEY_OSM_TYPE, doc.getOsmType()) @@ -47,11 +47,11 @@ public static XContentBuilder convert(PhotonDoc doc) throws IOException { builder.field("postcode", doc.getPostcode()); } - writeName(builder, doc.getName()); - writeIntlNames(builder, doc.getCity(), "city"); - writeIntlNames(builder, doc.getCountry(), "country"); - writeIntlNames(builder, doc.getStreet(), "street"); - writeContext(builder, doc.getContext()); + writeName(builder, doc.getName(), languages); + writeIntlNames(builder, doc.getCity(), "city", languages); + writeIntlNames(builder, doc.getCountry(), "country", languages); + writeIntlNames(builder, doc.getStreet(), "street", languages); + writeContext(builder, doc.getContext(), languages); writeExtent(builder, doc.getBbox()); return builder; @@ -74,8 +74,8 @@ private static void writeExtent(XContentBuilder builder, Envelope bbox) throws I builder.endObject(); } - private static void writeName(XContentBuilder builder, Map name) throws IOException { - Map fNames = filterNames(name); + private static void writeName(XContentBuilder builder, Map name, String[] languages) throws IOException { + Map fNames = filterNames(name, languages); if(name.get("alt_name") != null) fNames.put("alt", name.get("alt_name")); @@ -102,7 +102,7 @@ private static void write(XContentBuilder builder, Map fNames, S builder.endObject(); } - protected static void writeContext(XContentBuilder builder, Set> contexts) throws IOException { + protected static void writeContext(XContentBuilder builder, Set> contexts, String[] languages) throws IOException { final SetMultimap multimap = HashMultimap.create(); for(Map context : contexts) { @@ -129,16 +129,16 @@ protected static void writeContext(XContentBuilder builder, Set names, String name) throws IOException { - Map fNames = filterNames(names); + private static void writeIntlNames(XContentBuilder builder, Map names, String name, String[] languages) throws IOException { + Map fNames = filterNames(names, languages); write(builder, fNames, name); } - private static Map filterNames(Map names) { - return filterNames(names, new HashMap()); + private static Map filterNames(Map names, String[] languages) { + return filterNames(names, new HashMap(), languages); } - private static Map filterNames(Map names, HashMap filteredNames) { + private static Map filterNames(Map names, HashMap filteredNames, String[] languages) { if(names == null) return filteredNames; if(names.get("name") != null) { diff --git a/src/main/java/de/komoot/photon/importer/elasticsearch/Importer.java b/src/main/java/de/komoot/photon/importer/elasticsearch/Importer.java index 34f930f97..970f444be 100644 --- a/src/main/java/de/komoot/photon/importer/elasticsearch/Importer.java +++ b/src/main/java/de/komoot/photon/importer/elasticsearch/Importer.java @@ -23,17 +23,19 @@ public class Importer implements de.komoot.photon.importer.Importer { private final String indexType = "place"; private final Client esClient; private BulkRequestBuilder bulkRequest; + private String[] languages; - public Importer(Client esClient) { + public Importer(Client esClient, String languages) { this.esClient = esClient; this.bulkRequest = esClient.prepareBulk(); + this.languages = languages.split(","); } @Override public void add(PhotonDoc doc) { try { this.bulkRequest.add(this.esClient.prepareIndex(indexName, indexType). - setSource(Utils.convert(doc)).setId(String.valueOf(doc.getPlaceId()))); + setSource(Utils.convert(doc, languages)).setId(String.valueOf(doc.getPlaceId()))); } catch(IOException e) { log.error("could not ", e); } diff --git a/src/main/java/de/komoot/photon/importer/elasticsearch/Updater.java b/src/main/java/de/komoot/photon/importer/elasticsearch/Updater.java index dc0bc34e3..541bd7c32 100644 --- a/src/main/java/de/komoot/photon/importer/elasticsearch/Updater.java +++ b/src/main/java/de/komoot/photon/importer/elasticsearch/Updater.java @@ -18,10 +18,12 @@ public class Updater implements de.komoot.photon.importer.Updater { private Client esClient; private BulkRequestBuilder bulkRequest; + private String[] languages; - public Updater(Client esClient) { + public Updater(Client esClient, String languages) { this.esClient = esClient; this.bulkRequest = esClient.prepareBulk(); + this.languages = languages.split(","); } public void finish() { @@ -40,7 +42,7 @@ public void updateOrCreate(PhotonDoc updatedDoc) { public void create(PhotonDoc doc) { try { - this.bulkRequest.add(this.esClient.prepareIndex("photon", "place").setSource(Utils.convert(doc)).setId(String.valueOf(doc.getPlaceId()))); + this.bulkRequest.add(this.esClient.prepareIndex("photon", "place").setSource(Utils.convert(doc, this.languages)).setId(String.valueOf(doc.getPlaceId()))); } catch(IOException e) { log.error(String.format("creation of new doc [%s] failed", doc), e); } @@ -48,7 +50,7 @@ public void create(PhotonDoc doc) { public void update(PhotonDoc doc) { try { - this.bulkRequest.add(this.esClient.prepareUpdate("photon", "place", String.valueOf(doc.getPlaceId())).setDoc(Utils.convert(doc))); + this.bulkRequest.add(this.esClient.prepareUpdate("photon", "place", String.valueOf(doc.getPlaceId())).setDoc(Utils.convert(doc, this.languages))); } catch(IOException e) { log.error(String.format("update of new doc [%s] failed", doc), e); } diff --git a/src/main/java/de/komoot/photon/importer/json/JsonDumper.java b/src/main/java/de/komoot/photon/importer/json/JsonDumper.java index 349a134a0..1e351b9fc 100644 --- a/src/main/java/de/komoot/photon/importer/json/JsonDumper.java +++ b/src/main/java/de/komoot/photon/importer/json/JsonDumper.java @@ -17,16 +17,18 @@ @Slf4j public class JsonDumper implements Importer { private PrintWriter writer = null; + private String[] languages; - public JsonDumper(String filename) throws FileNotFoundException { + public JsonDumper(String filename, String languages) throws FileNotFoundException { this.writer = new PrintWriter(filename); + this.languages = languages.split(","); } @Override public void add(PhotonDoc doc) { try { writer.println("{\"index\": {}}"); - writer.println(Utils.convert(doc).string()); + writer.println(Utils.convert(doc,this.languages).string()); } catch(IOException e) { log.error("error writing json file", e); } From 68ae44f933f48277eb25c084e199a889d3937180 Mon Sep 17 00:00:00 2001 From: Sander Date: Mon, 8 Dec 2014 11:46:54 +0100 Subject: [PATCH 02/24] Fix invalid json --- es/query_location_bias.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/es/query_location_bias.json b/es/query_location_bias.json index 439fa95e6..d4f7244a9 100644 --- a/es/query_location_bias.json +++ b/es/query_location_bias.json @@ -67,8 +67,8 @@ "script_score": { "script": "dist = doc['coordinate'].distanceInKm(lat, lon); 0.5 + ( 1.5 / (1.0 + dist/40.0) )", "params": { - "lat": ${lat}, - "lon": ${lon} + "lat": "${lat}", + "lon": "${lon}" } } } From 0621d01b5d4a21430e8a7dfead9e45f291e7c0cc Mon Sep 17 00:00:00 2001 From: Sander Date: Mon, 8 Dec 2014 14:31:33 +0100 Subject: [PATCH 03/24] Seperated language commands for import/running Use -nominatim-import-languages for importing and -supported-languages for runtime languages --- src/main/java/de/komoot/photon/importer/App.java | 4 ++-- .../java/de/komoot/photon/importer/CommandLineArgs.java | 7 +++++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/main/java/de/komoot/photon/importer/App.java b/src/main/java/de/komoot/photon/importer/App.java index a52d9affc..d8ad77752 100644 --- a/src/main/java/de/komoot/photon/importer/App.java +++ b/src/main/java/de/komoot/photon/importer/App.java @@ -100,7 +100,7 @@ public void run() { }); final Searcher searcher = new Searcher(esNodeClient); - get(new RequestHandler("api", searcher, args.getNominatimImportLanguages())); - get(new RequestHandler("api/", searcher, args.getNominatimImportLanguages())); + get(new RequestHandler("api", searcher, args.getSupportedLanguages())); + get(new RequestHandler("api/", searcher, args.getSupportedLanguages())); } } diff --git a/src/main/java/de/komoot/photon/importer/CommandLineArgs.java b/src/main/java/de/komoot/photon/importer/CommandLineArgs.java index b81c37c5f..9223f6190 100644 --- a/src/main/java/de/komoot/photon/importer/CommandLineArgs.java +++ b/src/main/java/de/komoot/photon/importer/CommandLineArgs.java @@ -17,8 +17,11 @@ public class CommandLineArgs { @Parameter(names = "-nominatim-import", description = "import nominatim database into photon (this will delete previous index)") private boolean nominatimImport = false; - @Parameter(names = "-nominatim-import-languages", description = "languages nominatim importer should import, comma seperated (default: 'en,fr,de,it,nl')") - private String nominatimImportLanguages = "en,fr,de,it,nl"; + @Parameter(names = "-nominatim-import-languages", description = "languages nominatim importer should import, comma seperated (default: 'en,fr,de,it')") + private String nominatimImportLanguages = "en,fr,de,it"; + + @Parameter(names = "-supported-languages", description = "languages that are supported at run-time, comma seperated (default: 'en,fr,de,it')") + private String supportedLanguages = "en,fr,de,it"; @Parameter(names = "-json", description = "import nominatim database and dump it to a json like files in (useful for developing)") private String jsonDump = null; From 35bcfde7d99941885533a2d84feb75bd8eac3d39 Mon Sep 17 00:00:00 2001 From: Sander Date: Tue, 9 Dec 2014 16:19:53 +0100 Subject: [PATCH 04/24] Added dutch (nl) to mappings.json --- es/mappings.json | 58 +++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 57 insertions(+), 1 deletion(-) diff --git a/es/mappings.json b/es/mappings.json index 6d611e6fd..da6af2d3f 100644 --- a/es/mappings.json +++ b/es/mappings.json @@ -81,6 +81,13 @@ "copy_to": [ "collector.it" ] + }, + "nl": { + "type": "string", + "index": "no", + "copy_to": [ + "collector.nl" + ] } } }, @@ -121,6 +128,13 @@ "copy_to": [ "collector.it" ] + }, + "nl": { + "type": "string", + "index": "no", + "copy_to": [ + "collector.nl" + ] } } }, @@ -160,6 +174,13 @@ "copy_to": [ "collector.it" ] + }, + "nl": { + "type": "string", + "index": "no", + "copy_to": [ + "collector.nl" + ] } } }, @@ -243,7 +264,8 @@ "name.en", "name.de", "name.fr", - "name.it" + "name.it", + "name.nl" ] }, "en": { @@ -296,6 +318,23 @@ "copy_to": [ "collector.it" ] + }, + "nl": { + "type": "string", + "index": "no", + "fields": { + "ngrams": { + "type": "string", + "index_analyzer": "index_ngram" + }, + "raw": { + "type": "string", + "index_analyzer": "index_raw" + } + }, + "copy_to": [ + "collector.nl" + ] } } }, @@ -336,6 +375,13 @@ "copy_to": [ "collector.it" ] + }, + "nl": { + "index": "no", + "type": "string", + "copy_to": [ + "collector.nl" + ] } } }, @@ -391,6 +437,16 @@ "analyzer": "index_raw" } } + }, + "nl": { + "type": "string", + "analyzer": "index_ngram", + "fields": { + "raw": { + "type": "string", + "analyzer": "index_raw" + } + } } } } From e9fda4b516c614821a7bd2935c1b1454783ca40b Mon Sep 17 00:00:00 2001 From: Sander Date: Wed, 10 Dec 2014 10:27:29 +0100 Subject: [PATCH 05/24] Added stripNonDigits util --- src/main/java/de/komoot/photon/importer/Utils.java | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/main/java/de/komoot/photon/importer/Utils.java b/src/main/java/de/komoot/photon/importer/Utils.java index 705a29747..86b05276b 100644 --- a/src/main/java/de/komoot/photon/importer/Utils.java +++ b/src/main/java/de/komoot/photon/importer/Utils.java @@ -153,4 +153,18 @@ private static Map filterNames(Map names, HashMa return filteredNames; } + + // http://stackoverflow.com/a/4031040/1437096 + public static String stripNonDigits( + final CharSequence input /* inspired by seh's comment */){ + final StringBuilder sb = new StringBuilder( + input.length() /* also inspired by seh's comment */); + for(int i = 0; i < input.length(); i++){ + final char c = input.charAt(i); + if(c > 47 && c < 58){ + sb.append(c); + } + } + return sb.toString(); + } } From dacb24341446bb0b012a0d20a4a7808a5f612701 Mon Sep 17 00:00:00 2001 From: Sander Date: Wed, 10 Dec 2014 10:28:25 +0100 Subject: [PATCH 06/24] Fixed ImporterTest to use (static) languages --- .../de/komoot/photon/importer/elasticsearch/ImporterTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/de/komoot/photon/importer/elasticsearch/ImporterTest.java b/src/test/java/de/komoot/photon/importer/elasticsearch/ImporterTest.java index 9c5d3be26..45144d4f5 100644 --- a/src/test/java/de/komoot/photon/importer/elasticsearch/ImporterTest.java +++ b/src/test/java/de/komoot/photon/importer/elasticsearch/ImporterTest.java @@ -28,7 +28,7 @@ public void testAdd() { nameMap.put("name", "testing"); PhotonDoc doc = PhotonDoc.create(1, "way", 1, nameMap); - Importer instance = new Importer(getClient()); + Importer instance = new Importer(getClient(), "en,fr,de,it"); instance.add(doc); instance.finish(); From 84a37c2e3750d2f5aa839286075769f3a01674d5 Mon Sep 17 00:00:00 2001 From: Sander Date: Wed, 10 Dec 2014 10:39:28 +0100 Subject: [PATCH 07/24] Added postcode check for nl postcodes - Filter doubles out if there are highways with the same name & postcode numbers when language is 'nl' (i.e. the same street has 6532RA and 6532BE, which should only be returned once to prevent clutter in autocomplete) --- .../importer/elasticsearch/Searcher.java | 32 ++++++++++++++++--- 1 file changed, 28 insertions(+), 4 deletions(-) diff --git a/src/main/java/de/komoot/photon/importer/elasticsearch/Searcher.java b/src/main/java/de/komoot/photon/importer/elasticsearch/Searcher.java index f12041cb6..bfcffb7d1 100644 --- a/src/main/java/de/komoot/photon/importer/elasticsearch/Searcher.java +++ b/src/main/java/de/komoot/photon/importer/elasticsearch/Searcher.java @@ -6,6 +6,7 @@ import de.komoot.photon.importer.Tags; import de.komoot.photon.importer.osm.OSMTags; import lombok.extern.slf4j.Slf4j; +import de.komoot.photon.importer.Utils; import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringEscapeUtils; import org.apache.commons.lang3.text.StrSubstitutor; @@ -66,14 +67,14 @@ public List search(String query, String lang, Double lon, Double lat SearchResponse response = client.prepareSearch("photon").setSearchType(SearchType.QUERY_AND_FETCH).setQuery(query).setSize(limit).setTimeout(TimeValue.timeValueSeconds(7)).execute().actionGet(); List results = convert(response.getHits().getHits(), lang); - results = removeStreetDuplicates(results); + results = removeStreetDuplicates(results, lang); if(results.size() > limit) { results = results.subList(0, limit); } return results; } - private List removeStreetDuplicates(List results) { + private List removeStreetDuplicates(List results, String lang) { List filteredItems = Lists.newArrayListWithCapacity(results.size()); final HashSet keys = Sets.newHashSet(); for(JSONObject result : results) { @@ -84,12 +85,35 @@ private List removeStreetDuplicates(List results) { // street has a postcode and name String postcode = properties.getString(OSMTags.KEY_POSTCODE); String name = properties.getString(OSMTags.KEY_NAME); - String key = postcode + ":" + name; + String key = postcode + ":" + name; if(keys.contains(key)) { // a street with this name + postcode is already part of the result list continue; - } + } else if (lang.equals("nl")) { + // check for dutch postcodes (i.e. 6532RA). If a street has the same name and the same 4 numbers in the postcode, + // we can assume it is part of the same street, so only use the first + try { + String letterlessPostcode = Utils.stripNonDigits(postcode); + int postcodeNumbers = Integer.parseInt(letterlessPostcode); + boolean foundMatch = false; + + for (String keyString : keys) { + String letterlessKey = Utils.stripNonDigits(keyString); + // also check if name equals, + // which is a safety check for streets that partially match and have the same postcode numbers + String keyName = keyString.split(":")[1]; + if (postcodeNumbers == Integer.parseInt(letterlessKey) && keyName.equals(name)) { + foundMatch = true; + break; + } + } + + if (foundMatch) { + continue; + } + } catch (NumberFormatException e) {} + } keys.add(key); } } From a7ac5eb4c10992406f07c93b89849cd300856833 Mon Sep 17 00:00:00 2001 From: Sander Date: Wed, 10 Dec 2014 10:42:58 +0100 Subject: [PATCH 08/24] Fixed tests to use new lang scheme --- .../de/komoot/photon/importer/elasticsearch/ImporterTest.java | 2 +- .../importer/elasticsearch/RemoveStreetDuplicatesTest.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/java/de/komoot/photon/importer/elasticsearch/ImporterTest.java b/src/test/java/de/komoot/photon/importer/elasticsearch/ImporterTest.java index 45144d4f5..80e8f2b69 100644 --- a/src/test/java/de/komoot/photon/importer/elasticsearch/ImporterTest.java +++ b/src/test/java/de/komoot/photon/importer/elasticsearch/ImporterTest.java @@ -28,7 +28,7 @@ public void testAdd() { nameMap.put("name", "testing"); PhotonDoc doc = PhotonDoc.create(1, "way", 1, nameMap); - Importer instance = new Importer(getClient(), "en,fr,de,it"); + Importer instance = new Importer(getClient(), "en"); // hardcoded lang instance.add(doc); instance.finish(); diff --git a/src/test/java/de/komoot/photon/importer/elasticsearch/RemoveStreetDuplicatesTest.java b/src/test/java/de/komoot/photon/importer/elasticsearch/RemoveStreetDuplicatesTest.java index 97cb05479..9cda33230 100644 --- a/src/test/java/de/komoot/photon/importer/elasticsearch/RemoveStreetDuplicatesTest.java +++ b/src/test/java/de/komoot/photon/importer/elasticsearch/RemoveStreetDuplicatesTest.java @@ -27,7 +27,7 @@ public void setUp() { PhotonDoc street2 = this.createStreetDoc(2, "Walserstraße", "6993"); PhotonDoc street3 = this.createStreetDoc(3, "Walserstraße", "6991"); - Importer instance = new Importer(getClient()); + Importer instance = new Importer(getClient(), "en"); // hardcoded lang instance.add(street1); instance.add(street2); instance.add(street3); From 0636de815e33075061940c3538c0f3c08db5fa2b Mon Sep 17 00:00:00 2001 From: Sander Date: Wed, 10 Dec 2014 10:43:21 +0100 Subject: [PATCH 09/24] Added Netbeans xml to .gitignore --- .gitignore | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitignore b/.gitignore index a8a9bd80f..bb76caa7b 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,7 @@ dependency-reduced-pom.xml *.~* __pycache__ + +nb-configuration.xml + +nbactions.xml From b4c5d424597554f2ef371068d3bd592d026bb4c6 Mon Sep 17 00:00:00 2001 From: Sander Date: Mon, 15 Dec 2014 13:50:28 +0100 Subject: [PATCH 10/24] Reverted string lat/lon back to doubles --- es/query_location_bias.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/es/query_location_bias.json b/es/query_location_bias.json index d4f7244a9..439fa95e6 100644 --- a/es/query_location_bias.json +++ b/es/query_location_bias.json @@ -67,8 +67,8 @@ "script_score": { "script": "dist = doc['coordinate'].distanceInKm(lat, lon); 0.5 + ( 1.5 / (1.0 + dist/40.0) )", "params": { - "lat": "${lat}", - "lon": "${lon}" + "lat": ${lat}, + "lon": ${lon} } } } From b7e6ff34f4eb1502e713e580326c9a20637b9128 Mon Sep 17 00:00:00 2001 From: Sander Date: Mon, 15 Dec 2014 16:16:44 +0100 Subject: [PATCH 11/24] Deleted language specific obj from mappings.json --- es/mappings.json | 283 +---------------------------------------------- 1 file changed, 1 insertion(+), 282 deletions(-) diff --git a/es/mappings.json b/es/mappings.json index da6af2d3f..4a8f6f522 100644 --- a/es/mappings.json +++ b/es/mappings.json @@ -47,140 +47,34 @@ }, "city": { "properties": { - "de": { - "type": "string", - "index": "no", - "copy_to": [ - "collector.de" - ] - }, "default": { "type": "string", "index": "no", "copy_to": [ "collector.default" ] - }, - "en": { - "type": "string", - "index": "no", - "copy_to": [ - "collector.en" - ] - }, - "fr": { - "type": "string", - "index": "no", - "copy_to": [ - "collector.fr" - ] - }, - "it": { - "type": "string", - "index": "no", - "copy_to": [ - "collector.it" - ] - }, - "nl": { - "type": "string", - "index": "no", - "copy_to": [ - "collector.nl" - ] } } }, "context": { "properties": { - "de": { - "type": "string", - "index": "no", - "copy_to": [ - "collector.default", - "collector.de" - ] - }, "default": { "type": "string", "index": "no", "copy_to": [ "collector.default" ] - }, - "en": { - "type": "string", - "index": "no", - "copy_to": [ - "collector.en" - ] - }, - "fr": { - "type": "string", - "index": "no", - "copy_to": [ - "collector.fr" - ] - }, - "it": { - "type": "string", - "index": "no", - "copy_to": [ - "collector.it" - ] - }, - "nl": { - "type": "string", - "index": "no", - "copy_to": [ - "collector.nl" - ] } } }, "country": { "properties": { - "de": { - "type": "string", - "index": "no", - "copy_to": [ - "collector.de" - ] - }, "default": { "type": "string", "index": "no", "copy_to": [ "collector.default" ] - }, - "en": { - "type": "string", - "index": "no", - "copy_to": [ - "collector.en" - ] - }, - "fr": { - "type": "string", - "index": "no", - "copy_to": [ - "collector.fr" - ] - }, - "it": { - "type": "string", - "index": "no", - "copy_to": [ - "collector.it" - ] - }, - "nl": { - "type": "string", - "index": "no", - "copy_to": [ - "collector.nl" - ] } } }, @@ -239,101 +133,11 @@ "collector.default" ] }, - "de": { - "type": "string", - "index": "no", - "fields": { - "ngrams": { - "type": "string", - "index_analyzer": "index_ngram" - }, - "raw": { - "type": "string", - "index_analyzer": "index_raw" - } - }, - "copy_to": [ - "collector.de" - ] - }, "default": { "type": "string", "index": "no", "copy_to": [ - "collector.default", - "name.en", - "name.de", - "name.fr", - "name.it", - "name.nl" - ] - }, - "en": { - "type": "string", - "index": "no", - "fields": { - "ngrams": { - "type": "string", - "index_analyzer": "index_ngram" - }, - "raw": { - "type": "string", - "index_analyzer": "index_raw" - } - }, - "copy_to": [ - "collector.en" - ] - }, - "fr": { - "type": "string", - "index": "no", - "fields": { - "ngrams": { - "type": "string", - "index_analyzer": "index_ngram" - }, - "raw": { - "type": "string", - "index_analyzer": "index_raw" - } - }, - "copy_to": [ - "collector.fr" - ] - }, - "it": { - "type": "string", - "index": "no", - "fields": { - "ngrams": { - "type": "string", - "index_analyzer": "index_ngram" - }, - "raw": { - "type": "string", - "index_analyzer": "index_raw" - } - }, - "copy_to": [ - "collector.it" - ] - }, - "nl": { - "type": "string", - "index": "no", - "fields": { - "ngrams": { - "type": "string", - "index_analyzer": "index_ngram" - }, - "raw": { - "type": "string", - "index_analyzer": "index_raw" - } - }, - "copy_to": [ - "collector.nl" + "collector.default" ] } } @@ -347,41 +151,6 @@ "copy_to": [ "collector.default" ] - }, - "de": { - "index": "no", - "type": "string", - "copy_to": [ - "collector.de" - ] - }, - "en": { - "index": "no", - "type": "string", - "copy_to": [ - "collector.en" - ] - }, - "fr": { - "index": "no", - "type": "string", - "copy_to": [ - "collector.fr" - ] - }, - "it": { - "index": "no", - "type": "string", - "copy_to": [ - "collector.it" - ] - }, - "nl": { - "index": "no", - "type": "string", - "copy_to": [ - "collector.nl" - ] } } }, @@ -397,56 +166,6 @@ "analyzer": "index_raw" } } - }, - "de": { - "type": "string", - "analyzer": "index_ngram", - "fields": { - "raw": { - "type": "string", - "analyzer": "index_raw" - } - } - }, - "en": { - "type": "string", - "analyzer": "index_ngram", - "fields": { - "raw": { - "type": "string", - "analyzer": "index_raw" - } - } - }, - "fr": { - "type": "string", - "analyzer": "index_ngram", - "fields": { - "raw": { - "type": "string", - "analyzer": "index_raw" - } - } - }, - "it": { - "type": "string", - "analyzer": "index_ngram", - "fields": { - "raw": { - "type": "string", - "analyzer": "index_raw" - } - } - }, - "nl": { - "type": "string", - "analyzer": "index_ngram", - "fields": { - "raw": { - "type": "string", - "analyzer": "index_raw" - } - } } } } From fee73da5236724db40377793d6bdc0885f078d9f Mon Sep 17 00:00:00 2001 From: Sander Date: Mon, 15 Dec 2014 16:20:44 +0100 Subject: [PATCH 12/24] Use mappings template & add lang specific strings --- .../java/de/komoot/photon/importer/App.java | 2 +- .../photon/importer/elasticsearch/Server.java | 102 +++++++++++++++++- 2 files changed, 101 insertions(+), 3 deletions(-) diff --git a/src/main/java/de/komoot/photon/importer/App.java b/src/main/java/de/komoot/photon/importer/App.java index d8ad77752..8914c7676 100644 --- a/src/main/java/de/komoot/photon/importer/App.java +++ b/src/main/java/de/komoot/photon/importer/App.java @@ -52,7 +52,7 @@ public static void main(String[] rawArgs) { } } - final Server esServer = new Server(args.getCluster(), args.getDataDirectory()); + final Server esServer = new Server(args.getCluster(), args.getDataDirectory(), args.getNominatimImportLanguages()); esServer.start(); Client esNodeClient = esServer.getClient(); diff --git a/src/main/java/de/komoot/photon/importer/elasticsearch/Server.java b/src/main/java/de/komoot/photon/importer/elasticsearch/Server.java index ef4174310..a935fa762 100644 --- a/src/main/java/de/komoot/photon/importer/elasticsearch/Server.java +++ b/src/main/java/de/komoot/photon/importer/elasticsearch/Server.java @@ -21,6 +21,8 @@ import java.net.URL; import static org.elasticsearch.node.NodeBuilder.nodeBuilder; +import org.json.JSONArray; +import org.json.JSONObject; /** * Helper class to start/stop elasticserach node and get elasticsearch clients @@ -37,8 +39,9 @@ public class Server { private File updateDirectory; private File tempDirectory; private File importDirectory; + private String[] langs; - public Server(String clusterName, String mainDirectory) { + public Server(String clusterName, String mainDirectory, String langs) { try { setupDirectories(new URL("file://" + mainDirectory)); } catch(MalformedURLException e) { @@ -52,6 +55,7 @@ public Server(String clusterName, String mainDirectory) { } } this.clusterName = clusterName; + this.langs = langs.split(","); } public Server start() { @@ -141,8 +145,15 @@ public void recreateIndex() { final InputStream index_settings = Thread.currentThread().getContextClassLoader().getResourceAsStream("index_settings.json"); try { + // get mappings as JSONObject + String mappingsString = IOUtils.toString(mappings); + JSONObject mappingsJSON = new JSONObject(mappingsString); + + // add all langs to the mapping + mappingsJSON = addLangsToMapping(mappingsJSON); + client.admin().indices().prepareCreate("photon").setSettings(IOUtils.toString(index_settings)).execute().actionGet(); - client.admin().indices().preparePutMapping("photon").setType("place").setSource(IOUtils.toString(mappings)).execute().actionGet(); + client.admin().indices().preparePutMapping("photon").setType("place").setSource(mappingsJSON.toString()).execute().actionGet(); } catch(IOException e) { log.error("cannot setup index, elastic search config files not readable", e); } @@ -156,4 +167,91 @@ public DeleteIndexResponse deleteIndex() { return null; } } + + private JSONObject addLangsToMapping(JSONObject mappingsObject) { + // define collector json strings + String copyToCollectorString = "{\"type\":\"string\",\"index\":\"no\",\"copy_to\":[\"collector.{lang}\"]}"; + String nameToCollectorString = "{\"type\":\"string\",\"index\":\"no\",\"fields\":{\"ngrams\":{\"type\":\"string\",\"index_analyzer\":\"index_ngram\"},\"raw\":{\"type\":\"string\",\"index_analyzer\":\"index_raw\"}},\"copy_to\":[\"collector.{lang}\"]}"; + String collectorString = "{\"type\":\"string\",\"index\":\"no\",\"fields\":{\"ngrams\":{\"type\":\"string\",\"index_analyzer\":\"index_ngram\"},\"raw\":{\"type\":\"string\",\"index_analyzer\":\"index_raw\"}},\"copy_to\":[\"collector.{lang}\"]}}},\"street\":{\"type\":\"object\",\"properties\":{\"default\":{\"index\":\"no\",\"type\":\"string\",\"copy_to\":[\"collector.default\"]}"; + + if(mappingsObject != null) { + JSONObject placeObject = mappingsObject.optJSONObject("place"); + JSONObject propertiesObject = placeObject == null ? null : placeObject.optJSONObject("properties"); + + if(propertiesObject != null) { + for(String lang : langs) { + // create lang-specific json objects + JSONObject copyToCollectorObject = new JSONObject(copyToCollectorString.replace("{lang}", lang)); + JSONObject nameToCollectorObject = new JSONObject(nameToCollectorString.replace("{lang}", lang)); + JSONObject collectorObject = new JSONObject(collectorString.replace("{lang}", lang)); + + // add language specific city to the collector + JSONObject city = propertiesObject.optJSONObject("city"); + JSONObject cityProperties = city == null ? null : city.optJSONObject("properties"); + if(cityProperties != null) { + cityProperties.put(lang, copyToCollectorObject); + city.put("properties", cityProperties); + propertiesObject.put("city", city); + } + + // add language specific context to the collector + JSONObject context = propertiesObject.optJSONObject("context"); + JSONObject contextProperties = context == null ? null : context.optJSONObject("properties"); + if(contextProperties != null) { + contextProperties.put(lang, copyToCollectorObject); + context.put("properties", contextProperties); + propertiesObject.put("context", context); + } + + // add language specific country to the collector + JSONObject country = propertiesObject.optJSONObject("country"); + JSONObject countryProperties = country == null ? null : country.optJSONObject("properties"); + if(countryProperties != null) { + countryProperties.put(lang, copyToCollectorObject); + country.put("properties", countryProperties); + propertiesObject.put("country", country); + } + + // add language specific street to the collector + JSONObject street = propertiesObject.optJSONObject("street"); + JSONObject streetProperties = street == null ? null : street.optJSONObject("properties"); + if(streetProperties != null) { + streetProperties.put(lang, copyToCollectorObject); + street.put("properties", streetProperties); + propertiesObject.put("street", street); + } + + // add language specific name to the collector + JSONObject name = propertiesObject.optJSONObject("name"); + JSONObject nameProperties = name == null ? null : name.optJSONObject("properties"); + if(nameProperties != null) { + nameProperties.put(lang, nameToCollectorObject); + JSONObject defaultObject = nameProperties.optJSONObject("default"); + + // add language specific collector to default + JSONArray copyToArray = defaultObject == null ? null : defaultObject.optJSONArray("copy_to"); + copyToArray.put("name." + lang); + + defaultObject.put("copy_to", copyToArray); + nameProperties.put("default", defaultObject); + name.put("properties", nameProperties); + propertiesObject.put("name", name); + } + + // add language specific collector + JSONObject collector = propertiesObject.optJSONObject("collector"); + JSONObject collectorProperties = collector == null ? null : collector.optJSONObject("properties"); + if(collectorProperties != null) { + collectorProperties.put(lang, collectorObject); + collector.put("properties", collectorProperties); + propertiesObject.put("collector", collector); + } + } + placeObject.put("properties", propertiesObject); + return mappingsObject.put("place", placeObject); + } + } + log.error("cannot add langs to mapping.json, please double-check the mappings.json or the language values supplied"); + return null; + } } From 41ee0d13c5f69c2068710beb682eb8c44f6ec6a5 Mon Sep 17 00:00:00 2001 From: Sander Date: Mon, 15 Dec 2014 16:24:35 +0100 Subject: [PATCH 13/24] Removed spaces --- src/main/java/de/komoot/photon/importer/RequestHandler.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/de/komoot/photon/importer/RequestHandler.java b/src/main/java/de/komoot/photon/importer/RequestHandler.java index 4a8c3f303..2cd3ba209 100644 --- a/src/main/java/de/komoot/photon/importer/RequestHandler.java +++ b/src/main/java/de/komoot/photon/importer/RequestHandler.java @@ -22,7 +22,7 @@ public class RequestHandler extends Route { private final Searcher searcher; private final Set supportedLanguages; - protected RequestHandler(String path, Searcher searcher, String languages) { + protected RequestHandler(String path, Searcher searcher, String languages) { super(path); this.searcher = searcher; this.supportedLanguages = new HashSet(Arrays.asList(languages.split(","))); From d81292afdb8d5f770b5690eaceefac5a9bb85520 Mon Sep 17 00:00:00 2001 From: Sander Date: Mon, 15 Dec 2014 16:28:39 +0100 Subject: [PATCH 14/24] Unified commandline param to 'using-languages' --- .../java/de/komoot/photon/importer/App.java | 23 +++++++++++-------- .../photon/importer/CommandLineArgs.java | 7 ++---- 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/src/main/java/de/komoot/photon/importer/App.java b/src/main/java/de/komoot/photon/importer/App.java index 8914c7676..8f3e5f5a3 100644 --- a/src/main/java/de/komoot/photon/importer/App.java +++ b/src/main/java/de/komoot/photon/importer/App.java @@ -41,7 +41,7 @@ public static void main(String[] rawArgs) { if(args.getJsonDump() != null) { try { final String filename = args.getJsonDump(); - final JsonDumper jsonDumper = new JsonDumper(filename, args.getNominatimImportLanguages()); + final JsonDumper jsonDumper = new JsonDumper(filename, args.getUsingLanguages()); NominatimConnector nominatimConnector = new NominatimConnector(args.getHost(), args.getPort(), args.getDatabase(), args.getUser(), args.getPassword()); nominatimConnector.setImporter(jsonDumper); nominatimConnector.readEntireDatabase(); @@ -52,7 +52,7 @@ public static void main(String[] rawArgs) { } } - final Server esServer = new Server(args.getCluster(), args.getDataDirectory(), args.getNominatimImportLanguages()); + final Server esServer = new Server(args.getCluster(), args.getDataDirectory(), args.getUsingLanguages()); esServer.start(); Client esNodeClient = esServer.getClient(); @@ -65,17 +65,22 @@ public static void main(String[] rawArgs) { if(args.isNominatimImport()) { esServer.recreateIndex(); // dump previous data - log.info("starting import from nominatim to photon with languages: " + args.getNominatimImportLanguages()); - Importer importer = new Importer(esNodeClient, args.getNominatimImportLanguages()); + log.info("starting import from nominatim to photon with languages: " + args.getUsingLanguages()); + Importer importer = new Importer(esNodeClient, args.getUsingLanguages()); NominatimConnector nominatimConnector = new NominatimConnector(args.getHost(), args.getPort(), args.getDatabase(), args.getUser(), args.getPassword()); nominatimConnector.setImporter(importer); - nominatimConnector.readEntireDatabase(); - log.info("imported data from nominatim to photon with languages: " + args.getNominatimImportLanguages()); + try { + nominatimConnector.readEntireDatabase(); + } catch (Exception e) { + log.info("ERROR IMPORTING FROM NOMINATIM: "+e.getMessage()); + } + + log.info("imported data from nominatim to photon with languages: " + args.getUsingLanguages()); return; } final NominatimUpdater nominatimUpdater = new NominatimUpdater(args.getHost(), args.getPort(), args.getDatabase(), args.getUser(), args.getPassword()); - de.komoot.photon.importer.Updater updater = new de.komoot.photon.importer.elasticsearch.Updater(esNodeClient, args.getNominatimImportLanguages()); + de.komoot.photon.importer.Updater updater = new de.komoot.photon.importer.elasticsearch.Updater(esNodeClient, args.getUsingLanguages()); nominatimUpdater.setUpdater(updater); startApi(args, esNodeClient, nominatimUpdater); @@ -100,7 +105,7 @@ public void run() { }); final Searcher searcher = new Searcher(esNodeClient); - get(new RequestHandler("api", searcher, args.getSupportedLanguages())); - get(new RequestHandler("api/", searcher, args.getSupportedLanguages())); + get(new RequestHandler("api", searcher, args.getUsingLanguages())); + get(new RequestHandler("api/", searcher, args.getUsingLanguages())); } } diff --git a/src/main/java/de/komoot/photon/importer/CommandLineArgs.java b/src/main/java/de/komoot/photon/importer/CommandLineArgs.java index 9223f6190..625a29c49 100644 --- a/src/main/java/de/komoot/photon/importer/CommandLineArgs.java +++ b/src/main/java/de/komoot/photon/importer/CommandLineArgs.java @@ -17,11 +17,8 @@ public class CommandLineArgs { @Parameter(names = "-nominatim-import", description = "import nominatim database into photon (this will delete previous index)") private boolean nominatimImport = false; - @Parameter(names = "-nominatim-import-languages", description = "languages nominatim importer should import, comma seperated (default: 'en,fr,de,it')") - private String nominatimImportLanguages = "en,fr,de,it"; - - @Parameter(names = "-supported-languages", description = "languages that are supported at run-time, comma seperated (default: 'en,fr,de,it')") - private String supportedLanguages = "en,fr,de,it"; + @Parameter(names = "-using-languages", description = "languages nominatim importer should import and use at run-time, comma seperated (default: 'en,fr,de,it')") + private String usingLanguages = "en,fr,de,it"; @Parameter(names = "-json", description = "import nominatim database and dump it to a json like files in (useful for developing)") private String jsonDump = null; From 756aeb215a1630534c46b3d375e298aeb6933852 Mon Sep 17 00:00:00 2001 From: Sander Date: Mon, 15 Dec 2014 16:32:27 +0100 Subject: [PATCH 15/24] Modified test with static lang --- src/test/java/de/komoot/photon/ESBaseTester.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/de/komoot/photon/ESBaseTester.java b/src/test/java/de/komoot/photon/ESBaseTester.java index 4c8b685ec..4a427fece 100644 --- a/src/test/java/de/komoot/photon/ESBaseTester.java +++ b/src/test/java/de/komoot/photon/ESBaseTester.java @@ -30,7 +30,7 @@ public void setUpES() { if (server != null) return; - server = new Server("photon", new File("./target/es_photon").getAbsolutePath()).start(true); + server = new Server("photon", new File("./target/es_photon").getAbsolutePath(), "en").start(true); server.recreateIndex(); } From f3332f75d7f6abd7c1f0c868e0a404b8afa4ce22 Mon Sep 17 00:00:00 2001 From: Sander Date: Mon, 15 Dec 2014 16:33:21 +0100 Subject: [PATCH 16/24] Reverted NL postcode check - Should be submitted as separate PR --- .../importer/elasticsearch/Searcher.java | 25 +------------------ 1 file changed, 1 insertion(+), 24 deletions(-) diff --git a/src/main/java/de/komoot/photon/importer/elasticsearch/Searcher.java b/src/main/java/de/komoot/photon/importer/elasticsearch/Searcher.java index bfcffb7d1..054e6ebf6 100644 --- a/src/main/java/de/komoot/photon/importer/elasticsearch/Searcher.java +++ b/src/main/java/de/komoot/photon/importer/elasticsearch/Searcher.java @@ -90,30 +90,7 @@ private List removeStreetDuplicates(List results, String if(keys.contains(key)) { // a street with this name + postcode is already part of the result list continue; - } else if (lang.equals("nl")) { - // check for dutch postcodes (i.e. 6532RA). If a street has the same name and the same 4 numbers in the postcode, - // we can assume it is part of the same street, so only use the first - try { - String letterlessPostcode = Utils.stripNonDigits(postcode); - int postcodeNumbers = Integer.parseInt(letterlessPostcode); - boolean foundMatch = false; - - for (String keyString : keys) { - String letterlessKey = Utils.stripNonDigits(keyString); - // also check if name equals, - // which is a safety check for streets that partially match and have the same postcode numbers - String keyName = keyString.split(":")[1]; - if (postcodeNumbers == Integer.parseInt(letterlessKey) && keyName.equals(name)) { - foundMatch = true; - break; - } - } - - if (foundMatch) { - continue; - } - } catch (NumberFormatException e) {} - } + } keys.add(key); } } From 0b7f608d989aa9612af97360d6218437d83c8d07 Mon Sep 17 00:00:00 2001 From: Sander Date: Tue, 16 Dec 2014 09:03:13 +0100 Subject: [PATCH 17/24] Removed commented code --- src/main/java/de/komoot/photon/importer/Utils.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/main/java/de/komoot/photon/importer/Utils.java b/src/main/java/de/komoot/photon/importer/Utils.java index 86b05276b..da67731a5 100644 --- a/src/main/java/de/komoot/photon/importer/Utils.java +++ b/src/main/java/de/komoot/photon/importer/Utils.java @@ -22,8 +22,6 @@ public class Utils { static final Joiner commaJoiner = Joiner.on(", ").skipNulls(); - //final static String[] languages = new String[]{"de", "en", "fr", "it"}; - public static XContentBuilder convert(PhotonDoc doc, String[] languages) throws IOException { XContentBuilder builder = XContentFactory.jsonBuilder().startObject() .field(Tags.KEY_OSM_ID, doc.getOsmId()) From e4c6a6ef78e2adb2fa044409a562a7478d7aaf94 Mon Sep 17 00:00:00 2001 From: Sander Date: Tue, 16 Dec 2014 09:03:38 +0100 Subject: [PATCH 18/24] Renamed commandline arg to just '-languages' --- src/main/java/de/komoot/photon/importer/App.java | 16 ++++++++-------- .../komoot/photon/importer/CommandLineArgs.java | 4 ++-- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/main/java/de/komoot/photon/importer/App.java b/src/main/java/de/komoot/photon/importer/App.java index 8f3e5f5a3..2b256c631 100644 --- a/src/main/java/de/komoot/photon/importer/App.java +++ b/src/main/java/de/komoot/photon/importer/App.java @@ -41,7 +41,7 @@ public static void main(String[] rawArgs) { if(args.getJsonDump() != null) { try { final String filename = args.getJsonDump(); - final JsonDumper jsonDumper = new JsonDumper(filename, args.getUsingLanguages()); + final JsonDumper jsonDumper = new JsonDumper(filename, args.getLanguages()); NominatimConnector nominatimConnector = new NominatimConnector(args.getHost(), args.getPort(), args.getDatabase(), args.getUser(), args.getPassword()); nominatimConnector.setImporter(jsonDumper); nominatimConnector.readEntireDatabase(); @@ -52,7 +52,7 @@ public static void main(String[] rawArgs) { } } - final Server esServer = new Server(args.getCluster(), args.getDataDirectory(), args.getUsingLanguages()); + final Server esServer = new Server(args.getCluster(), args.getDataDirectory(), args.getLanguages()); esServer.start(); Client esNodeClient = esServer.getClient(); @@ -65,8 +65,8 @@ public static void main(String[] rawArgs) { if(args.isNominatimImport()) { esServer.recreateIndex(); // dump previous data - log.info("starting import from nominatim to photon with languages: " + args.getUsingLanguages()); - Importer importer = new Importer(esNodeClient, args.getUsingLanguages()); + log.info("starting import from nominatim to photon with languages: " + args.getLanguages()); + Importer importer = new Importer(esNodeClient, args.getLanguages()); NominatimConnector nominatimConnector = new NominatimConnector(args.getHost(), args.getPort(), args.getDatabase(), args.getUser(), args.getPassword()); nominatimConnector.setImporter(importer); try { @@ -75,12 +75,12 @@ public static void main(String[] rawArgs) { log.info("ERROR IMPORTING FROM NOMINATIM: "+e.getMessage()); } - log.info("imported data from nominatim to photon with languages: " + args.getUsingLanguages()); + log.info("imported data from nominatim to photon with languages: " + args.getLanguages()); return; } final NominatimUpdater nominatimUpdater = new NominatimUpdater(args.getHost(), args.getPort(), args.getDatabase(), args.getUser(), args.getPassword()); - de.komoot.photon.importer.Updater updater = new de.komoot.photon.importer.elasticsearch.Updater(esNodeClient, args.getUsingLanguages()); + de.komoot.photon.importer.Updater updater = new de.komoot.photon.importer.elasticsearch.Updater(esNodeClient, args.getLanguages()); nominatimUpdater.setUpdater(updater); startApi(args, esNodeClient, nominatimUpdater); @@ -105,7 +105,7 @@ public void run() { }); final Searcher searcher = new Searcher(esNodeClient); - get(new RequestHandler("api", searcher, args.getUsingLanguages())); - get(new RequestHandler("api/", searcher, args.getUsingLanguages())); + get(new RequestHandler("api", searcher, args.getLanguages())); + get(new RequestHandler("api/", searcher, args.getLanguages())); } } diff --git a/src/main/java/de/komoot/photon/importer/CommandLineArgs.java b/src/main/java/de/komoot/photon/importer/CommandLineArgs.java index 625a29c49..61fd8bbc3 100644 --- a/src/main/java/de/komoot/photon/importer/CommandLineArgs.java +++ b/src/main/java/de/komoot/photon/importer/CommandLineArgs.java @@ -17,8 +17,8 @@ public class CommandLineArgs { @Parameter(names = "-nominatim-import", description = "import nominatim database into photon (this will delete previous index)") private boolean nominatimImport = false; - @Parameter(names = "-using-languages", description = "languages nominatim importer should import and use at run-time, comma seperated (default: 'en,fr,de,it')") - private String usingLanguages = "en,fr,de,it"; + @Parameter(names = "-languages", description = "languages nominatim importer should import and use at run-time, comma seperated (default: 'en,fr,de,it')") + private String languages = "en,fr,de,it"; @Parameter(names = "-json", description = "import nominatim database and dump it to a json like files in (useful for developing)") private String jsonDump = null; From 0c82ecbd123f1f5993acafbf6264f589bceef23d Mon Sep 17 00:00:00 2001 From: Sander Date: Tue, 16 Dec 2014 11:40:40 +0100 Subject: [PATCH 19/24] Refactored mappings.json code --- .../importer/elasticsearch/Importer.java | 2 +- .../photon/importer/elasticsearch/Server.java | 65 ++++++------------- 2 files changed, 20 insertions(+), 47 deletions(-) diff --git a/src/main/java/de/komoot/photon/importer/elasticsearch/Importer.java b/src/main/java/de/komoot/photon/importer/elasticsearch/Importer.java index 970f444be..d5403b3e5 100644 --- a/src/main/java/de/komoot/photon/importer/elasticsearch/Importer.java +++ b/src/main/java/de/komoot/photon/importer/elasticsearch/Importer.java @@ -23,7 +23,7 @@ public class Importer implements de.komoot.photon.importer.Importer { private final String indexType = "place"; private final Client esClient; private BulkRequestBuilder bulkRequest; - private String[] languages; + private final String[] languages; public Importer(Client esClient, String languages) { this.esClient = esClient; diff --git a/src/main/java/de/komoot/photon/importer/elasticsearch/Server.java b/src/main/java/de/komoot/photon/importer/elasticsearch/Server.java index a935fa762..9aa93fb3f 100644 --- a/src/main/java/de/komoot/photon/importer/elasticsearch/Server.java +++ b/src/main/java/de/komoot/photon/importer/elasticsearch/Server.java @@ -185,50 +185,18 @@ private JSONObject addLangsToMapping(JSONObject mappingsObject) { JSONObject nameToCollectorObject = new JSONObject(nameToCollectorString.replace("{lang}", lang)); JSONObject collectorObject = new JSONObject(collectorString.replace("{lang}", lang)); - // add language specific city to the collector - JSONObject city = propertiesObject.optJSONObject("city"); - JSONObject cityProperties = city == null ? null : city.optJSONObject("properties"); - if(cityProperties != null) { - cityProperties.put(lang, copyToCollectorObject); - city.put("properties", cityProperties); - propertiesObject.put("city", city); - } - - // add language specific context to the collector - JSONObject context = propertiesObject.optJSONObject("context"); - JSONObject contextProperties = context == null ? null : context.optJSONObject("properties"); - if(contextProperties != null) { - contextProperties.put(lang, copyToCollectorObject); - context.put("properties", contextProperties); - propertiesObject.put("context", context); - } + // add language specific tags to the collector + propertiesObject = addToCollector("city", propertiesObject, copyToCollectorObject, lang); + propertiesObject = addToCollector("context", propertiesObject, copyToCollectorObject, lang); + propertiesObject = addToCollector("country", propertiesObject, copyToCollectorObject, lang); + propertiesObject = addToCollector("street", propertiesObject, copyToCollectorObject, lang); + propertiesObject = addToCollector("name", propertiesObject, nameToCollectorObject, lang); - // add language specific country to the collector - JSONObject country = propertiesObject.optJSONObject("country"); - JSONObject countryProperties = country == null ? null : country.optJSONObject("properties"); - if(countryProperties != null) { - countryProperties.put(lang, copyToCollectorObject); - country.put("properties", countryProperties); - propertiesObject.put("country", country); - } - - // add language specific street to the collector - JSONObject street = propertiesObject.optJSONObject("street"); - JSONObject streetProperties = street == null ? null : street.optJSONObject("properties"); - if(streetProperties != null) { - streetProperties.put(lang, copyToCollectorObject); - street.put("properties", streetProperties); - propertiesObject.put("street", street); - } - - // add language specific name to the collector + // add language specific collector to default for name JSONObject name = propertiesObject.optJSONObject("name"); JSONObject nameProperties = name == null ? null : name.optJSONObject("properties"); if(nameProperties != null) { - nameProperties.put(lang, nameToCollectorObject); JSONObject defaultObject = nameProperties.optJSONObject("default"); - - // add language specific collector to default JSONArray copyToArray = defaultObject == null ? null : defaultObject.optJSONArray("copy_to"); copyToArray.put("name." + lang); @@ -239,13 +207,7 @@ private JSONObject addLangsToMapping(JSONObject mappingsObject) { } // add language specific collector - JSONObject collector = propertiesObject.optJSONObject("collector"); - JSONObject collectorProperties = collector == null ? null : collector.optJSONObject("properties"); - if(collectorProperties != null) { - collectorProperties.put(lang, collectorObject); - collector.put("properties", collectorProperties); - propertiesObject.put("collector", collector); - } + propertiesObject = addToCollector("collector", propertiesObject, collectorObject, lang); } placeObject.put("properties", propertiesObject); return mappingsObject.put("place", placeObject); @@ -254,4 +216,15 @@ private JSONObject addLangsToMapping(JSONObject mappingsObject) { log.error("cannot add langs to mapping.json, please double-check the mappings.json or the language values supplied"); return null; } + + private JSONObject addToCollector(String key, JSONObject properties, JSONObject collectorObject, String lang) { + JSONObject keyObject = properties.optJSONObject(key); + JSONObject keyProperties = keyObject == null ? null : keyObject.optJSONObject("properties"); + if(keyProperties != null) { + keyProperties.put(lang, collectorObject); + keyObject.put("properties", keyProperties); + return properties.put(key, keyObject); + } + return properties; + } } From d586d528e05d51b88d0a6f229ac98613ba78fc66 Mon Sep 17 00:00:00 2001 From: Sander Date: Tue, 16 Dec 2014 11:48:29 +0100 Subject: [PATCH 20/24] Some refactoring --- .../photon/importer/elasticsearch/Server.java | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/src/main/java/de/komoot/photon/importer/elasticsearch/Server.java b/src/main/java/de/komoot/photon/importer/elasticsearch/Server.java index 9aa93fb3f..d465df4da 100644 --- a/src/main/java/de/komoot/photon/importer/elasticsearch/Server.java +++ b/src/main/java/de/komoot/photon/importer/elasticsearch/Server.java @@ -143,15 +143,19 @@ public void recreateIndex() { final Client client = this.getClient(); final InputStream mappings = Thread.currentThread().getContextClassLoader().getResourceAsStream("mappings.json"); final InputStream index_settings = Thread.currentThread().getContextClassLoader().getResourceAsStream("index_settings.json"); - + + String mappingsString = ""; try { // get mappings as JSONObject - String mappingsString = IOUtils.toString(mappings); - JSONObject mappingsJSON = new JSONObject(mappingsString); - - // add all langs to the mapping - mappingsJSON = addLangsToMapping(mappingsJSON); + mappingsString = IOUtils.toString(mappings); + } catch(IOException e) { + log.error("cannot setup index, elastic search config files not readable", e); + } + JSONObject mappingsJSON = new JSONObject(mappingsString); + // add all langs to the mapping + mappingsJSON = addLangsToMapping(mappingsJSON); + try { client.admin().indices().prepareCreate("photon").setSettings(IOUtils.toString(index_settings)).execute().actionGet(); client.admin().indices().preparePutMapping("photon").setType("place").setSource(mappingsJSON.toString()).execute().actionGet(); } catch(IOException e) { From 80116a9639422d3ef2c77f5cee8bca6be0ec4bb7 Mon Sep 17 00:00:00 2001 From: svantulden Date: Fri, 19 Dec 2014 09:15:55 +0100 Subject: [PATCH 21/24] Deleted spaces --- src/main/java/de/komoot/photon/importer/RequestHandler.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/de/komoot/photon/importer/RequestHandler.java b/src/main/java/de/komoot/photon/importer/RequestHandler.java index 2cd3ba209..06c6f6149 100644 --- a/src/main/java/de/komoot/photon/importer/RequestHandler.java +++ b/src/main/java/de/komoot/photon/importer/RequestHandler.java @@ -21,7 +21,7 @@ public class RequestHandler extends Route { private final Searcher searcher; private final Set supportedLanguages; - + protected RequestHandler(String path, Searcher searcher, String languages) { super(path); this.searcher = searcher; From 8b6796332f5004b0782c2a17f9af086624e0fc3c Mon Sep 17 00:00:00 2001 From: svantulden Date: Fri, 19 Dec 2014 14:08:12 +0100 Subject: [PATCH 22/24] Fixes some consistency issues --- .../java/de/komoot/photon/importer/elasticsearch/Updater.java | 2 +- src/main/java/de/komoot/photon/importer/json/JsonDumper.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/de/komoot/photon/importer/elasticsearch/Updater.java b/src/main/java/de/komoot/photon/importer/elasticsearch/Updater.java index 541bd7c32..c13fe33aa 100644 --- a/src/main/java/de/komoot/photon/importer/elasticsearch/Updater.java +++ b/src/main/java/de/komoot/photon/importer/elasticsearch/Updater.java @@ -18,7 +18,7 @@ public class Updater implements de.komoot.photon.importer.Updater { private Client esClient; private BulkRequestBuilder bulkRequest; - private String[] languages; + private final String[] languages; public Updater(Client esClient, String languages) { this.esClient = esClient; diff --git a/src/main/java/de/komoot/photon/importer/json/JsonDumper.java b/src/main/java/de/komoot/photon/importer/json/JsonDumper.java index 1e351b9fc..d8363134c 100644 --- a/src/main/java/de/komoot/photon/importer/json/JsonDumper.java +++ b/src/main/java/de/komoot/photon/importer/json/JsonDumper.java @@ -17,7 +17,7 @@ @Slf4j public class JsonDumper implements Importer { private PrintWriter writer = null; - private String[] languages; + private final String[] languages; public JsonDumper(String filename, String languages) throws FileNotFoundException { this.writer = new PrintWriter(filename); From 1a50011bee381b3c8dcfc108546a3246031004f8 Mon Sep 17 00:00:00 2001 From: svantulden Date: Fri, 19 Dec 2014 14:19:10 +0100 Subject: [PATCH 23/24] Reworked exception handling for recreateIndex() --- .../java/de/komoot/photon/importer/App.java | 19 ++++++++++++++++--- .../photon/importer/elasticsearch/Server.java | 18 ++++-------------- .../java/de/komoot/photon/ESBaseTester.java | 3 ++- .../importer/elasticsearch/ImporterTest.java | 3 ++- .../RemoveStreetDuplicatesTest.java | 3 ++- 5 files changed, 26 insertions(+), 20 deletions(-) diff --git a/src/main/java/de/komoot/photon/importer/App.java b/src/main/java/de/komoot/photon/importer/App.java index 2b256c631..39f88ed38 100644 --- a/src/main/java/de/komoot/photon/importer/App.java +++ b/src/main/java/de/komoot/photon/importer/App.java @@ -15,6 +15,7 @@ import spark.Route; import java.io.FileNotFoundException; +import java.io.IOException; import static spark.Spark.*; @@ -57,14 +58,26 @@ public static void main(String[] rawArgs) { Client esNodeClient = esServer.getClient(); - if(args.isDeleteIndex()) { - esServer.recreateIndex(); + if(args.isDeleteIndex()) { + try { + esServer.recreateIndex(); + } catch (IOException e) { + log.error("cannot setup index, elastic search config files not readable", e); + return; + } + log.info("deleted photon index and created an empty new one."); return; } if(args.isNominatimImport()) { - esServer.recreateIndex(); // dump previous data + try { + esServer.recreateIndex(); // dump previous data + } catch (IOException e) { + log.error("cannot setup index, elastic search config files not readable", e); + return; + } + log.info("starting import from nominatim to photon with languages: " + args.getLanguages()); Importer importer = new Importer(esNodeClient, args.getLanguages()); NominatimConnector nominatimConnector = new NominatimConnector(args.getHost(), args.getPort(), args.getDatabase(), args.getUser(), args.getPassword()); diff --git a/src/main/java/de/komoot/photon/importer/elasticsearch/Server.java b/src/main/java/de/komoot/photon/importer/elasticsearch/Server.java index d465df4da..da4ec62da 100644 --- a/src/main/java/de/komoot/photon/importer/elasticsearch/Server.java +++ b/src/main/java/de/komoot/photon/importer/elasticsearch/Server.java @@ -137,30 +137,20 @@ private File setupDirectories(URL directoryName) { return mainDirectory; } - public void recreateIndex() { + public void recreateIndex() throws IOException { deleteIndex(); final Client client = this.getClient(); final InputStream mappings = Thread.currentThread().getContextClassLoader().getResourceAsStream("mappings.json"); final InputStream index_settings = Thread.currentThread().getContextClassLoader().getResourceAsStream("index_settings.json"); - String mappingsString = ""; - try { - // get mappings as JSONObject - mappingsString = IOUtils.toString(mappings); - } catch(IOException e) { - log.error("cannot setup index, elastic search config files not readable", e); - } + String mappingsString = IOUtils.toString(mappings); JSONObject mappingsJSON = new JSONObject(mappingsString); // add all langs to the mapping mappingsJSON = addLangsToMapping(mappingsJSON); - try { - client.admin().indices().prepareCreate("photon").setSettings(IOUtils.toString(index_settings)).execute().actionGet(); - client.admin().indices().preparePutMapping("photon").setType("place").setSource(mappingsJSON.toString()).execute().actionGet(); - } catch(IOException e) { - log.error("cannot setup index, elastic search config files not readable", e); - } + client.admin().indices().prepareCreate("photon").setSettings(IOUtils.toString(index_settings)).execute().actionGet(); + client.admin().indices().preparePutMapping("photon").setType("place").setSource(mappingsJSON.toString()).execute().actionGet(); } public DeleteIndexResponse deleteIndex() { diff --git a/src/test/java/de/komoot/photon/ESBaseTester.java b/src/test/java/de/komoot/photon/ESBaseTester.java index 4a427fece..c3279f756 100644 --- a/src/test/java/de/komoot/photon/ESBaseTester.java +++ b/src/test/java/de/komoot/photon/ESBaseTester.java @@ -2,6 +2,7 @@ import de.komoot.photon.importer.elasticsearch.Server; import java.io.File; +import java.io.IOException; import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; import org.elasticsearch.client.Client; import org.elasticsearch.index.query.QueryBuilders; @@ -26,7 +27,7 @@ public static void tearDownClass() { shutdownES(); } - public void setUpES() { + public void setUpES() throws IOException { if (server != null) return; diff --git a/src/test/java/de/komoot/photon/importer/elasticsearch/ImporterTest.java b/src/test/java/de/komoot/photon/importer/elasticsearch/ImporterTest.java index 80e8f2b69..d1782e81f 100644 --- a/src/test/java/de/komoot/photon/importer/elasticsearch/ImporterTest.java +++ b/src/test/java/de/komoot/photon/importer/elasticsearch/ImporterTest.java @@ -2,6 +2,7 @@ import de.komoot.photon.ESBaseTester; import de.komoot.photon.importer.model.PhotonDoc; +import java.io.IOException; import org.json.JSONObject; import org.junit.*; @@ -17,7 +18,7 @@ public class ImporterTest extends ESBaseTester { @Before - public void setUp() { + public void setUp() throws IOException { setUpES(); deleteAll(); } diff --git a/src/test/java/de/komoot/photon/importer/elasticsearch/RemoveStreetDuplicatesTest.java b/src/test/java/de/komoot/photon/importer/elasticsearch/RemoveStreetDuplicatesTest.java index 9cda33230..7166edcd1 100644 --- a/src/test/java/de/komoot/photon/importer/elasticsearch/RemoveStreetDuplicatesTest.java +++ b/src/test/java/de/komoot/photon/importer/elasticsearch/RemoveStreetDuplicatesTest.java @@ -6,6 +6,7 @@ import com.vividsolutions.jts.geom.PrecisionModel; import de.komoot.photon.ESBaseTester; import de.komoot.photon.importer.model.PhotonDoc; +import java.io.IOException; import org.json.JSONObject; import org.junit.*; @@ -20,7 +21,7 @@ public class RemoveStreetDuplicatesTest extends ESBaseTester { GeometryFactory FACTORY = new GeometryFactory(new PrecisionModel(), 4326); @Before - public void setUp() { + public void setUp() throws IOException { setUpES(); deleteAll(); PhotonDoc street1 = this.createStreetDoc(1, "Walserstraße", "6993"); From c4726f253c06de51b8f3ae3f114b75d67e2fe126 Mon Sep 17 00:00:00 2001 From: svantulden Date: Fri, 19 Dec 2014 14:19:21 +0100 Subject: [PATCH 24/24] Removed unneeded null check --- .../photon/importer/elasticsearch/Server.java | 71 +++++++++---------- 1 file changed, 35 insertions(+), 36 deletions(-) diff --git a/src/main/java/de/komoot/photon/importer/elasticsearch/Server.java b/src/main/java/de/komoot/photon/importer/elasticsearch/Server.java index da4ec62da..d0e76ed1b 100644 --- a/src/main/java/de/komoot/photon/importer/elasticsearch/Server.java +++ b/src/main/java/de/komoot/photon/importer/elasticsearch/Server.java @@ -168,45 +168,44 @@ private JSONObject addLangsToMapping(JSONObject mappingsObject) { String nameToCollectorString = "{\"type\":\"string\",\"index\":\"no\",\"fields\":{\"ngrams\":{\"type\":\"string\",\"index_analyzer\":\"index_ngram\"},\"raw\":{\"type\":\"string\",\"index_analyzer\":\"index_raw\"}},\"copy_to\":[\"collector.{lang}\"]}"; String collectorString = "{\"type\":\"string\",\"index\":\"no\",\"fields\":{\"ngrams\":{\"type\":\"string\",\"index_analyzer\":\"index_ngram\"},\"raw\":{\"type\":\"string\",\"index_analyzer\":\"index_raw\"}},\"copy_to\":[\"collector.{lang}\"]}}},\"street\":{\"type\":\"object\",\"properties\":{\"default\":{\"index\":\"no\",\"type\":\"string\",\"copy_to\":[\"collector.default\"]}"; - if(mappingsObject != null) { - JSONObject placeObject = mappingsObject.optJSONObject("place"); - JSONObject propertiesObject = placeObject == null ? null : placeObject.optJSONObject("properties"); - - if(propertiesObject != null) { - for(String lang : langs) { - // create lang-specific json objects - JSONObject copyToCollectorObject = new JSONObject(copyToCollectorString.replace("{lang}", lang)); - JSONObject nameToCollectorObject = new JSONObject(nameToCollectorString.replace("{lang}", lang)); - JSONObject collectorObject = new JSONObject(collectorString.replace("{lang}", lang)); - - // add language specific tags to the collector - propertiesObject = addToCollector("city", propertiesObject, copyToCollectorObject, lang); - propertiesObject = addToCollector("context", propertiesObject, copyToCollectorObject, lang); - propertiesObject = addToCollector("country", propertiesObject, copyToCollectorObject, lang); - propertiesObject = addToCollector("street", propertiesObject, copyToCollectorObject, lang); - propertiesObject = addToCollector("name", propertiesObject, nameToCollectorObject, lang); - - // add language specific collector to default for name - JSONObject name = propertiesObject.optJSONObject("name"); - JSONObject nameProperties = name == null ? null : name.optJSONObject("properties"); - if(nameProperties != null) { - JSONObject defaultObject = nameProperties.optJSONObject("default"); - JSONArray copyToArray = defaultObject == null ? null : defaultObject.optJSONArray("copy_to"); - copyToArray.put("name." + lang); - - defaultObject.put("copy_to", copyToArray); - nameProperties.put("default", defaultObject); - name.put("properties", nameProperties); - propertiesObject.put("name", name); - } - - // add language specific collector - propertiesObject = addToCollector("collector", propertiesObject, collectorObject, lang); + JSONObject placeObject = mappingsObject.optJSONObject("place"); + JSONObject propertiesObject = placeObject == null ? null : placeObject.optJSONObject("properties"); + + if(propertiesObject != null) { + for(String lang : langs) { + // create lang-specific json objects + JSONObject copyToCollectorObject = new JSONObject(copyToCollectorString.replace("{lang}", lang)); + JSONObject nameToCollectorObject = new JSONObject(nameToCollectorString.replace("{lang}", lang)); + JSONObject collectorObject = new JSONObject(collectorString.replace("{lang}", lang)); + + // add language specific tags to the collector + propertiesObject = addToCollector("city", propertiesObject, copyToCollectorObject, lang); + propertiesObject = addToCollector("context", propertiesObject, copyToCollectorObject, lang); + propertiesObject = addToCollector("country", propertiesObject, copyToCollectorObject, lang); + propertiesObject = addToCollector("street", propertiesObject, copyToCollectorObject, lang); + propertiesObject = addToCollector("name", propertiesObject, nameToCollectorObject, lang); + + // add language specific collector to default for name + JSONObject name = propertiesObject.optJSONObject("name"); + JSONObject nameProperties = name == null ? null : name.optJSONObject("properties"); + if(nameProperties != null) { + JSONObject defaultObject = nameProperties.optJSONObject("default"); + JSONArray copyToArray = defaultObject == null ? null : defaultObject.optJSONArray("copy_to"); + copyToArray.put("name." + lang); + + defaultObject.put("copy_to", copyToArray); + nameProperties.put("default", defaultObject); + name.put("properties", nameProperties); + propertiesObject.put("name", name); } - placeObject.put("properties", propertiesObject); - return mappingsObject.put("place", placeObject); + + // add language specific collector + propertiesObject = addToCollector("collector", propertiesObject, collectorObject, lang); } + placeObject.put("properties", propertiesObject); + return mappingsObject.put("place", placeObject); } + log.error("cannot add langs to mapping.json, please double-check the mappings.json or the language values supplied"); return null; }