diff --git a/.gitignore b/.gitignore index a8a9bd80f..bb76caa7b 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,7 @@ dependency-reduced-pom.xml *.~* __pycache__ + +nb-configuration.xml + +nbactions.xml diff --git a/es/mappings.json b/es/mappings.json index 6d611e6fd..4a8f6f522 100644 --- a/es/mappings.json +++ b/es/mappings.json @@ -47,119 +47,34 @@ }, "city": { "properties": { - "de": { - "type": "string", - "index": "no", - "copy_to": [ - "collector.de" - ] - }, "default": { "type": "string", "index": "no", "copy_to": [ "collector.default" ] - }, - "en": { - "type": "string", - "index": "no", - "copy_to": [ - "collector.en" - ] - }, - "fr": { - "type": "string", - "index": "no", - "copy_to": [ - "collector.fr" - ] - }, - "it": { - "type": "string", - "index": "no", - "copy_to": [ - "collector.it" - ] } } }, "context": { "properties": { - "de": { - "type": "string", - "index": "no", - "copy_to": [ - "collector.default", - "collector.de" - ] - }, "default": { "type": "string", "index": "no", "copy_to": [ "collector.default" ] - }, - "en": { - "type": "string", - "index": "no", - "copy_to": [ - "collector.en" - ] - }, - "fr": { - "type": "string", - "index": "no", - "copy_to": [ - "collector.fr" - ] - }, - "it": { - "type": "string", - "index": "no", - "copy_to": [ - "collector.it" - ] } } }, "country": { "properties": { - "de": { - "type": "string", - "index": "no", - "copy_to": [ - "collector.de" - ] - }, "default": { "type": "string", "index": "no", "copy_to": [ "collector.default" ] - }, - "en": { - "type": "string", - "index": "no", - "copy_to": [ - "collector.en" - ] - }, - "fr": { - "type": "string", - "index": "no", - "copy_to": [ - "collector.fr" - ] - }, - "it": { - "type": "string", - "index": "no", - "copy_to": [ - "collector.it" - ] } } }, @@ -218,83 +133,11 @@ "collector.default" ] }, - "de": { - "type": "string", - "index": "no", - "fields": { - "ngrams": { - "type": "string", - "index_analyzer": "index_ngram" - }, - "raw": { - "type": "string", - "index_analyzer": "index_raw" - } - }, - "copy_to": [ - "collector.de" - ] - }, "default": { "type": "string", "index": "no", "copy_to": [ - "collector.default", - "name.en", - "name.de", - "name.fr", - "name.it" - ] - }, - "en": { - "type": "string", - "index": "no", - "fields": { - "ngrams": { - "type": "string", - "index_analyzer": "index_ngram" - }, - "raw": { - "type": "string", - "index_analyzer": "index_raw" - } - }, - "copy_to": [ - "collector.en" - ] - }, - "fr": { - "type": "string", - "index": "no", - "fields": { - "ngrams": { - "type": "string", - "index_analyzer": "index_ngram" - }, - "raw": { - "type": "string", - "index_analyzer": "index_raw" - } - }, - "copy_to": [ - "collector.fr" - ] - }, - "it": { - "type": "string", - "index": "no", - "fields": { - "ngrams": { - "type": "string", - "index_analyzer": "index_ngram" - }, - "raw": { - "type": "string", - "index_analyzer": "index_raw" - } - }, - "copy_to": [ - "collector.it" + "collector.default" ] } } @@ -308,34 +151,6 @@ "copy_to": [ "collector.default" ] - }, - "de": { - "index": "no", - "type": "string", - "copy_to": [ - "collector.de" - ] - }, - "en": { - "index": "no", - "type": "string", - "copy_to": [ - "collector.en" - ] - }, - "fr": { - "index": "no", - "type": "string", - "copy_to": [ - "collector.fr" - ] - }, - "it": { - "index": "no", - "type": "string", - "copy_to": [ - "collector.it" - ] } } }, @@ -351,46 +166,6 @@ "analyzer": "index_raw" } } - }, - "de": { - "type": "string", - "analyzer": "index_ngram", - "fields": { - "raw": { - "type": "string", - "analyzer": "index_raw" - } - } - }, - "en": { - "type": "string", - "analyzer": "index_ngram", - "fields": { - "raw": { - "type": "string", - "analyzer": "index_raw" - } - } - }, - "fr": { - "type": "string", - "analyzer": "index_ngram", - "fields": { - "raw": { - "type": "string", - "analyzer": "index_raw" - } - } - }, - "it": { - "type": "string", - "analyzer": "index_ngram", - "fields": { - "raw": { - "type": "string", - "analyzer": "index_raw" - } - } } } } diff --git a/src/main/java/de/komoot/photon/importer/App.java b/src/main/java/de/komoot/photon/importer/App.java index 6b59ddf3f..39f88ed38 100644 --- a/src/main/java/de/komoot/photon/importer/App.java +++ b/src/main/java/de/komoot/photon/importer/App.java @@ -15,6 +15,7 @@ import spark.Route; import java.io.FileNotFoundException; +import java.io.IOException; import static spark.Spark.*; @@ -41,7 +42,7 @@ public static void main(String[] rawArgs) { if(args.getJsonDump() != null) { try { final String filename = args.getJsonDump(); - final JsonDumper jsonDumper = new JsonDumper(filename); + final JsonDumper jsonDumper = new JsonDumper(filename, args.getLanguages()); NominatimConnector nominatimConnector = new NominatimConnector(args.getHost(), args.getPort(), args.getDatabase(), args.getUser(), args.getPassword()); nominatimConnector.setImporter(jsonDumper); nominatimConnector.readEntireDatabase(); @@ -52,29 +53,47 @@ public static void main(String[] rawArgs) { } } - final Server esServer = new Server(args.getCluster(), args.getDataDirectory()); + final Server esServer = new Server(args.getCluster(), args.getDataDirectory(), args.getLanguages()); esServer.start(); Client esNodeClient = esServer.getClient(); - if(args.isDeleteIndex()) { - esServer.recreateIndex(); + if(args.isDeleteIndex()) { + try { + esServer.recreateIndex(); + } catch (IOException e) { + log.error("cannot setup index, elastic search config files not readable", e); + return; + } + log.info("deleted photon index and created an empty new one."); return; } if(args.isNominatimImport()) { - esServer.recreateIndex(); // dump previous data - Importer importer = new Importer(esNodeClient); + try { + esServer.recreateIndex(); // dump previous data + } catch (IOException e) { + log.error("cannot setup index, elastic search config files not readable", e); + return; + } + + log.info("starting import from nominatim to photon with languages: " + args.getLanguages()); + Importer importer = new Importer(esNodeClient, args.getLanguages()); NominatimConnector nominatimConnector = new NominatimConnector(args.getHost(), args.getPort(), args.getDatabase(), args.getUser(), args.getPassword()); nominatimConnector.setImporter(importer); - nominatimConnector.readEntireDatabase(); - log.info("imported data from nominatim to photon."); + try { + nominatimConnector.readEntireDatabase(); + } catch (Exception e) { + log.info("ERROR IMPORTING FROM NOMINATIM: "+e.getMessage()); + } + + log.info("imported data from nominatim to photon with languages: " + args.getLanguages()); return; } final NominatimUpdater nominatimUpdater = new NominatimUpdater(args.getHost(), args.getPort(), args.getDatabase(), args.getUser(), args.getPassword()); - de.komoot.photon.importer.Updater updater = new de.komoot.photon.importer.elasticsearch.Updater(esNodeClient); + de.komoot.photon.importer.Updater updater = new de.komoot.photon.importer.elasticsearch.Updater(esNodeClient, args.getLanguages()); nominatimUpdater.setUpdater(updater); startApi(args, esNodeClient, nominatimUpdater); @@ -99,7 +118,7 @@ public void run() { }); final Searcher searcher = new Searcher(esNodeClient); - get(new RequestHandler("api", searcher)); - get(new RequestHandler("api/", searcher)); + get(new RequestHandler("api", searcher, args.getLanguages())); + get(new RequestHandler("api/", searcher, args.getLanguages())); } } diff --git a/src/main/java/de/komoot/photon/importer/CommandLineArgs.java b/src/main/java/de/komoot/photon/importer/CommandLineArgs.java index 7ea80a9b4..61fd8bbc3 100644 --- a/src/main/java/de/komoot/photon/importer/CommandLineArgs.java +++ b/src/main/java/de/komoot/photon/importer/CommandLineArgs.java @@ -16,6 +16,9 @@ public class CommandLineArgs { @Parameter(names = "-nominatim-import", description = "import nominatim database into photon (this will delete previous index)") private boolean nominatimImport = false; + + @Parameter(names = "-languages", description = "languages nominatim importer should import and use at run-time, comma seperated (default: 'en,fr,de,it')") + private String languages = "en,fr,de,it"; @Parameter(names = "-json", description = "import nominatim database and dump it to a json like files in (useful for developing)") private String jsonDump = null; diff --git a/src/main/java/de/komoot/photon/importer/RequestHandler.java b/src/main/java/de/komoot/photon/importer/RequestHandler.java index 0fa9f5419..06c6f6149 100644 --- a/src/main/java/de/komoot/photon/importer/RequestHandler.java +++ b/src/main/java/de/komoot/photon/importer/RequestHandler.java @@ -2,7 +2,8 @@ import com.google.common.base.Joiner; import de.komoot.photon.importer.elasticsearch.Searcher; -import org.elasticsearch.common.collect.ImmutableSet; +import java.util.Arrays; +import java.util.HashSet; import org.json.JSONArray; import org.json.JSONObject; import spark.Request; @@ -19,11 +20,12 @@ */ public class RequestHandler extends Route { private final Searcher searcher; - private static final Set supportedLanguages = ImmutableSet.of("de", "en", "fr", "it"); + private final Set supportedLanguages; - protected RequestHandler(String path, Searcher searcher) { + protected RequestHandler(String path, Searcher searcher, String languages) { super(path); this.searcher = searcher; + this.supportedLanguages = new HashSet(Arrays.asList(languages.split(","))); } @Override diff --git a/src/main/java/de/komoot/photon/importer/Utils.java b/src/main/java/de/komoot/photon/importer/Utils.java index e98fdc5cb..da67731a5 100644 --- a/src/main/java/de/komoot/photon/importer/Utils.java +++ b/src/main/java/de/komoot/photon/importer/Utils.java @@ -22,9 +22,7 @@ public class Utils { static final Joiner commaJoiner = Joiner.on(", ").skipNulls(); - final static String[] languages = new String[]{"de", "en", "fr", "it"}; - - public static XContentBuilder convert(PhotonDoc doc) throws IOException { + public static XContentBuilder convert(PhotonDoc doc, String[] languages) throws IOException { XContentBuilder builder = XContentFactory.jsonBuilder().startObject() .field(Tags.KEY_OSM_ID, doc.getOsmId()) .field(Tags.KEY_OSM_TYPE, doc.getOsmType()) @@ -47,11 +45,11 @@ public static XContentBuilder convert(PhotonDoc doc) throws IOException { builder.field("postcode", doc.getPostcode()); } - writeName(builder, doc.getName()); - writeIntlNames(builder, doc.getCity(), "city"); - writeIntlNames(builder, doc.getCountry(), "country"); - writeIntlNames(builder, doc.getStreet(), "street"); - writeContext(builder, doc.getContext()); + writeName(builder, doc.getName(), languages); + writeIntlNames(builder, doc.getCity(), "city", languages); + writeIntlNames(builder, doc.getCountry(), "country", languages); + writeIntlNames(builder, doc.getStreet(), "street", languages); + writeContext(builder, doc.getContext(), languages); writeExtent(builder, doc.getBbox()); return builder; @@ -74,8 +72,8 @@ private static void writeExtent(XContentBuilder builder, Envelope bbox) throws I builder.endObject(); } - private static void writeName(XContentBuilder builder, Map name) throws IOException { - Map fNames = filterNames(name); + private static void writeName(XContentBuilder builder, Map name, String[] languages) throws IOException { + Map fNames = filterNames(name, languages); if(name.get("alt_name") != null) fNames.put("alt", name.get("alt_name")); @@ -102,7 +100,7 @@ private static void write(XContentBuilder builder, Map fNames, S builder.endObject(); } - protected static void writeContext(XContentBuilder builder, Set> contexts) throws IOException { + protected static void writeContext(XContentBuilder builder, Set> contexts, String[] languages) throws IOException { final SetMultimap multimap = HashMultimap.create(); for(Map context : contexts) { @@ -129,16 +127,16 @@ protected static void writeContext(XContentBuilder builder, Set names, String name) throws IOException { - Map fNames = filterNames(names); + private static void writeIntlNames(XContentBuilder builder, Map names, String name, String[] languages) throws IOException { + Map fNames = filterNames(names, languages); write(builder, fNames, name); } - private static Map filterNames(Map names) { - return filterNames(names, new HashMap()); + private static Map filterNames(Map names, String[] languages) { + return filterNames(names, new HashMap(), languages); } - private static Map filterNames(Map names, HashMap filteredNames) { + private static Map filterNames(Map names, HashMap filteredNames, String[] languages) { if(names == null) return filteredNames; if(names.get("name") != null) { @@ -153,4 +151,18 @@ private static Map filterNames(Map names, HashMa return filteredNames; } + + // http://stackoverflow.com/a/4031040/1437096 + public static String stripNonDigits( + final CharSequence input /* inspired by seh's comment */){ + final StringBuilder sb = new StringBuilder( + input.length() /* also inspired by seh's comment */); + for(int i = 0; i < input.length(); i++){ + final char c = input.charAt(i); + if(c > 47 && c < 58){ + sb.append(c); + } + } + return sb.toString(); + } } diff --git a/src/main/java/de/komoot/photon/importer/elasticsearch/Importer.java b/src/main/java/de/komoot/photon/importer/elasticsearch/Importer.java index 34f930f97..d5403b3e5 100644 --- a/src/main/java/de/komoot/photon/importer/elasticsearch/Importer.java +++ b/src/main/java/de/komoot/photon/importer/elasticsearch/Importer.java @@ -23,17 +23,19 @@ public class Importer implements de.komoot.photon.importer.Importer { private final String indexType = "place"; private final Client esClient; private BulkRequestBuilder bulkRequest; + private final String[] languages; - public Importer(Client esClient) { + public Importer(Client esClient, String languages) { this.esClient = esClient; this.bulkRequest = esClient.prepareBulk(); + this.languages = languages.split(","); } @Override public void add(PhotonDoc doc) { try { this.bulkRequest.add(this.esClient.prepareIndex(indexName, indexType). - setSource(Utils.convert(doc)).setId(String.valueOf(doc.getPlaceId()))); + setSource(Utils.convert(doc, languages)).setId(String.valueOf(doc.getPlaceId()))); } catch(IOException e) { log.error("could not ", e); } diff --git a/src/main/java/de/komoot/photon/importer/elasticsearch/Searcher.java b/src/main/java/de/komoot/photon/importer/elasticsearch/Searcher.java index f12041cb6..054e6ebf6 100644 --- a/src/main/java/de/komoot/photon/importer/elasticsearch/Searcher.java +++ b/src/main/java/de/komoot/photon/importer/elasticsearch/Searcher.java @@ -6,6 +6,7 @@ import de.komoot.photon.importer.Tags; import de.komoot.photon.importer.osm.OSMTags; import lombok.extern.slf4j.Slf4j; +import de.komoot.photon.importer.Utils; import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringEscapeUtils; import org.apache.commons.lang3.text.StrSubstitutor; @@ -66,14 +67,14 @@ public List search(String query, String lang, Double lon, Double lat SearchResponse response = client.prepareSearch("photon").setSearchType(SearchType.QUERY_AND_FETCH).setQuery(query).setSize(limit).setTimeout(TimeValue.timeValueSeconds(7)).execute().actionGet(); List results = convert(response.getHits().getHits(), lang); - results = removeStreetDuplicates(results); + results = removeStreetDuplicates(results, lang); if(results.size() > limit) { results = results.subList(0, limit); } return results; } - private List removeStreetDuplicates(List results) { + private List removeStreetDuplicates(List results, String lang) { List filteredItems = Lists.newArrayListWithCapacity(results.size()); final HashSet keys = Sets.newHashSet(); for(JSONObject result : results) { @@ -84,12 +85,12 @@ private List removeStreetDuplicates(List results) { // street has a postcode and name String postcode = properties.getString(OSMTags.KEY_POSTCODE); String name = properties.getString(OSMTags.KEY_NAME); - String key = postcode + ":" + name; + String key = postcode + ":" + name; if(keys.contains(key)) { // a street with this name + postcode is already part of the result list continue; - } + } keys.add(key); } } diff --git a/src/main/java/de/komoot/photon/importer/elasticsearch/Server.java b/src/main/java/de/komoot/photon/importer/elasticsearch/Server.java index ef4174310..d0e76ed1b 100644 --- a/src/main/java/de/komoot/photon/importer/elasticsearch/Server.java +++ b/src/main/java/de/komoot/photon/importer/elasticsearch/Server.java @@ -21,6 +21,8 @@ import java.net.URL; import static org.elasticsearch.node.NodeBuilder.nodeBuilder; +import org.json.JSONArray; +import org.json.JSONObject; /** * Helper class to start/stop elasticserach node and get elasticsearch clients @@ -37,8 +39,9 @@ public class Server { private File updateDirectory; private File tempDirectory; private File importDirectory; + private String[] langs; - public Server(String clusterName, String mainDirectory) { + public Server(String clusterName, String mainDirectory, String langs) { try { setupDirectories(new URL("file://" + mainDirectory)); } catch(MalformedURLException e) { @@ -52,6 +55,7 @@ public Server(String clusterName, String mainDirectory) { } } this.clusterName = clusterName; + this.langs = langs.split(","); } public Server start() { @@ -133,19 +137,20 @@ private File setupDirectories(URL directoryName) { return mainDirectory; } - public void recreateIndex() { + public void recreateIndex() throws IOException { deleteIndex(); final Client client = this.getClient(); final InputStream mappings = Thread.currentThread().getContextClassLoader().getResourceAsStream("mappings.json"); final InputStream index_settings = Thread.currentThread().getContextClassLoader().getResourceAsStream("index_settings.json"); - - try { - client.admin().indices().prepareCreate("photon").setSettings(IOUtils.toString(index_settings)).execute().actionGet(); - client.admin().indices().preparePutMapping("photon").setType("place").setSource(IOUtils.toString(mappings)).execute().actionGet(); - } catch(IOException e) { - log.error("cannot setup index, elastic search config files not readable", e); - } + + String mappingsString = IOUtils.toString(mappings); + JSONObject mappingsJSON = new JSONObject(mappingsString); + + // add all langs to the mapping + mappingsJSON = addLangsToMapping(mappingsJSON); + client.admin().indices().prepareCreate("photon").setSettings(IOUtils.toString(index_settings)).execute().actionGet(); + client.admin().indices().preparePutMapping("photon").setType("place").setSource(mappingsJSON.toString()).execute().actionGet(); } public DeleteIndexResponse deleteIndex() { @@ -156,4 +161,63 @@ public DeleteIndexResponse deleteIndex() { return null; } } + + private JSONObject addLangsToMapping(JSONObject mappingsObject) { + // define collector json strings + String copyToCollectorString = "{\"type\":\"string\",\"index\":\"no\",\"copy_to\":[\"collector.{lang}\"]}"; + String nameToCollectorString = "{\"type\":\"string\",\"index\":\"no\",\"fields\":{\"ngrams\":{\"type\":\"string\",\"index_analyzer\":\"index_ngram\"},\"raw\":{\"type\":\"string\",\"index_analyzer\":\"index_raw\"}},\"copy_to\":[\"collector.{lang}\"]}"; + String collectorString = "{\"type\":\"string\",\"index\":\"no\",\"fields\":{\"ngrams\":{\"type\":\"string\",\"index_analyzer\":\"index_ngram\"},\"raw\":{\"type\":\"string\",\"index_analyzer\":\"index_raw\"}},\"copy_to\":[\"collector.{lang}\"]}}},\"street\":{\"type\":\"object\",\"properties\":{\"default\":{\"index\":\"no\",\"type\":\"string\",\"copy_to\":[\"collector.default\"]}"; + + JSONObject placeObject = mappingsObject.optJSONObject("place"); + JSONObject propertiesObject = placeObject == null ? null : placeObject.optJSONObject("properties"); + + if(propertiesObject != null) { + for(String lang : langs) { + // create lang-specific json objects + JSONObject copyToCollectorObject = new JSONObject(copyToCollectorString.replace("{lang}", lang)); + JSONObject nameToCollectorObject = new JSONObject(nameToCollectorString.replace("{lang}", lang)); + JSONObject collectorObject = new JSONObject(collectorString.replace("{lang}", lang)); + + // add language specific tags to the collector + propertiesObject = addToCollector("city", propertiesObject, copyToCollectorObject, lang); + propertiesObject = addToCollector("context", propertiesObject, copyToCollectorObject, lang); + propertiesObject = addToCollector("country", propertiesObject, copyToCollectorObject, lang); + propertiesObject = addToCollector("street", propertiesObject, copyToCollectorObject, lang); + propertiesObject = addToCollector("name", propertiesObject, nameToCollectorObject, lang); + + // add language specific collector to default for name + JSONObject name = propertiesObject.optJSONObject("name"); + JSONObject nameProperties = name == null ? null : name.optJSONObject("properties"); + if(nameProperties != null) { + JSONObject defaultObject = nameProperties.optJSONObject("default"); + JSONArray copyToArray = defaultObject == null ? null : defaultObject.optJSONArray("copy_to"); + copyToArray.put("name." + lang); + + defaultObject.put("copy_to", copyToArray); + nameProperties.put("default", defaultObject); + name.put("properties", nameProperties); + propertiesObject.put("name", name); + } + + // add language specific collector + propertiesObject = addToCollector("collector", propertiesObject, collectorObject, lang); + } + placeObject.put("properties", propertiesObject); + return mappingsObject.put("place", placeObject); + } + + log.error("cannot add langs to mapping.json, please double-check the mappings.json or the language values supplied"); + return null; + } + + private JSONObject addToCollector(String key, JSONObject properties, JSONObject collectorObject, String lang) { + JSONObject keyObject = properties.optJSONObject(key); + JSONObject keyProperties = keyObject == null ? null : keyObject.optJSONObject("properties"); + if(keyProperties != null) { + keyProperties.put(lang, collectorObject); + keyObject.put("properties", keyProperties); + return properties.put(key, keyObject); + } + return properties; + } } diff --git a/src/main/java/de/komoot/photon/importer/elasticsearch/Updater.java b/src/main/java/de/komoot/photon/importer/elasticsearch/Updater.java index dc0bc34e3..c13fe33aa 100644 --- a/src/main/java/de/komoot/photon/importer/elasticsearch/Updater.java +++ b/src/main/java/de/komoot/photon/importer/elasticsearch/Updater.java @@ -18,10 +18,12 @@ public class Updater implements de.komoot.photon.importer.Updater { private Client esClient; private BulkRequestBuilder bulkRequest; + private final String[] languages; - public Updater(Client esClient) { + public Updater(Client esClient, String languages) { this.esClient = esClient; this.bulkRequest = esClient.prepareBulk(); + this.languages = languages.split(","); } public void finish() { @@ -40,7 +42,7 @@ public void updateOrCreate(PhotonDoc updatedDoc) { public void create(PhotonDoc doc) { try { - this.bulkRequest.add(this.esClient.prepareIndex("photon", "place").setSource(Utils.convert(doc)).setId(String.valueOf(doc.getPlaceId()))); + this.bulkRequest.add(this.esClient.prepareIndex("photon", "place").setSource(Utils.convert(doc, this.languages)).setId(String.valueOf(doc.getPlaceId()))); } catch(IOException e) { log.error(String.format("creation of new doc [%s] failed", doc), e); } @@ -48,7 +50,7 @@ public void create(PhotonDoc doc) { public void update(PhotonDoc doc) { try { - this.bulkRequest.add(this.esClient.prepareUpdate("photon", "place", String.valueOf(doc.getPlaceId())).setDoc(Utils.convert(doc))); + this.bulkRequest.add(this.esClient.prepareUpdate("photon", "place", String.valueOf(doc.getPlaceId())).setDoc(Utils.convert(doc, this.languages))); } catch(IOException e) { log.error(String.format("update of new doc [%s] failed", doc), e); } diff --git a/src/main/java/de/komoot/photon/importer/json/JsonDumper.java b/src/main/java/de/komoot/photon/importer/json/JsonDumper.java index 349a134a0..d8363134c 100644 --- a/src/main/java/de/komoot/photon/importer/json/JsonDumper.java +++ b/src/main/java/de/komoot/photon/importer/json/JsonDumper.java @@ -17,16 +17,18 @@ @Slf4j public class JsonDumper implements Importer { private PrintWriter writer = null; + private final String[] languages; - public JsonDumper(String filename) throws FileNotFoundException { + public JsonDumper(String filename, String languages) throws FileNotFoundException { this.writer = new PrintWriter(filename); + this.languages = languages.split(","); } @Override public void add(PhotonDoc doc) { try { writer.println("{\"index\": {}}"); - writer.println(Utils.convert(doc).string()); + writer.println(Utils.convert(doc,this.languages).string()); } catch(IOException e) { log.error("error writing json file", e); } diff --git a/src/test/java/de/komoot/photon/ESBaseTester.java b/src/test/java/de/komoot/photon/ESBaseTester.java index 4c8b685ec..c3279f756 100644 --- a/src/test/java/de/komoot/photon/ESBaseTester.java +++ b/src/test/java/de/komoot/photon/ESBaseTester.java @@ -2,6 +2,7 @@ import de.komoot.photon.importer.elasticsearch.Server; import java.io.File; +import java.io.IOException; import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; import org.elasticsearch.client.Client; import org.elasticsearch.index.query.QueryBuilders; @@ -26,11 +27,11 @@ public static void tearDownClass() { shutdownES(); } - public void setUpES() { + public void setUpES() throws IOException { if (server != null) return; - server = new Server("photon", new File("./target/es_photon").getAbsolutePath()).start(true); + server = new Server("photon", new File("./target/es_photon").getAbsolutePath(), "en").start(true); server.recreateIndex(); } diff --git a/src/test/java/de/komoot/photon/importer/elasticsearch/ImporterTest.java b/src/test/java/de/komoot/photon/importer/elasticsearch/ImporterTest.java index 9c5d3be26..d1782e81f 100644 --- a/src/test/java/de/komoot/photon/importer/elasticsearch/ImporterTest.java +++ b/src/test/java/de/komoot/photon/importer/elasticsearch/ImporterTest.java @@ -2,6 +2,7 @@ import de.komoot.photon.ESBaseTester; import de.komoot.photon.importer.model.PhotonDoc; +import java.io.IOException; import org.json.JSONObject; import org.junit.*; @@ -17,7 +18,7 @@ public class ImporterTest extends ESBaseTester { @Before - public void setUp() { + public void setUp() throws IOException { setUpES(); deleteAll(); } @@ -28,7 +29,7 @@ public void testAdd() { nameMap.put("name", "testing"); PhotonDoc doc = PhotonDoc.create(1, "way", 1, nameMap); - Importer instance = new Importer(getClient()); + Importer instance = new Importer(getClient(), "en"); // hardcoded lang instance.add(doc); instance.finish(); diff --git a/src/test/java/de/komoot/photon/importer/elasticsearch/RemoveStreetDuplicatesTest.java b/src/test/java/de/komoot/photon/importer/elasticsearch/RemoveStreetDuplicatesTest.java index 97cb05479..7166edcd1 100644 --- a/src/test/java/de/komoot/photon/importer/elasticsearch/RemoveStreetDuplicatesTest.java +++ b/src/test/java/de/komoot/photon/importer/elasticsearch/RemoveStreetDuplicatesTest.java @@ -6,6 +6,7 @@ import com.vividsolutions.jts.geom.PrecisionModel; import de.komoot.photon.ESBaseTester; import de.komoot.photon.importer.model.PhotonDoc; +import java.io.IOException; import org.json.JSONObject; import org.junit.*; @@ -20,14 +21,14 @@ public class RemoveStreetDuplicatesTest extends ESBaseTester { GeometryFactory FACTORY = new GeometryFactory(new PrecisionModel(), 4326); @Before - public void setUp() { + public void setUp() throws IOException { setUpES(); deleteAll(); PhotonDoc street1 = this.createStreetDoc(1, "Walserstraße", "6993"); PhotonDoc street2 = this.createStreetDoc(2, "Walserstraße", "6993"); PhotonDoc street3 = this.createStreetDoc(3, "Walserstraße", "6991"); - Importer instance = new Importer(getClient()); + Importer instance = new Importer(getClient(), "en"); // hardcoded lang instance.add(street1); instance.add(street2); instance.add(street3);