Skip to content

Commit

Permalink
Merge pull request #125 from Svantulden/master
Browse files Browse the repository at this point in the history 
[Not finished, discussion] Making languages dynamic with command arguments
  • Loading branch information
@christophlingg
christophlingg committed Dec 20, 2014
2 parents bd70b45 + c4726f2 commit e1ab765
Show file tree
Hide file tree
Showing 14 changed files with 171 additions and 282 deletions.
4 changes: 4 additions & 0 deletions .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,3 +7,7 @@ dependency-reduced-pom.xml
*.~*

__pycache__

nb-configuration.xml

nbactions.xml
227 changes: 1 addition & 226 deletions es/mappings.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,119 +47,34 @@
},
"city": {
"properties": {
"de": {
"type": "string",
"index": "no",
"copy_to": [
"collector.de"
]
},
"default": {
"type": "string",
"index": "no",
"copy_to": [
"collector.default"
]
},
"en": {
"type": "string",
"index": "no",
"copy_to": [
"collector.en"
]
},
"fr": {
"type": "string",
"index": "no",
"copy_to": [
"collector.fr"
]
},
"it": {
"type": "string",
"index": "no",
"copy_to": [
"collector.it"
]
}
}
},
"context": {
"properties": {
"de": {
"type": "string",
"index": "no",
"copy_to": [
"collector.default",
"collector.de"
]
},
"default": {
"type": "string",
"index": "no",
"copy_to": [
"collector.default"
]
},
"en": {
"type": "string",
"index": "no",
"copy_to": [
"collector.en"
]
},
"fr": {
"type": "string",
"index": "no",
"copy_to": [
"collector.fr"
]
},
"it": {
"type": "string",
"index": "no",
"copy_to": [
"collector.it"
]
}
}
},
"country": {
"properties": {
"de": {
"type": "string",
"index": "no",
"copy_to": [
"collector.de"
]
},
"default": {
"type": "string",
"index": "no",
"copy_to": [
"collector.default"
]
},
"en": {
"type": "string",
"index": "no",
"copy_to": [
"collector.en"
]
},
"fr": {
"type": "string",
"index": "no",
"copy_to": [
"collector.fr"
]
},
"it": {
"type": "string",
"index": "no",
"copy_to": [
"collector.it"
]
}
}
},
Expand Down Expand Up @@ -218,83 +133,11 @@
"collector.default"
]
},
"de": {
"type": "string",
"index": "no",
"fields": {
"ngrams": {
"type": "string",
"index_analyzer": "index_ngram"
},
"raw": {
"type": "string",
"index_analyzer": "index_raw"
}
},
"copy_to": [
"collector.de"
]
},
"default": {
"type": "string",
"index": "no",
"copy_to": [
"collector.default",
"name.en",
"name.de",
"name.fr",
"name.it"
]
},
"en": {
"type": "string",
"index": "no",
"fields": {
"ngrams": {
"type": "string",
"index_analyzer": "index_ngram"
},
"raw": {
"type": "string",
"index_analyzer": "index_raw"
}
},
"copy_to": [
"collector.en"
]
},
"fr": {
"type": "string",
"index": "no",
"fields": {
"ngrams": {
"type": "string",
"index_analyzer": "index_ngram"
},
"raw": {
"type": "string",
"index_analyzer": "index_raw"
}
},
"copy_to": [
"collector.fr"
]
},
"it": {
"type": "string",
"index": "no",
"fields": {
"ngrams": {
"type": "string",
"index_analyzer": "index_ngram"
},
"raw": {
"type": "string",
"index_analyzer": "index_raw"
}
},
"copy_to": [
"collector.it"
"collector.default"
]
}
}
Expand All @@ -308,34 +151,6 @@
"copy_to": [
"collector.default"
]
},
"de": {
"index": "no",
"type": "string",
"copy_to": [
"collector.de"
]
},
"en": {
"index": "no",
"type": "string",
"copy_to": [
"collector.en"
]
},
"fr": {
"index": "no",
"type": "string",
"copy_to": [
"collector.fr"
]
},
"it": {
"index": "no",
"type": "string",
"copy_to": [
"collector.it"
]
}
}
},
Expand All @@ -351,46 +166,6 @@
"analyzer": "index_raw"
}
}
},
"de": {
"type": "string",
"analyzer": "index_ngram",
"fields": {
"raw": {
"type": "string",
"analyzer": "index_raw"
}
}
},
"en": {
"type": "string",
"analyzer": "index_ngram",
"fields": {
"raw": {
"type": "string",
"analyzer": "index_raw"
}
}
},
"fr": {
"type": "string",
"analyzer": "index_ngram",
"fields": {
"raw": {
"type": "string",
"analyzer": "index_raw"
}
}
},
"it": {
"type": "string",
"analyzer": "index_ngram",
"fields": {
"raw": {
"type": "string",
"analyzer": "index_raw"
}
}
}
}
}
Expand Down
41 changes: 30 additions & 11 deletions src/main/java/de/komoot/photon/importer/App.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
import spark.Route;

import java.io.FileNotFoundException;
import java.io.IOException;

import static spark.Spark.*;

Expand All @@ -41,7 +42,7 @@ public static void main(String[] rawArgs) {
if(args.getJsonDump() != null) {
try {
final String filename = args.getJsonDump();
final JsonDumper jsonDumper = new JsonDumper(filename);
final JsonDumper jsonDumper = new JsonDumper(filename, args.getLanguages());
NominatimConnector nominatimConnector = new NominatimConnector(args.getHost(), args.getPort(), args.getDatabase(), args.getUser(), args.getPassword());
nominatimConnector.setImporter(jsonDumper);
nominatimConnector.readEntireDatabase();
Expand All @@ -52,29 +53,47 @@ public static void main(String[] rawArgs) {
}
}

final Server esServer = new Server(args.getCluster(), args.getDataDirectory());
final Server esServer = new Server(args.getCluster(), args.getDataDirectory(), args.getLanguages());
esServer.start();

Client esNodeClient = esServer.getClient();

if(args.isDeleteIndex()) {
esServer.recreateIndex();
if(args.isDeleteIndex()) {
try {
esServer.recreateIndex();
} catch (IOException e) {
log.error("cannot setup index, elastic search config files not readable", e);
return;
}

log.info("deleted photon index and created an empty new one.");
return;
}

if(args.isNominatimImport()) {
esServer.recreateIndex(); // dump previous data
Importer importer = new Importer(esNodeClient);
try {
esServer.recreateIndex(); // dump previous data
} catch (IOException e) {
log.error("cannot setup index, elastic search config files not readable", e);
return;
}

log.info("starting import from nominatim to photon with languages: " + args.getLanguages());
Importer importer = new Importer(esNodeClient, args.getLanguages());
NominatimConnector nominatimConnector = new NominatimConnector(args.getHost(), args.getPort(), args.getDatabase(), args.getUser(), args.getPassword());
nominatimConnector.setImporter(importer);
nominatimConnector.readEntireDatabase();
log.info("imported data from nominatim to photon.");
try {
nominatimConnector.readEntireDatabase();
} catch (Exception e) {
log.info("ERROR IMPORTING FROM NOMINATIM: "+e.getMessage());
}

log.info("imported data from nominatim to photon with languages: " + args.getLanguages());
return;
}

final NominatimUpdater nominatimUpdater = new NominatimUpdater(args.getHost(), args.getPort(), args.getDatabase(), args.getUser(), args.getPassword());
de.komoot.photon.importer.Updater updater = new de.komoot.photon.importer.elasticsearch.Updater(esNodeClient);
de.komoot.photon.importer.Updater updater = new de.komoot.photon.importer.elasticsearch.Updater(esNodeClient, args.getLanguages());
nominatimUpdater.setUpdater(updater);

startApi(args, esNodeClient, nominatimUpdater);
Expand All @@ -99,7 +118,7 @@ public void run() {
});

final Searcher searcher = new Searcher(esNodeClient);
get(new RequestHandler("api", searcher));
get(new RequestHandler("api/", searcher));
get(new RequestHandler("api", searcher, args.getLanguages()));
get(new RequestHandler("api/", searcher, args.getLanguages()));
}
}
3 changes: 3 additions & 0 deletions src/main/java/de/komoot/photon/importer/CommandLineArgs.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,9 @@ public class CommandLineArgs {

@Parameter(names = "-nominatim-import", description = "import nominatim database into photon (this will delete previous index)")
private boolean nominatimImport = false;

@Parameter(names = "-languages", description = "languages nominatim importer should import and use at run-time, comma seperated (default: 'en,fr,de,it')")
private String languages = "en,fr,de,it";

@Parameter(names = "-json", description = "import nominatim database and dump it to a json like files in (useful for developing)")
private String jsonDump = null;
Expand Down
Loading

0 comments on commit e1ab765

Please sign in to comment.