Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Making languages dynamic with command arguments #125

Merged
merged 24 commits into from
Dec 20, 2014
Merged

Making languages dynamic with command arguments #125

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
24 commits
Select commit Hold shift + click to select a range
00933ac
Added dynamic languages via command
Dec 8, 2014
68ae44f
Fix invalid json
Dec 8, 2014
0621d01
Seperated language commands for import/running
Dec 8, 2014
35bcfde
Added dutch (nl) to mappings.json
Dec 9, 2014
e9fda4b
Added stripNonDigits util
Dec 10, 2014
dacb243
Fixed ImporterTest to use (static) languages
Dec 10, 2014
84a37c2
Added postcode check for nl postcodes
Dec 10, 2014
a7ac5eb
Fixed tests to use new lang scheme
Dec 10, 2014
0636de8
Added Netbeans xml to .gitignore
Dec 10, 2014
b4c5d42
Reverted string lat/lon back to doubles
Dec 15, 2014
b7e6ff3
Deleted language specific obj from mappings.json
Dec 15, 2014
fee73da
Use mappings template & add lang specific strings
Dec 15, 2014
41ee0d1
Removed spaces
Dec 15, 2014
d81292a
Unified commandline param to 'using-languages'
Dec 15, 2014
756aeb2
Modified test with static lang
Dec 15, 2014
f3332f7
Reverted NL postcode check
Dec 15, 2014
0b7f608
Removed commented code
Dec 16, 2014
e4c6a6e
Renamed commandline arg to just '-languages'
Dec 16, 2014
0c82ecb
Refactored mappings.json code
Dec 16, 2014
d586d52
Some refactoring
Dec 16, 2014
80116a9
Deleted spaces
Dec 19, 2014
8b67963
Fixes some consistency issues
Dec 19, 2014
1a50011
Reworked exception handling for recreateIndex()
Dec 19, 2014
c4726f2
Removed unneeded null check
Dec 19, 2014
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions .gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,3 +7,7 @@ dependency-reduced-pom.xml
*.~*

__pycache__

nb-configuration.xml

nbactions.xml
227 changes: 1 addition & 226 deletions es/mappings.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,119 +47,34 @@
},
"city": {
"properties": {
"de": {
"type": "string",
"index": "no",
"copy_to": [
"collector.de"
]
},
"default": {
"type": "string",
"index": "no",
"copy_to": [
"collector.default"
]
},
"en": {
"type": "string",
"index": "no",
"copy_to": [
"collector.en"
]
},
"fr": {
"type": "string",
"index": "no",
"copy_to": [
"collector.fr"
]
},
"it": {
"type": "string",
"index": "no",
"copy_to": [
"collector.it"
]
}
}
},
"context": {
"properties": {
"de": {
"type": "string",
"index": "no",
"copy_to": [
"collector.default",
"collector.de"
]
},
"default": {
"type": "string",
"index": "no",
"copy_to": [
"collector.default"
]
},
"en": {
"type": "string",
"index": "no",
"copy_to": [
"collector.en"
]
},
"fr": {
"type": "string",
"index": "no",
"copy_to": [
"collector.fr"
]
},
"it": {
"type": "string",
"index": "no",
"copy_to": [
"collector.it"
]
}
}
},
"country": {
"properties": {
"de": {
"type": "string",
"index": "no",
"copy_to": [
"collector.de"
]
},
"default": {
"type": "string",
"index": "no",
"copy_to": [
"collector.default"
]
},
"en": {
"type": "string",
"index": "no",
"copy_to": [
"collector.en"
]
},
"fr": {
"type": "string",
"index": "no",
"copy_to": [
"collector.fr"
]
},
"it": {
"type": "string",
"index": "no",
"copy_to": [
"collector.it"
]
}
}
},
Expand Down Expand Up @@ -218,83 +133,11 @@
"collector.default"
]
},
"de": {
"type": "string",
"index": "no",
"fields": {
"ngrams": {
"type": "string",
"index_analyzer": "index_ngram"
},
"raw": {
"type": "string",
"index_analyzer": "index_raw"
}
},
"copy_to": [
"collector.de"
]
},
"default": {
"type": "string",
"index": "no",
"copy_to": [
"collector.default",
"name.en",
"name.de",
"name.fr",
"name.it"
]
},
"en": {
"type": "string",
"index": "no",
"fields": {
"ngrams": {
"type": "string",
"index_analyzer": "index_ngram"
},
"raw": {
"type": "string",
"index_analyzer": "index_raw"
}
},
"copy_to": [
"collector.en"
]
},
"fr": {
"type": "string",
"index": "no",
"fields": {
"ngrams": {
"type": "string",
"index_analyzer": "index_ngram"
},
"raw": {
"type": "string",
"index_analyzer": "index_raw"
}
},
"copy_to": [
"collector.fr"
]
},
"it": {
"type": "string",
"index": "no",
"fields": {
"ngrams": {
"type": "string",
"index_analyzer": "index_ngram"
},
"raw": {
"type": "string",
"index_analyzer": "index_raw"
}
},
"copy_to": [
"collector.it"
"collector.default"
]
}
}
Expand All @@ -308,34 +151,6 @@
"copy_to": [
"collector.default"
]
},
"de": {
"index": "no",
"type": "string",
"copy_to": [
"collector.de"
]
},
"en": {
"index": "no",
"type": "string",
"copy_to": [
"collector.en"
]
},
"fr": {
"index": "no",
"type": "string",
"copy_to": [
"collector.fr"
]
},
"it": {
"index": "no",
"type": "string",
"copy_to": [
"collector.it"
]
}
}
},
Expand All @@ -351,46 +166,6 @@
"analyzer": "index_raw"
}
}
},
"de": {
"type": "string",
"analyzer": "index_ngram",
"fields": {
"raw": {
"type": "string",
"analyzer": "index_raw"
}
}
},
"en": {
"type": "string",
"analyzer": "index_ngram",
"fields": {
"raw": {
"type": "string",
"analyzer": "index_raw"
}
}
},
"fr": {
"type": "string",
"analyzer": "index_ngram",
"fields": {
"raw": {
"type": "string",
"analyzer": "index_raw"
}
}
},
"it": {
"type": "string",
"analyzer": "index_ngram",
"fields": {
"raw": {
"type": "string",
"analyzer": "index_raw"
}
}
}
}
}
Expand Down
41 changes: 30 additions & 11 deletions src/main/java/de/komoot/photon/importer/App.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
import spark.Route;

import java.io.FileNotFoundException;
import java.io.IOException;

import static spark.Spark.*;

Expand All @@ -41,7 +42,7 @@ public static void main(String[] rawArgs) {
if(args.getJsonDump() != null) {
try {
final String filename = args.getJsonDump();
final JsonDumper jsonDumper = new JsonDumper(filename);
final JsonDumper jsonDumper = new JsonDumper(filename, args.getLanguages());
NominatimConnector nominatimConnector = new NominatimConnector(args.getHost(), args.getPort(), args.getDatabase(), args.getUser(), args.getPassword());
nominatimConnector.setImporter(jsonDumper);
nominatimConnector.readEntireDatabase();
Expand All @@ -52,29 +53,47 @@ public static void main(String[] rawArgs) {
}
}

final Server esServer = new Server(args.getCluster(), args.getDataDirectory());
final Server esServer = new Server(args.getCluster(), args.getDataDirectory(), args.getLanguages());
esServer.start();

Client esNodeClient = esServer.getClient();

if(args.isDeleteIndex()) {
esServer.recreateIndex();
if(args.isDeleteIndex()) {
try {
esServer.recreateIndex();
} catch (IOException e) {
log.error("cannot setup index, elastic search config files not readable", e);
return;
}

log.info("deleted photon index and created an empty new one.");
return;
}

if(args.isNominatimImport()) {
esServer.recreateIndex(); // dump previous data
Importer importer = new Importer(esNodeClient);
try {
esServer.recreateIndex(); // dump previous data
} catch (IOException e) {
log.error("cannot setup index, elastic search config files not readable", e);
return;
}

log.info("starting import from nominatim to photon with languages: " + args.getLanguages());
Importer importer = new Importer(esNodeClient, args.getLanguages());
NominatimConnector nominatimConnector = new NominatimConnector(args.getHost(), args.getPort(), args.getDatabase(), args.getUser(), args.getPassword());
nominatimConnector.setImporter(importer);
nominatimConnector.readEntireDatabase();
log.info("imported data from nominatim to photon.");
try {
nominatimConnector.readEntireDatabase();
} catch (Exception e) {
log.info("ERROR IMPORTING FROM NOMINATIM: "+e.getMessage());
}

log.info("imported data from nominatim to photon with languages: " + args.getLanguages());
return;
}

final NominatimUpdater nominatimUpdater = new NominatimUpdater(args.getHost(), args.getPort(), args.getDatabase(), args.getUser(), args.getPassword());
de.komoot.photon.importer.Updater updater = new de.komoot.photon.importer.elasticsearch.Updater(esNodeClient);
de.komoot.photon.importer.Updater updater = new de.komoot.photon.importer.elasticsearch.Updater(esNodeClient, args.getLanguages());
nominatimUpdater.setUpdater(updater);

startApi(args, esNodeClient, nominatimUpdater);
Expand All @@ -99,7 +118,7 @@ public void run() {
});

final Searcher searcher = new Searcher(esNodeClient);
get(new RequestHandler("api", searcher));
get(new RequestHandler("api/", searcher));
get(new RequestHandler("api", searcher, args.getLanguages()));
get(new RequestHandler("api/", searcher, args.getLanguages()));
}
}
3 changes: 3 additions & 0 deletions src/main/java/de/komoot/photon/importer/CommandLineArgs.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,9 @@ public class CommandLineArgs {

@Parameter(names = "-nominatim-import", description = "import nominatim database into photon (this will delete previous index)")
private boolean nominatimImport = false;

@Parameter(names = "-languages", description = "languages nominatim importer should import and use at run-time, comma seperated (default: 'en,fr,de,it')")
private String languages = "en,fr,de,it";

@Parameter(names = "-json", description = "import nominatim database and dump it to a json like files in (useful for developing)")
private String jsonDump = null;
Expand Down
Loading