From 1b4284cd2fc65afddc27f9d51f5a987683d196d0 Mon Sep 17 00:00:00 2001 From: ClemDoum Date: Tue, 16 Oct 2018 15:13:05 +0200 Subject: [PATCH 1/7] Add a CLI to download all builtin entity resources by language --- snips_nlu/__main__.py | 4 +++- snips_nlu/cli/download_entity.py | 28 +++++++++++++++++++++++++++- snips_nlu/cli/link.py | 2 +- 3 files changed, 31 insertions(+), 3 deletions(-) diff --git a/snips_nlu/__main__.py b/snips_nlu/__main__.py index 9b88e5bbd..b398ae083 100644 --- a/snips_nlu/__main__.py +++ b/snips_nlu/__main__.py @@ -7,7 +7,8 @@ from snips_nlu.cli import ( cross_val_metrics, download, download_all_languages, generate_dataset, link, train_test_metrics) -from snips_nlu.cli.download_entity import download_builtin_entity +from snips_nlu.cli.download_entity import ( + download_builtin_entity, download_language_builtin_entities) from snips_nlu.cli.inference import parse from snips_nlu.cli.training import train from snips_nlu.cli.utils import PrettyPrintLevel, pretty_print @@ -20,6 +21,7 @@ def main(): "download": download, "download-all-languages": download_all_languages, "download-entity": download_builtin_entity, + "download-language-entities": download_language_builtin_entities, "link": link, "generate-dataset": generate_dataset, "cross-val-metrics": cross_val_metrics, diff --git a/snips_nlu/cli/download_entity.py b/snips_nlu/cli/download_entity.py index f69627b61..5486b66fe 100644 --- a/snips_nlu/cli/download_entity.py +++ b/snips_nlu/cli/download_entity.py @@ -3,7 +3,10 @@ import sys import plac -from snips_nlu_ontology import get_builtin_entity_shortname + +from future.builtins import str +from snips_nlu_ontology import ( + get_builtin_entity_shortname, get_supported_gazetteer_entities) from snips_nlu import __about__ from snips_nlu.cli.download import download_from_resource_name @@ -40,6 +43,29 @@ def download_builtin_entity(entity_name, language, *pip_args): long_resource_name, entity_name, language, compatibility, pip_args) +@plac.annotations( + language=("Language of the builtin entity", "positional", None, str), + pip_args=("Additional arguments to be passed to `pip install` when " + "installing the builtin entity package")) +# pylint: disable=keyword-arg-before-vararg +def download_language_builtin_entities(language, *pip_args): + """Download all gazetteer entity resources for a given language as well as + basic language resources for this language""" + download_from_resource_name(language, pip_args, verbose=False) + + shortcuts = get_json(__about__.__shortcuts__, "Resource shortcuts") + for entity_name in get_supported_gazetteer_entities(language): + check_resources_alias(entity_name, shortcuts) + + compatibility = get_compatibility() + resource_name_lower = entity_name.lower() + long_resource_name = shortcuts.get(resource_name_lower, + resource_name_lower) + + _download_and_link_entity( + long_resource_name, entity_name, language, compatibility, pip_args) + + # pylint: enable=keyword-arg-before-vararg diff --git a/snips_nlu/cli/link.py b/snips_nlu/cli/link.py index 19c1601db..19b44f73c 100644 --- a/snips_nlu/cli/link.py +++ b/snips_nlu/cli/link.py @@ -41,7 +41,7 @@ def link_resources(origin, link_name, force, resources_path): else Path(resources_path) if not resources_path.exists(): raise OSError("%s not found" % str(resources_path)) - link_path = DATA_PATH / link_name + link_path = DATA_PATH / str(link_name) if link_path.is_symlink() and not force: raise OSError("Symlink already exists: %s" % str(link_path)) elif link_path.is_symlink(): From ff9849dccbb8295fa60fe441b8e792bed42ac6cf Mon Sep 17 00:00:00 2001 From: ClemDoum Date: Fri, 12 Oct 2018 10:33:55 +0200 Subject: [PATCH 2/7] Bump snips-nlu-ontology to 0.61.1 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 46caa545a..3c0ff3d39 100644 --- a/setup.py +++ b/setup.py @@ -25,7 +25,7 @@ "sklearn-crfsuite>=0.3.6,<0.4", "semantic_version>=2.6,<3.0", "snips_nlu_utils>=0.7,<0.8", - "snips_nlu_ontology>=0.60,<0.61", + "snips_nlu_ontology>=0.61.1,<0.62", "num2words>=0.5.6,<0.6", "plac>=0.9.6,<1.0", "requests>=2.0,<3.0", From 9200c60e613466b8044c41fb6f405ae6b17c557e Mon Sep 17 00:00:00 2001 From: Adrien Ball Date: Fri, 12 Oct 2018 15:48:45 +0200 Subject: [PATCH 3/7] Bump version to 0.17.2 --- snips_nlu/__about__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/snips_nlu/__about__.py b/snips_nlu/__about__.py index 2622c9afa..f91f43ef2 100644 --- a/snips_nlu/__about__.py +++ b/snips_nlu/__about__.py @@ -11,7 +11,7 @@ __email__ = "clement.doumouro@snips.ai, adrien.ball@snips.ai" __license__ = "Apache License, Version 2.0" -__version__ = "0.17.1" +__version__ = "0.17.2" __model_version__ = "0.17.0" __download_url__ = "https://github.com/snipsco/snips-nlu-language-resources/releases/download" From 6849264ef9341a616cf9a2b03da379afcfbcdaf8 Mon Sep 17 00:00:00 2001 From: Adrien Ball Date: Mon, 15 Oct 2018 19:10:53 +0200 Subject: [PATCH 4/7] Update Changelog --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 064f07f92..9781d2f04 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,10 @@ # Changelog All notable changes to this project will be documented in this file. +## [0.17.2] - 2018-10-15 +### Added +- Support for builtin music entities in english + ## [0.17.1] - 2018-10-09 ### Fixed - `DeterministicIntentParser` now relies on the custom entity parser @@ -157,6 +161,7 @@ several commands. - Fix compiling issue with `bindgen` dependency when installing from source - Fix issue in `CRFSlotFiller` when handling builtin entities +[0.17.2]: https://github.com/snipsco/snips-nlu/compare/0.17.1...0.17.2 [0.17.1]: https://github.com/snipsco/snips-nlu/compare/0.17.0...0.17.1 [0.17.0]: https://github.com/snipsco/snips-nlu/compare/0.16.5...0.17.0 [0.16.5]: https://github.com/snipsco/snips-nlu/compare/0.16.4...0.16.5 From 481eb5ab952cb7045a3eff06d0b0f4dfaab2818d Mon Sep 17 00:00:00 2001 From: Adrien Ball Date: Wed, 17 Oct 2018 11:00:15 +0200 Subject: [PATCH 5/7] Fix CI --- .appveyor.yml | 5 ++--- tox.ini | 5 ++--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index e86538588..324888c15 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -24,8 +24,7 @@ build: false test_script: - "%PYTHON%\\python.exe -m snips_nlu download-all-languages" - - "%PYTHON%\\python.exe -m snips_nlu download-entity snips/musicAlbum fr" - - "%PYTHON%\\python.exe -m snips_nlu download-entity snips/musicArtist fr" - - "%PYTHON%\\python.exe -m snips_nlu download-entity snips/musicTrack fr" + - "%PYTHON%\\python.exe -m snips_nlu download-language-entities fr" + - "%PYTHON%\\python.exe -m snips_nlu download-language-entities en" - "%PYTHON%\\python.exe -m unittest discover" - "%PYTHON%\\python.exe snips_nlu_samples/sample.py" diff --git a/tox.ini b/tox.ini index d8546336d..b4ac91e1a 100644 --- a/tox.ini +++ b/tox.ini @@ -6,9 +6,8 @@ skip_install = true commands = pip install -e ".[test]" snips-nlu download-all-languages - snips-nlu download-entity snips/musicAlbum fr - snips-nlu download-entity snips/musicArtist fr - snips-nlu download-entity snips/musicTrack fr + snips-nlu download-language-entities fr + snips-nlu download-language-entities en coverage run -m unittest discover python snips_nlu_samples/sample.py setenv= From 51e60915708c7331cb5330eb5a6c65d08fe6b7a8 Mon Sep 17 00:00:00 2001 From: Adrien Ball Date: Wed, 17 Oct 2018 11:11:33 +0200 Subject: [PATCH 6/7] Fix test --- snips_nlu/tests/test_builtin_entity_parser.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/snips_nlu/tests/test_builtin_entity_parser.py b/snips_nlu/tests/test_builtin_entity_parser.py index 3ad0d5695..ed98902ce 100644 --- a/snips_nlu/tests/test_builtin_entity_parser.py +++ b/snips_nlu/tests/test_builtin_entity_parser.py @@ -19,9 +19,7 @@ def test_should_parse_grammar_entities(self): # Given text = "we'll be 2 at the meeting" language = "en" - parser = BuiltinEntityParser.build( - language=language, - gazetteer_entity_scope=get_supported_gazetteer_entities(language)) + parser = BuiltinEntityParser.build(language=language) # When / Then parse = parser.parse(text) From c40fb24335f04938d42cb94032cd58e5f06fa20e Mon Sep 17 00:00:00 2001 From: Adrien Ball Date: Wed, 17 Oct 2018 11:17:21 +0200 Subject: [PATCH 7/7] Fix linting --- snips_nlu/tests/test_builtin_entity_parser.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/snips_nlu/tests/test_builtin_entity_parser.py b/snips_nlu/tests/test_builtin_entity_parser.py index ed98902ce..f24525111 100644 --- a/snips_nlu/tests/test_builtin_entity_parser.py +++ b/snips_nlu/tests/test_builtin_entity_parser.py @@ -2,8 +2,7 @@ from __future__ import unicode_literals from mock import patch -from snips_nlu_ontology import ( - get_all_languages, get_supported_gazetteer_entities) +from snips_nlu_ontology import get_all_languages from snips_nlu.constants import ENTITIES, ENTITY_KIND, LANGUAGE from snips_nlu.entity_parser.builtin_entity_parser import (