Merge pull request #824 from snipsco/release/0.19.8

Release 0.19.8

.appveyor.yml

@@ -11,6 +11,7 @@ branches:
   only:
   - develop
   - master
+  - /lts-.*/
 
 cache:
   # Cache downloaded pip packages and built wheels.

CHANGELOG.md

@@ -1,6 +1,18 @@
 # Changelog
 All notable changes to this project will be documented in this file.
 
+## [0.19.8]
+### Added
+- Add filter for entity match feature [#814](https://github.com/snipsco/snips-nlu/pull/814)
+- Add noise re-weight factor in `LogRegIntentClassifier` [#815](https://github.com/snipsco/snips-nlu/pull/815)
+- Add warning logs and improve errors [#821](https://github.com/snipsco/snips-nlu/pull/821)
+- Add random seed parameter in training CLI [#819](https://github.com/snipsco/snips-nlu/pull/819)
+
+### Fixed
+- Fix non-deterministic behavior [#817](https://github.com/snipsco/snips-nlu/pull/817)
+- Import modules lazily to speed up CLI startup time [#819](https://github.com/snipsco/snips-nlu/pull/819)
+- Removed dependency on `semantic_version` to accept `"subpatches"` number [#825](https://github.com/snipsco/snips-nlu/pull/825)
+
 ## [0.19.7]
 ### Changed
 - Re-score ambiguous `DeterministicIntentParser` results based on slots [#791](https://github.com/snipsco/snips-nlu/pull/791)
@@ -284,6 +296,7 @@ several commands.
 - Fix compiling issue with `bindgen` dependency when installing from source
 - Fix issue in `CRFSlotFiller` when handling builtin entities
 
+[0.19.8]: https://github.com/snipsco/snips-nlu/compare/0.19.7...0.19.8
 [0.19.7]: https://github.com/snipsco/snips-nlu/compare/0.19.6...0.19.7
 [0.19.6]: https://github.com/snipsco/snips-nlu/compare/0.19.5...0.19.6
 [0.19.5]: https://github.com/snipsco/snips-nlu/compare/0.19.4...0.19.5

setup.py

@@ -24,16 +24,15 @@
     "num2words>=0.5.6,<0.6",
     "numpy>=1.15,<2.0",
     "pathlib>=1.0,<2.0; python_version<'3.4'",
-    "plac>=0.9.6,<2.0",
     "pyaml>=17.0,<20.0",
     "requests>=2.0,<3.0",
     "scikit-learn>=0.20,<0.21; python_version<'3.5'",
     "scikit-learn>=0.21.1,<0.22; python_version>='3.5'",
     "scipy>=1.0,<2.0",
-    "semantic_version>=2.6,<3.0",
     "sklearn-crfsuite>=0.3.6,<0.4",
     "snips-nlu-parsers>=0.2,<0.3",
     "snips-nlu-utils>=0.8,<0.9",
+    "deprecation>=2,<3",
 ]
 
 extras_require = {
@@ -50,7 +49,8 @@
         "mock>=2.0,<3.0",
         "snips_nlu_metrics>=0.14.1,<0.15",
         "pylint<2",
-        "coverage>=4.4.2,<5.0"
+        "coverage>=4.4.2,<5.0",
+        "checksumdir~=1.1.6",
     ]
 }
 
@@ -84,7 +84,7 @@
       include_package_data=True,
       entry_points={
           "console_scripts": [
-              "snips-nlu=snips_nlu.__main__:main"
+              "snips-nlu=snips_nlu.cli:main"
           ]
       },
       zip_safe=False)

snips_nlu/__about__.py

@@ -13,7 +13,7 @@
 __email__ = "[email protected], [email protected]"
 __license__ = "Apache License, Version 2.0"
 
-__version__ = "0.19.7"
+__version__ = "0.19.8"
 __model_version__ = "0.19.0"
 
 __download_url__ = "https://github.com/snipsco/snips-nlu-language-resources/releases/download"

snips_nlu/__init__.py

@@ -1,8 +1,15 @@
-from snips_nlu_parsers import get_ontology_version
+from deprecation import deprecated
 
 from snips_nlu.__about__ import __model_version__, __version__
 from snips_nlu.nlu_engine import SnipsNLUEngine
 from snips_nlu.pipeline.configs import NLUEngineConfig
-from snips_nlu.resources import load_resources
 
-__builtin_entities_version__ = get_ontology_version()
+
+@deprecated(deprecated_in="0.19.7", removed_in="0.21.0",
+            current_version=__version__,
+            details="Loading resources in the client code is no longer "
+                    "required")
+def load_resources(name, required_resources=None):
+    from snips_nlu.resources import load_resources as _load_resources
+
+    return _load_resources(name, required_resources)

snips_nlu/__main__.py

@@ -1,47 +1,6 @@
 from __future__ import print_function, unicode_literals
 
 
-def main():
-    import sys
-
-    import plac
-
-    from snips_nlu.__about__ import __version__, __model_version__
-    from snips_nlu.cli import (
-        cross_val_metrics, download, download_all_languages, generate_dataset,
-        link, train_test_metrics)
-    from snips_nlu.cli.download_entity import (
-        download_builtin_entity, download_language_builtin_entities)
-    from snips_nlu.cli.inference import parse
-    from snips_nlu.cli.training import train
-    from snips_nlu.cli.utils import PrettyPrintLevel, pretty_print
-
-    commands = {
-        "train": train,
-        "parse": parse,
-        "download": download,
-        "download-all-languages": download_all_languages,
-        "download-entity": download_builtin_entity,
-        "download-language-entities": download_language_builtin_entities,
-        "version": lambda: print(__version__),
-        "model-version": lambda: print(__model_version__),
-        "link": link,
-        "generate-dataset": generate_dataset,
-        "cross-val-metrics": cross_val_metrics,
-        "train-test-metrics": train_test_metrics,
-    }
-    if len(sys.argv) == 1:
-        pretty_print(', '.join(commands), title="Available commands", exits=1,
-                     level=PrettyPrintLevel.INFO)
-    command = sys.argv.pop(1)
-    sys.argv[0] = 'snips-nlu %s' % command
-    if command in commands:
-        plac.call(commands[command], sys.argv[1:])
-    else:
-        pretty_print("Available: %s" % ', '.join(commands),
-                     title="Unknown command: %s" % command, exits=1,
-                     level=PrettyPrintLevel.INFO)
-
-
 if __name__ == "__main__":
+    from snips_nlu.cli import main
     main()

snips_nlu/cli/__init__.py

@@ -1,6 +1,53 @@
-from snips_nlu.cli.download import download, download_all_languages
-from snips_nlu.cli.generate_dataset import generate_dataset
-from snips_nlu.cli.inference import parse
-from snips_nlu.cli.link import link
-from snips_nlu.cli.metrics import train_test_metrics, cross_val_metrics
-from snips_nlu.cli.training import train
+import argparse
+
+
+class Formatter(argparse.HelpFormatter):
+    def __init__(self, prog):
+        super(Formatter, self).__init__(prog, max_help_position=35, width=150)
+
+
+def get_arg_parser():
+    from snips_nlu.cli.download import (
+        add_download_parser, add_download_all_languages_parser)
+    from snips_nlu.cli.download_entity import (
+        add_download_entity_parser, add_download_language_entities_parser)
+    from snips_nlu.cli.generate_dataset import add_generate_dataset_subparser
+    from snips_nlu.cli.inference import add_parse_parser
+    from snips_nlu.cli.link import add_link_parser
+    from snips_nlu.cli.metrics import (
+        add_cross_val_metrics_parser, add_train_test_metrics_parser)
+    from snips_nlu.cli.training import add_train_parser
+    from snips_nlu.cli.versions import (
+        add_version_parser, add_model_version_parser)
+
+    arg_parser = argparse.ArgumentParser(
+        description="Snips NLU command line interface",
+        prog="python -m snips_nlu", formatter_class=Formatter)
+    arg_parser.add_argument("-v", "--version", action="store_true",
+                            help="Print package version")
+    subparsers = arg_parser.add_subparsers(
+        title="available commands", metavar="command [options ...]")
+    add_generate_dataset_subparser(subparsers)
+    add_train_parser(subparsers)
+    add_parse_parser(subparsers)
+    add_download_parser(subparsers)
+    add_download_all_languages_parser(subparsers)
+    add_download_entity_parser(subparsers)
+    add_download_language_entities_parser(subparsers)
+    add_link_parser(subparsers)
+    add_cross_val_metrics_parser(subparsers)
+    add_train_test_metrics_parser(subparsers)
+    add_version_parser(subparsers)
+    add_model_version_parser(subparsers)
+    return arg_parser
+
+
+def main():
+    arg_parser = get_arg_parser()
+    args = arg_parser.parse_args()
+
+    if hasattr(args, "func"):
+        args.func(args)
+    else:
+        arg_parser.print_help()
+        exit(1)

snips_nlu/cli/download.py

@@ -1,32 +1,43 @@
 from __future__ import print_function, unicode_literals
 
-import sys
 
-import plac
-from snips_nlu_parsers import get_all_languages
+def add_download_parser(subparsers):
+    subparser = subparsers.add_parser(
+        "download", help="Download the language resources required by the "
+                         "snips-nlu library")
+    subparser.add_argument(
+        "resource_name", type=str,
+        help="Name of the language resources to download. Can be either "
+             "a shortcut, like 'en', or the full name of the resources "
+             "like 'snips_nlu_en'")
+    subparser.add_argument(
+        "-d", "--direct", action="store_true",
+        help="Force direct download. Needs resource name with version and "
+             "won't perform compatibility check")
+    subparser.add_argument(
+        "extra_pip_args", nargs="*", type=str,
+        help="Additional arguments to be passed to `pip install` when "
+             "installing the language resources package")
+    subparser.set_defaults(func=_download)
+    return subparser
 
-from snips_nlu import __about__
-from snips_nlu.cli.link import link_resources
-from snips_nlu.cli.utils import (
-    PrettyPrintLevel, check_resources_alias, get_compatibility, get_json,
-    get_resources_version, install_remote_package, pretty_print)
-from snips_nlu.common.utils import get_package_path
+
+def _download(args_namespace):
+    return download(args_namespace.resource_name, args_namespace.direct,
+                    *args_namespace.extra_pip_args)
 
 
 # inspired from
 # https://github.com/explosion/spaCy/blob/master/spacy/cli/download.py
 
-@plac.annotations(
-    resource_name=("Name of the language resources to download. Can be either "
-                   "a shortcut, like 'en', or the full name of the resources "
-                   "like 'snips_nlu_en'", "positional", None, str),
-    direct=("force direct download. Needs resource name with version and "
-            "won't perform compatibility check", "flag", "d", bool),
-    pip_args=("Additional arguments to be passed to `pip install` when "
-              "installing the language resources package"))
 def download(resource_name, direct=False,
              *pip_args):  # pylint:disable=keyword-arg-before-vararg
+
     """Download compatible language resources"""
+    import sys
+    from snips_nlu import __about__
+    from snips_nlu.cli.utils import install_remote_package
+
     if direct:
         components = resource_name.split("-")
         name = "".join(components[:-1])
@@ -42,6 +53,10 @@ def download(resource_name, direct=False,
 
 
 def download_from_resource_name(resource_name, pip_args, verbose=True):
+    from snips_nlu import __about__
+    from snips_nlu.cli.utils import (
+        check_resources_alias, get_compatibility, get_json)
+
     shortcuts = get_json(__about__.__shortcuts__, "Resource shortcuts")
     check_resources_alias(resource_name, shortcuts)
     compatibility = get_compatibility()
@@ -51,17 +66,40 @@ def download_from_resource_name(resource_name, pip_args, verbose=True):
                        pip_args, verbose)
 
 
-@plac.annotations(
-    pip_args=("Additional arguments to be passed to `pip install` when "
-              "installing the resources"))
+def add_download_all_languages_parser(subparsers):
+    subparser = subparsers.add_parser(
+        "download-all-languages",
+        help="Download language resources for all languages")
+    subparser.add_argument(
+        "extra_pip_args", nargs="*", type=str,
+        help="Additional arguments to be passed to `pip install` when "
+             "installing the language resources packages")
+    subparser.set_defaults(func=_download_all_languages)
+    return subparser
+
+
+def _download_all_languages(args_namespace):
+    return download_all_languages(*args_namespace.extra_pip_args)
+
+
 def download_all_languages(*pip_args):
     """Download compatible resources for all supported languages"""
+    from snips_nlu_parsers import get_all_languages
+
     for language in get_all_languages():
         download(language, False, *pip_args)
 
 
 def _download_and_link(resource_alias, resource_fullname, compatibility,
                        pip_args, verbose):
+    import sys
+    from snips_nlu import __about__
+    from snips_nlu.cli.link import link_resources
+    from snips_nlu.cli.utils import (
+        PrettyPrintLevel, get_resources_version, install_remote_package,
+        pretty_print)
+    from snips_nlu.common.utils import get_package_path
+
     version = get_resources_version(resource_fullname, resource_alias,
                                     compatibility)
     url_tail = '{r}-{v}/{r}-{v}.tar.gz#egg={r}=={v}'.format(