Merge branch 'develop'

Author: johncordeiro

README.md

@@ -1,6 +1,6 @@
 # Bothub NLP - Natural Language Processing services
 
-![version 2.1.1](https://img.shields.io/badge/version-2.1.1-blue.svg) [![python 3.6](https://img.shields.io/badge/python-3.6-green.svg)](https://docs.python.org/3.6/whatsnew/changelog.html) [![license AGPL-3.0](https://img.shields.io/badge/license-AGPL--3.0-red.svg)](https://github.com/udomobi/bothub-nlp/blob/master/LICENSE)
+![version 2.2.0](https://img.shields.io/badge/version-2.2.0-blue.svg) [![python 3.6](https://img.shields.io/badge/python-3.6-green.svg)](https://docs.python.org/3.6/whatsnew/changelog.html) [![license AGPL-3.0](https://img.shields.io/badge/license-AGPL--3.0-red.svg)](https://github.com/udomobi/bothub-nlp/blob/master/LICENSE)
 
 Check the [main Bothub project repository](https://github.com/Ilhasoft/bothub).
 

bothub-nlp-api/Pipfile

@@ -10,7 +10,7 @@ lint = "flake8"
 [packages]
 tornado = "==5.1.1"
 raven = "*"
-bothub-engine = {ref = "1.19.1", git = "https://github.com/Ilhasoft/bothub-engine"}
+bothub-engine = {ref = "1.21.0", git = "https://github.com/Ilhasoft/bothub-engine"}
 bothub-nlp-celery = {path = "./../bothub-nlp-celery"}
 python-decouple = "*"
 bothub-nlp = {path = "./../bothub-nlp"}

bothub-nlp-api/Pipfile.lock

@@ -11,6 +11,7 @@
 from .handlers.parse import ParseHandler
 from .handlers.train import TrainHandler
 from .handlers.info import InfoHandler
+from .handlers.evaluate import EvaluateHandler
 
 
 app = None
@@ -24,6 +25,7 @@ def make_app():
         url('/parse/', ParseHandler),
         url('/train/', TrainHandler),
         url('/info/', InfoHandler),
+        url('/evaluate/', EvaluateHandler)
     ])
 
 

bothub-nlp-api/bothub_nlp_api/app.py

@@ -0,0 +1,73 @@
+import tornado.web
+from tornado import gen
+
+from bothub_nlp_celery.actions import ACTION_EVALUATE, queue_name
+from bothub_nlp_celery.tasks import TASK_NLU_EVALUATE_UPDATE
+from bothub_nlp_celery.app import celery_app
+from bothub_nlp import settings as bothub_nlp_settings
+
+from . import ApiHandler
+from ..utils import ValidationError
+from ..utils import authorization_required
+from ..utils import AuthorizationIsRequired
+from ..utils import NEXT_LANGS
+
+
+EVALUATE_STATUS_EVALUATED = 'evaluated'
+EVALUATE_STATUS_FAILED = 'failed'
+
+
+class EvaluateHandler(ApiHandler):
+    @tornado.web.asynchronous
+    @gen.engine
+    @authorization_required
+    def post(self):
+        language = self.get_argument('language', default=None)
+
+        if language and (
+            language not in bothub_nlp_settings.SUPPORTED_LANGUAGES.keys() and
+            language not in NEXT_LANGS.keys()
+        ):
+            raise ValidationError(
+                'Language \'{}\' not supported by now.'.format(language),
+                field='language')
+
+        repository_authorization = self.repository_authorization()
+        if not repository_authorization:
+            raise AuthorizationIsRequired()
+
+        repository = repository_authorization.repository
+        update = repository.last_trained_update(language)
+
+        if not update:
+            raise ValidationError(
+                'This repository has never been trained',
+                field='language')
+
+        try:
+            evaluate_task = celery_app.send_task(
+                TASK_NLU_EVALUATE_UPDATE,
+                args=[
+                    update.id,
+                    repository_authorization.user.id,
+                ],
+                queue=queue_name(ACTION_EVALUATE, update.language))
+            evaluate_task.wait()
+            evaluate = evaluate_task.result
+            evaluate_report = {
+                'language': language,
+                'status': EVALUATE_STATUS_EVALUATED,
+                'update_id': update.id,
+                'evaluate_id': evaluate.get('id'),
+                'evaluate_version': evaluate.get('version'),
+            }
+        except Exception as e:
+            from .. import logger
+            logger.exception(e)
+
+            evaluate_report = {
+                'status': EVALUATE_STATUS_FAILED,
+                'error': str(e),
+            }
+
+        self.finish(evaluate_report)

bothub-nlp-api/bothub_nlp_api/handlers/evaluate.py

@@ -3,6 +3,7 @@
 
 ACTION_PARSE = 'parse'
 ACTION_TRAIN = 'train'
+ACTION_EVALUATE = 'evaluate'
 
 
 def queue_name(action, language):

bothub-nlp-celery/bothub_nlp_celery/actions.py

@@ -4,6 +4,7 @@
 
 from .actions import ACTION_PARSE
 from .actions import ACTION_TRAIN
+from .actions import ACTION_EVALUATE
 from .actions import queue_name
 from . import settings
 
@@ -19,6 +20,9 @@
 ] + [
     queue_name(ACTION_TRAIN, lang)
     for lang in bothub_nlp_settings.SUPPORTED_LANGUAGES.keys()
+] + [
+    queue_name(ACTION_EVALUATE, lang)
+    for lang in bothub_nlp_settings.SUPPORTED_LANGUAGES.keys()
 ])
 celery_app.conf.task_queues = [
     Queue(queue)

bothub-nlp-celery/bothub_nlp_celery/app.py

@@ -1,2 +1,3 @@
 TASK_NLU_PARSE_TEXT = 'nlu-parse-text'
 TASK_NLU_TRAIN_UPDATE = 'nlu-train-update'
+TASK_NLU_EVALUATE_UPDATE = 'nlu-evaluate-update'

bothub-nlp-celery/bothub_nlp_celery/tasks.py

@@ -13,7 +13,7 @@ celery-worker-on-demand = {ref = "0.2.0", git = "https://github.com/Ilhasoft/cel
 bothub-nlp-celery = {path = "./../bothub-nlp-celery"}
 docker = "==3.7.0"
 bothub-nlp = {path = "./../bothub-nlp"}
-bothub-engine = {ref = "1.19.1", git = "https://github.com/Ilhasoft/bothub-engine"}
+bothub-engine = {ref = "1.21.0", git = "https://github.com/Ilhasoft/bothub-engine"}
 
 [dev-packages]
 redis = "*"

bothub-nlp-nlu-worker-on-demand/Pipfile

@@ -47,6 +47,9 @@
         'BOTHUB_NLP_CELERY_BROKER_URL',
         'BOTHUB_NLP_CELERY_BACKEND_URL',
         'BOTHUB_NLP_NLU_AGROUP_LANGUAGE_QUEUE',
+        'BOTHUB_NLP_AWS_S3_BUCKET_NAME',
+        'BOTHUB_NLP_AWS_ACCESS_KEY_ID',
+        'BOTHUB_NLP_AWS_SECRET_ACCESS_KEY',
     ]
 ]