Restrict acceptable values for model_url parameter of ConveRTTokenizer

changelog/7089.bugfix.md

@@ -0,0 +1,11 @@
+Fix [ConveRTTokenizer](https://rasa.com/docs/rasa/components#converttokenizer) failing because of wrong model URL by making the `model_url` parameter of `ConveRTTokenizer` mandatory.
+
+Since the ConveRT model was taken [offline](https://github.com/RasaHQ/rasa/issues/6806), we can no longer use 
+the earlier public URL of the model. Additionally, since the licence for the model is unknown, 
+we cannot host it ourselves. Users can still use the component by setting `model_url` to a community/self-hosted 
+model URL or path to a local directory containing model files. For example:
+```yaml
+pipeline:
+    - name: ConveRTTokenizer
+      model_url: <remote/local path to model>
+```

docs/docs/components.mdx

@@ -453,10 +453,16 @@ word vectors in your pipeline.
       "intent_split_symbol": "_"
       # Regular expression to detect tokens
       "token_pattern": None
-      # Remote URL of hosted model
-      "model_url": TF_HUB_MODULE_URL
+      # Remote URL/Local directory of model files(Required)
+      "model_url": None
     ```
 
+  :::note
+  Since the public URL of the ConveRT model was taken offline recently, it is now mandatory
+  to set the parameter `model_url` to a community/self-hosted URL or path to a local directory containing model files.
+
+  :::
+
 
   ### LanguageModelTokenizer
 

rasa/nlu/tokenizers/convert_tokenizer.py

@@ -6,18 +6,27 @@
 from rasa.nlu.tokenizers.tokenizer import Token
 from rasa.nlu.tokenizers.whitespace_tokenizer import WhitespaceTokenizer
 from rasa.shared.nlu.training_data.message import Message
-from rasa.utils import common
+from rasa.utils import common, io
 import rasa.utils.train_utils as train_utils
+from rasa.exceptions import RasaException
 import tensorflow as tf
+import os
 
 
-TF_HUB_MODULE_URL = (
+# URL to the old remote location of the model which
+# users might use. The model is no longer hosted here.
+ORIGINAL_TF_HUB_MODULE_URL = (
     "https://github.com/PolyAI-LDN/polyai-models/releases/download/v1.0/model.tar.gz"
 )
 
+# Warning: This URL is only intended for running pytests on ConveRT
+# related components. This URL should not be allowed to be used by the user.
+RESTRICTED_ACCESS_URL = "https://storage.googleapis.com/continuous-integration-model-storage/convert_tf2.tar.gz"
+
 
 class ConveRTTokenizer(WhitespaceTokenizer):
     """Tokenizer using ConveRT model.
+
     Loads the ConveRT(https://github.com/PolyAI-LDN/polyai-models#convert)
     model from TFHub and computes sub-word tokens for dense
     featurizable attributes of each message object.
@@ -30,21 +39,117 @@ class ConveRTTokenizer(WhitespaceTokenizer):
         "intent_split_symbol": "_",
         # Regular expression to detect tokens
         "token_pattern": None,
-        # Remote URL of hosted model
-        "model_url": TF_HUB_MODULE_URL,
+        # Remote URL/Local path to model files
+        "model_url": None,
     }
 
     def __init__(self, component_config: Dict[Text, Any] = None) -> None:
-        """Construct a new tokenizer using the WhitespaceTokenizer framework."""
+        """Construct a new tokenizer using the WhitespaceTokenizer framework.
 
+        Args:
+            component_config: User configuration for the component
+        """
         super().__init__(component_config)
 
-        self.model_url = self.component_config.get("model_url", TF_HUB_MODULE_URL)
+        self.model_url = self._get_validated_model_url()
 
         self.module = train_utils.load_tf_hub_model(self.model_url)
 
         self.tokenize_signature = self.module.signatures["tokenize"]
 
+    @staticmethod
+    def _validate_model_files_exist(model_directory: Text) -> None:
+        """Check if essential model files exist inside the model_directory.
+
+        Args:
+            model_directory: Directory to investigate
+        """
+        files_to_check = [
+            os.path.join(model_directory, "saved_model.pb"),
+            os.path.join(model_directory, "variables/variables.index"),
+            os.path.join(model_directory, "variables/variables.data-00001-of-00002"),
+            os.path.join(model_directory, "variables/variables.data-00000-of-00002"),
+        ]
+
+        for file_path in files_to_check:
+            if not os.path.exists(file_path):
+                raise RasaException(
+                    f"""File {file_path} does not exist.
+                    Re-check the files inside the directory {model_directory}.
+                    It should contain the following model 
+                    files - [{", ".join(files_to_check)}]"""
+                )
+
+    def _get_validated_model_url(self) -> Text:
+        """Validates the specified `model_url` parameter.
+
+        The `model_url` parameter cannot be left empty. It can either
+        be set to a remote URL where the model is hosted or it can be
+        a path to a local directory.
+
+        Returns:
+            Validated path to model
+        """
+        model_url = self.component_config.get("model_url", None)
+
+        if not model_url:
+            raise RasaException(
+                """Parameter "model_url" was not specified in the configuration of "ConveRTTokenizer".
+                You can either use a community hosted URL of the model
+                or if you have a local copy of the model, pass the
+                path to the directory containing the model files."""
+            )
+
+        if io.is_valid_remote_url(model_url):
+
+            if model_url == ORIGINAL_TF_HUB_MODULE_URL:
+                # Can't use the originally hosted URL
+                raise RasaException(
+                    f"""Parameter "model_url" of "ConveRTTokenizer" was
+                    set to "{model_url}" which does not contain the model any longer.
+                    You can either use a community hosted URL or if you have a
+                    local copy of the model, pass the path to the directory
+                    containing the model files."""
+                )
+
+            if model_url == RESTRICTED_ACCESS_URL:
+                # Can't use the URL that is reserved for tests only
+                raise RasaException(
+                    f"""Parameter "model_url" of "ConveRTTokenizer" was
+                    set to "{model_url}" which is strictly reserved for pytests of Rasa Open Source only.
+                    Due to licensing issues you are not allowed to use the model from this URL.
+                    You can either use a community hosted URL or if you have a
+                    local copy of the model, pass the path to the directory
+                    containing the model files."""
+                )
+
+        elif os.path.isfile(model_url):
+            # Definitely invalid since the specified path should be a directory
+            raise RasaException(
+                """Parameter "model_url" of "ConveRTTokenizer" was
+                set to the path of a file which is invalid. You
+                can either use a community hosted URL or if you have a
+                local copy of the model, pass the path to the directory
+                containing the model files."""
+            )
+
+        elif os.path.isdir(model_url):
+            # Looks like a local directory. Inspect the directory
+            # to see if model files exist.
+            self._validate_model_files_exist(model_url)
+            # Convert the path to an absolute one since
+            # TFHUB doesn't like relative paths
+            model_url = os.path.abspath(model_url)
+
+        else:
+            raise RasaException(
+                f"""{model_url} is neither a valid remote URL nor a local directory.
+                You can either use a community hosted URL or if you have a
+                local copy of the model, pass the path to
+                the directory containing the model files."""
+            )
+        return model_url
+
     @classmethod
     def cache_key(
         cls, component_meta: Dict[Text, Any], model_metadata: Metadata

rasa/utils/io.py

@@ -10,6 +10,7 @@
 from io import BytesIO as IOReader
 from pathlib import Path
 from typing import Text, Any, Union, List, Type, Callable, TYPE_CHECKING
+import re
 
 import rasa.shared.constants
 import rasa.shared.utils.io
@@ -212,3 +213,26 @@ def json_pickle(file_name: Union[Text, Path], obj: Any) -> None:
     jsonpickle_numpy.register_handlers()
 
     rasa.shared.utils.io.write_text_file(jsonpickle.dumps(obj), file_name)
+
+
+def is_valid_remote_url(url: Text) -> bool:
+    """Check whether the url specified is a well formed one.
+
+    Regex taken from https://stackoverflow.com/a/7160778/3001665
+
+    Args:
+        url: Remote URL to validate
+
+    Returns: Boolean for validity
+    """
+    regex = re.compile(
+        r"^(?:http|ftp)s?://"  # http:// or https://
+        r"(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|"  # domain...
+        r"localhost|"  # localhost...
+        r"\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})"  # ...or ip
+        r"(?::\d+)?"  # optional port
+        r"(?:/?|[/?]\S+)$",
+        re.IGNORECASE,
+    )
+
+    return re.match(regex, url) is not None

tests/nlu/featurizers/test_convert_featurizer.py

@@ -1,7 +1,11 @@
 import numpy as np
 import pytest
+from typing import Text
 
-from rasa.nlu.tokenizers.convert_tokenizer import ConveRTTokenizer
+from rasa.nlu.tokenizers.convert_tokenizer import (
+    ConveRTTokenizer,
+    RESTRICTED_ACCESS_URL,
+)
 from rasa.shared.nlu.training_data.training_data import TrainingData
 from rasa.shared.nlu.training_data.message import Message
 from rasa.nlu.constants import TOKENS_NAMES
@@ -10,13 +14,15 @@
 from rasa.nlu.featurizers.dense_featurizer.convert_featurizer import ConveRTFeaturizer
 
 
-# TODO
-#   skip tests as the ConveRT model is not publicly available anymore (see https://github.com/RasaHQ/rasa/issues/6806)
+@pytest.mark.skip_on_windows
+def test_convert_featurizer_process(component_builder, monkeypatch):
 
+    monkeypatch.setattr(
+        ConveRTTokenizer, "_get_validated_model_url", lambda x: RESTRICTED_ACCESS_URL
+    )
 
-@pytest.mark.skip
-def test_convert_featurizer_process(component_builder):
-    tokenizer = component_builder.create_component_from_class(ConveRTTokenizer)
+    component_config = {"name": "ConveRTTokenizer", "model_url": RESTRICTED_ACCESS_URL}
+    tokenizer = ConveRTTokenizer(component_config)
     featurizer = component_builder.create_component_from_class(ConveRTFeaturizer)
 
     sentence = "Hey how are you today ?"
@@ -41,9 +47,14 @@ def test_convert_featurizer_process(component_builder):
     assert np.allclose(sent_vecs[-1][:5], expected_cls, atol=1e-5)
 
 
-@pytest.mark.skip
-def test_convert_featurizer_train(component_builder):
-    tokenizer = component_builder.create_component_from_class(ConveRTTokenizer)
+@pytest.mark.skip_on_windows
+def test_convert_featurizer_train(component_builder, monkeypatch):
+
+    monkeypatch.setattr(
+        ConveRTTokenizer, "_get_validated_model_url", lambda x: RESTRICTED_ACCESS_URL
+    )
+    component_config = {"name": "ConveRTTokenizer", "model_url": RESTRICTED_ACCESS_URL}
+    tokenizer = ConveRTTokenizer(component_config)
     featurizer = component_builder.create_component_from_class(ConveRTFeaturizer)
 
     sentence = "Hey how are you today ?"
@@ -88,6 +99,7 @@ def test_convert_featurizer_train(component_builder):
     assert sent_vecs is None
 
 
+@pytest.mark.skip_on_windows
 @pytest.mark.parametrize(
     "sentence, expected_text",
     [
@@ -98,9 +110,15 @@ def test_convert_featurizer_train(component_builder):
         ("ńöñàśçií", "ńöñàśçií"),
     ],
 )
-@pytest.mark.skip
-def test_convert_featurizer_tokens_to_text(component_builder, sentence, expected_text):
-    tokenizer = component_builder.create_component_from_class(ConveRTTokenizer)
+def test_convert_featurizer_tokens_to_text(
+    sentence: Text, expected_text: Text, monkeypatch
+):
+
+    monkeypatch.setattr(
+        ConveRTTokenizer, "_get_validated_model_url", lambda x: RESTRICTED_ACCESS_URL
+    )
+    component_config = {"name": "ConveRTTokenizer", "model_url": RESTRICTED_ACCESS_URL}
+    tokenizer = ConveRTTokenizer(component_config)
     tokens = tokenizer.tokenize(Message(data={TEXT: sentence}), attribute=TEXT)
 
     actual_text = ConveRTFeaturizer._tokens_to_text([tokens])[0]