hide only rule dialogue turns

changelog/7701.improvement.md

@@ -0,0 +1,10 @@
+Hide dialogue turns predicted by `RulePolicy` in the tracker states
+for ML-only policies like `TEDPolicy`
+if those dialogue turns only appear as rules in the training data and do not appear in stories.
+
+Add `set_shared_policy_states(...)` method to all policies.
+This method sets `_rule_only_data` dict with keys:
+- `rule_only_slots`: Slot names, which only occur in rules but not in stories.
+- `rule_only_loops`: Loop names, which only occur in rules but not in stories.
+
+This information is needed for correct featurization to hide dialogue turns that appear only in rules.

data/test_dialogues/default.json

@@ -9,7 +9,8 @@
       "confidence": null,
       "policy": null,
       "timestamp": 1551952977.4850519,
-      "unpredictable": false
+      "unpredictable": false,
+      "hide_rule_turn": false
     },
     {
       "py/object": "rasa.shared.core.events.UserUttered",
@@ -61,7 +62,8 @@
       "confidence": null,
       "policy": null,
       "timestamp": 1551953040.607782,
-      "unpredictable": false
+      "unpredictable": false,
+      "hide_rule_turn": false
     },
     {
       "py/object": "rasa.shared.core.events.BotUttered",

data/test_dialogues/formbot.json

@@ -9,7 +9,8 @@
       "confidence":null,
       "policy":null,
       "timestamp":1551884035.892855,
-      "unpredictable":false
+      "unpredictable":false,
+      "hide_rule_turn": false
     },
     {
       "py/object":"rasa.shared.core.events.UserUttered",
@@ -43,7 +44,8 @@
       "confidence":1.0,
       "policy":"policy_2_MemoizationPolicy",
       "timestamp":1551884060.466681,
-      "unpredictable":false
+      "unpredictable":false,
+      "hide_rule_turn": false
     },
     {
       "py/object":"rasa.shared.core.events.BotUttered",
@@ -62,7 +64,8 @@
       "confidence":1.0,
       "policy":"policy_2_MemoizationPolicy",
       "timestamp":1551884061.9350882,
-      "unpredictable":false
+      "unpredictable":false,
+      "hide_rule_turn": false
     },
     {
       "py/object":"rasa.shared.core.events.UserUttered",
@@ -118,7 +121,8 @@
       "confidence":1.0,
       "policy":"policy_2_MemoizationPolicy",
       "timestamp":1551884095.542748,
-      "unpredictable":false
+      "unpredictable":false,
+      "hide_rule_turn": false
     },
     {
       "py/object":"rasa.shared.core.events.ActionExecuted",
@@ -127,7 +131,8 @@
       "confidence":1.0,
       "policy":"policy_2_MemoizationPolicy",
       "timestamp":1551884097.570883,
-      "unpredictable":false
+      "unpredictable":false,
+      "hide_rule_turn": false
     },
     {
       "py/object":"rasa.shared.core.events.BotUttered",
@@ -146,7 +151,8 @@
       "confidence":1.0,
       "policy":"policy_2_MemoizationPolicy",
       "timestamp":1551884098.8006358,
-      "unpredictable":false
+      "unpredictable":false,
+      "hide_rule_turn": false
     },
     {
       "py/object":"rasa.shared.core.events.UserUttered",
@@ -180,7 +186,8 @@
       "confidence":null,
       "policy":null,
       "timestamp":1551884214.951055,
-      "unpredictable":false
+      "unpredictable":false,
+      "hide_rule_turn": false
     },
     {
       "py/object":"rasa.shared.core.events.ActiveLoop",
@@ -200,7 +207,8 @@
       "confidence":0.7680902069097734,
       "policy":"policy_0_TEDPolicy",
       "timestamp":1551884216.705635,
-      "unpredictable":false
+      "unpredictable":false,
+      "hide_rule_turn": false
     }
   ]
 }

data/test_dialogues/moodbot.json

@@ -9,7 +9,8 @@
       "confidence":null,
       "policy":null,
       "timestamp":1551883958.346432,
-      "unpredictable":false
+      "unpredictable":false,
+      "hide_rule_turn": false
     },
     {
       "py/object":"rasa.shared.core.events.UserUttered",
@@ -69,7 +70,8 @@
       "confidence":1.0,
       "policy":"policy_2_MemoizationPolicy",
       "timestamp":1551883975.6456478,
-      "unpredictable":false
+      "unpredictable":false,
+      "hide_rule_turn": false
     },
     {
       "py/object":"rasa.shared.core.events.BotUttered",
@@ -97,7 +99,8 @@
       "confidence":1.0,
       "policy":"policy_2_MemoizationPolicy",
       "timestamp":1551883979.098331,
-      "unpredictable":false
+      "unpredictable":false,
+      "hide_rule_turn": false
     },
     {
       "py/object":"rasa.shared.core.events.UserUttered",
@@ -157,7 +160,8 @@
       "confidence":1.0,
       "policy":"policy_2_MemoizationPolicy",
       "timestamp":1551883985.031668,
-      "unpredictable":false
+      "unpredictable":false,
+      "hide_rule_turn": false
     },
     {
       "py/object":"rasa.shared.core.events.BotUttered",
@@ -176,7 +180,8 @@
       "confidence":1.0,
       "policy":"policy_2_MemoizationPolicy",
       "timestamp":1551883985.940413,
-      "unpredictable":false
+      "unpredictable":false,
+      "hide_rule_turn": false
     },
     {
       "py/object":"rasa.shared.core.events.BotUttered",
@@ -195,7 +200,8 @@
       "confidence":1.0,
       "policy":"policy_2_MemoizationPolicy",
       "timestamp":1551883986.958556,
-      "unpredictable":false
+      "unpredictable":false,
+      "hide_rule_turn": false
     },
     {
       "py/object":"rasa.shared.core.events.UserUttered",
@@ -255,7 +261,8 @@
       "confidence":1.0,
       "policy":"policy_2_MemoizationPolicy",
       "timestamp":1551883991.061463,
-      "unpredictable":false
+      "unpredictable":false,
+      "hide_rule_turn": false
     },
     {
       "py/object":"rasa.shared.core.events.BotUttered",

data/test_trackers/tracker_moodbot.json

@@ -39,7 +39,8 @@
       "name": "action_listen",
       "action_text": null,
       "policy": null,
-      "confidence": null
+      "confidence": null,
+      "hide_rule_turn": false
     },
     {
       "timestamp": 1517821726.200036,
@@ -78,15 +79,17 @@
       "name": "utter_greet",
       "action_text": null,
       "policy": null,
-      "confidence": null
+      "confidence": null,
+      "hide_rule_turn": false
     },
     {
       "timestamp": 1517821726.211038,
       "event": "action",
       "name": "action_listen",
       "action_text": null,
       "policy": null,
-      "confidence": null
+      "confidence": null,
+      "hide_rule_turn": false
     },
     {
       "timestamp": 1517821726.209836,
@@ -125,15 +128,17 @@
       "name": "utter_happy",
       "action_text": null,
       "policy": "policy_1_TEDPolicy",
-      "confidence": 0.8
+      "confidence": 0.8,
+      "hide_rule_turn": false
     },
     {
       "timestamp": 1517821726.211042,
       "event": "action",
       "name": "action_listen",
       "action_text": null,
       "policy": "policy_2_MemoizationPolicy",
-      "confidence": 1.0
+      "confidence": 1.0,
+      "hide_rule_turn": false
     }
   ]
 }

data/test_trackers/tracker_moodbot_with_new_utterances.json

@@ -38,7 +38,8 @@
       "event": "action",
       "name": "action_listen",
       "policy": null,
-      "confidence": null
+      "confidence": null,
+      "hide_rule_turn": false
     },
     {
       "timestamp": 1517821726.200036,
@@ -76,14 +77,16 @@
       "event": "action",
       "name": "utter_greet",
       "policy": null,
-      "confidence": null
+      "confidence": null,
+      "hide_rule_turn": false
     },
     {
       "timestamp": 1517821726.211038,
       "event": "action",
       "name": "action_listen",
       "policy": null,
-      "confidence": null
+      "confidence": null,
+      "hide_rule_turn": false
     },
     {
       "timestamp": 1517821726.209836,
@@ -121,14 +124,16 @@
       "event": "action",
       "name": "utter_happy",
       "policy": "policy_1_TEDPolicy",
-      "confidence": 0.8
+      "confidence": 0.8,
+      "hide_rule_turn": false
     },
     {
       "timestamp": 1517821726.211042,
       "event": "action",
       "name": "action_listen",
       "policy": "policy_2_MemoizationPolicy",
-      "confidence": 1.0
+      "confidence": 1.0,
+      "hide_rule_turn": false
     },
     {
       "timestamp": 1517821730.209836,
@@ -166,14 +171,16 @@
       "event": "action",
       "name": "utter_happy",
       "policy": "policy_1_TEDPolicy",
-      "confidence": 0.8
+      "confidence": 0.8,
+      "hide_rule_turn": false
     },
     {
       "timestamp": 1517821750.211042,
       "event": "action",
       "name": "action_listen",
       "policy": "policy_2_MemoizationPolicy",
-      "confidence": 1.0
+      "confidence": 1.0,
+      "hide_rule_turn": false
     }
   ]
 }

docs/docs/rules.mdx

@@ -47,6 +47,29 @@ rules:
 This example rule applies at the start of conversation as well as when the user decides
 to a send a message with an intent `greet` in the middle of an ongoing conversation.
 
+Dialogue turns that only appear as rules in the training data and do not appear in stories
+will be ignored by ML-only policies like `TEDPolicy` at prediction time.
+
+```yaml-rasa
+rules:
+- rule: Say `hello` whenever the user sends a message with intent `greet`
+  steps:
+  - intent: greet
+  - action: utter_greet
+
+stories:
+- story: story to find a restaurant
+  steps:
+  - intent: find_restaurant
+  - action: restaurant_form
+  - action: utter_restaurant_found
+```
+
+For example if you define the greeting rule as above and don't add it to any of your stories,
+after `RulePolicy` predicts `utter_greet`, `TEDPolicy` will
+make predictions as if the `greet, utter_greet` turn did not occur.
+
+
 ### Rules for the Conversation Start
 
 To write a rule which only applies at the beginning of a conversation, add a

examples/e2ebot/config.yml

@@ -2,6 +2,7 @@ language: en
 pipeline:
   - name: WhitespaceTokenizer
     intent_tokenization_flag: True
+  - name: LanguageModelFeaturizer
   - name: RegexFeaturizer
   - name: LexicalSyntacticFeaturizer
   - name: CountVectorsFeaturizer

rasa/core/actions/action.py

@@ -246,7 +246,10 @@ def event_for_successful_execution(
             Event which should be logged onto the tracker.
         """
         return ActionExecuted(
-            self.name(), prediction.policy_name, prediction.max_confidence
+            self.name(),
+            prediction.policy_name,
+            prediction.max_confidence,
+            hide_rule_turn=prediction.hide_rule_turn,
         )
 
 
@@ -332,6 +335,7 @@ def event_for_successful_execution(
             policy=prediction.policy_name,
             confidence=prediction.max_confidence,
             action_text=self.action_text,
+            hide_rule_turn=prediction.hide_rule_turn,
         )