From 29901e35de4b35e8c3b73e30b9984b9da4a4fdc7 Mon Sep 17 00:00:00 2001 From: Vova Vv Date: Fri, 8 Jan 2021 14:06:30 +0100 Subject: [PATCH 01/68] hide only rule dialogue turns --- rasa/core/featurizers/tracker_featurizers.py | 40 +++- rasa/core/policies/rule_policy.py | 200 ++++++++++++++----- rasa/core/policies/ted_policy.py | 41 +++- rasa/shared/core/domain.py | 79 +++++++- rasa/shared/core/events.py | 77 ++++++- rasa/shared/core/generator.py | 14 +- rasa/shared/core/trackers.py | 14 +- 7 files changed, 382 insertions(+), 83 deletions(-) diff --git a/rasa/core/featurizers/tracker_featurizers.py b/rasa/core/featurizers/tracker_featurizers.py index 26560266f440..b4ba43180923 100644 --- a/rasa/core/featurizers/tracker_featurizers.py +++ b/rasa/core/featurizers/tracker_featurizers.py @@ -51,17 +51,21 @@ def __init__( self.state_featurizer = state_featurizer @staticmethod - def _create_states(tracker: DialogueStateTracker, domain: Domain) -> List[State]: + def _create_states( + tracker: DialogueStateTracker, domain: Domain, for_only_ml_policy: bool = False + ) -> List[State]: """Create states for the given tracker. Args: tracker: a :class:`rasa.core.trackers.DialogueStateTracker` domain: a :class:`rasa.shared.core.domain.Domain` + for_only_ml_policy: If True ignore dialogue turns that are present + only in rules. Returns: a list of states """ - return tracker.past_states(domain) + return tracker.past_states(domain, for_only_ml_policy) def _featurize_states( self, @@ -235,14 +239,17 @@ def prediction_states( trackers: List[DialogueStateTracker], domain: Domain, use_text_for_last_user_input: bool = False, + for_only_ml_policy: bool = False, ) -> List[List[State]]: """Transforms list of trackers to lists of states for prediction. Args: - trackers: The trackers to transform - domain: The domain + trackers: The trackers to transform. + domain: The domain. use_text_for_last_user_input: Indicates whether to use text or intent label for featurizing last user input. + for_only_ml_policy: If True ignore dialogue turns that are present + only in rules. Returns: A list of states. @@ -257,6 +264,7 @@ def create_state_features( domain: Domain, interpreter: NaturalLanguageInterpreter, use_text_for_last_user_input: bool = False, + for_only_ml_policy: bool = False, ) -> List[List[Dict[Text, List["Features"]]]]: """Create state features for prediction. @@ -266,6 +274,8 @@ def create_state_features( interpreter: The interpreter use_text_for_last_user_input: Indicates whether to use text or intent label for featurizing last user input. + for_only_ml_policy: If True ignore dialogue turns that are present + only in rules. Returns: A dictionary of state type (INTENT, TEXT, ACTION_NAME, ACTION_TEXT, @@ -273,7 +283,7 @@ def create_state_features( turns in all trackers. """ trackers_as_states = self.prediction_states( - trackers, domain, use_text_for_last_user_input + trackers, domain, use_text_for_last_user_input, for_only_ml_policy ) return self._featurize_states(trackers_as_states, interpreter) @@ -393,20 +403,24 @@ def prediction_states( trackers: List[DialogueStateTracker], domain: Domain, use_text_for_last_user_input: bool = False, + for_only_ml_policy: bool = False, ) -> List[List[State]]: """Transforms list of trackers to lists of states for prediction. Args: - trackers: The trackers to transform - domain: The domain, + trackers: The trackers to transform. + domain: The domain. use_text_for_last_user_input: Indicates whether to use text or intent label for featurizing last user input. + for_only_ml_policy: If True ignore dialogue turns that are present + only in rules. Returns: A list of states. """ trackers_as_states = [ - self._create_states(tracker, domain) for tracker in trackers + self._create_states(tracker, domain, for_only_ml_policy) + for tracker in trackers ] self._choose_last_user_input(trackers_as_states, use_text_for_last_user_input) @@ -548,20 +562,24 @@ def prediction_states( trackers: List[DialogueStateTracker], domain: Domain, use_text_for_last_user_input: bool = False, + for_only_ml_policy: bool = False, ) -> List[List[State]]: """Transforms list of trackers to lists of states for prediction. Args: - trackers: The trackers to transform - domain: The domain + trackers: The trackers to transform. + domain: The domain. use_text_for_last_user_input: Indicates whether to use text or intent label for featurizing last user input. + for_only_ml_policy: If True ignore dialogue turns that are present + only in rules. Returns: A list of states. """ trackers_as_states = [ - self._create_states(tracker, domain) for tracker in trackers + self._create_states(tracker, domain, for_only_ml_policy) + for tracker in trackers ] trackers_as_states = [ self.slice_state_history(states, self.max_history) diff --git a/rasa/core/policies/rule_policy.py b/rasa/core/policies/rule_policy.py index 4d056569a09c..15fbc991ac0e 100644 --- a/rasa/core/policies/rule_policy.py +++ b/rasa/core/policies/rule_policy.py @@ -9,7 +9,12 @@ from rasa.shared.constants import DOCS_URL_RULES from rasa.shared.exceptions import RasaException import rasa.shared.utils.io -from rasa.shared.core.events import LoopInterrupted, UserUttered, ActionExecuted +from rasa.shared.core.events import ( + LoopInterrupted, + UserUttered, + ActionExecuted, + HideRuleTurn, +) from rasa.core.featurizers.tracker_featurizers import TrackerFeaturizer from rasa.shared.nlu.interpreter import NaturalLanguageInterpreter from rasa.core.policies.memoization import MemoizationPolicy @@ -42,6 +47,7 @@ from rasa.shared.nlu.constants import ACTION_NAME, INTENT_NAME_KEY import rasa.core.test import rasa.core.training.training +from rasa.shared.core.events import Event if TYPE_CHECKING: @@ -58,6 +64,9 @@ RULES = "rules" RULES_FOR_LOOP_UNHAPPY_PATH = "rules_for_loop_unhappy_path" +RULES_NOT_IN_STORIES = "rules_not_in_stories" +ONLY_RULE_SLOTS = "only_rule_slots" +ONLY_RULE_LOOPS = "only_rule_loops" LOOP_WAS_INTERRUPTED = "loop_was_interrupted" DO_NOT_PREDICT_LOOP_ACTION = "do_not_predict_loop_action" @@ -404,6 +413,38 @@ def _check_for_incomplete_rules( logger.debug("Found no incompletions in rules.") + @staticmethod + def _get_slots_loops_from_states( + trackers_as_states: List[List[State]], + ) -> Tuple[Set[Text], Set[Text]]: + slots = set() + loops = set() + for states in trackers_as_states: + for state in states: + slots.update(set(state.get(SLOTS, {}).keys())) + active_loop = state.get(ACTIVE_LOOP, {}).get(LOOP_NAME) + if active_loop: + loops.add(active_loop) + return slots, loops + + def _find_only_rule_slots_loops( + self, + rule_trackers_as_states: List[List[State]], + story_trackers_as_states: List[List[State]], + ) -> Tuple[List[Text], List[Text]]: + rule_slots, rule_loops = self._get_slots_loops_from_states( + rule_trackers_as_states + ) + story_slots, story_loops = self._get_slots_loops_from_states( + story_trackers_as_states + ) + + # set is not json serializable, so convert to list + return ( + list(rule_slots - story_slots - {SHOULD_NOT_BE_SET}), + list(rule_loops - story_loops - {SHOULD_NOT_BE_SET}), + ) + def _predict_next_action( self, tracker: TrackerWithCachedStates, @@ -426,15 +467,13 @@ def _predict_next_action( return predicted_action_name - def _check_prediction( + def _predicted_action_name( self, tracker: TrackerWithCachedStates, domain: Domain, interpreter: NaturalLanguageInterpreter, gold_action_name: Text, - collect_sources: bool, ) -> Optional[Text]: - predicted_action_name = self._predict_next_action(tracker, domain, interpreter) # if there is an active_loop, # RulePolicy will always predict active_loop first, @@ -449,22 +488,34 @@ def _check_prediction( tracker, domain, interpreter ) - if collect_sources: - # we need to remember which action should be predicted by the rule - # in order to correctly output the names of the contradicting rules - rule_name = tracker.sender_id - if self._prediction_source in {DEFAULT_RULES, LOOP_RULES}: - # the real gold action contradict the one in the rules in this case - gold_action_name = predicted_action_name - rule_name = self._prediction_source - - self._rules_sources[self._prediction_source].append( - (rule_name, gold_action_name) - ) - return + return predicted_action_name + + def _collect_sources( + self, + tracker: TrackerWithCachedStates, + predicted_action_name: Optional[Text], + gold_action_name: Text, + ) -> None: + # we need to remember which action should be predicted by the rule + # in order to correctly output the names of the contradicting rules + rule_name = tracker.sender_id + if self._prediction_source in {DEFAULT_RULES, LOOP_RULES}: + # the real gold action contradict the one in the rules in this case + gold_action_name = predicted_action_name + rule_name = self._prediction_source + + self._rules_sources[self._prediction_source].append( + (rule_name, gold_action_name) + ) + def _check_prediction( + self, + tracker: TrackerWithCachedStates, + predicted_action_name: Optional[Text], + gold_action_name: Text, + ) -> List[Text]: if not predicted_action_name or predicted_action_name == gold_action_name: - return + return [] tracker_type = "rule" if tracker.is_rule_tracker else "story" contradicting_rules = { @@ -482,7 +533,7 @@ def _check_prediction( if predicted_action_name != self._fallback_action_name: error_message += f" which predicted action '{predicted_action_name}'" - return error_message + "." + return [error_message + "."] def _run_prediction_on_trackers( self, @@ -490,11 +541,12 @@ def _run_prediction_on_trackers( domain: Domain, interpreter: NaturalLanguageInterpreter, collect_sources: bool, - ) -> List[Text]: + ) -> Tuple[List[Text], Set[Text]]: if collect_sources: self._rules_sources = defaultdict(list) error_messages = [] + rules_used_in_stories = set() pbar = tqdm( trackers, desc="Processed trackers", @@ -524,19 +576,31 @@ def _run_prediction_on_trackers( continue gold_action_name = event.action_name or event.action_text - error_message = self._check_prediction( - running_tracker, - domain, - interpreter, - gold_action_name, - collect_sources, + predicted_action_name = self._predicted_action_name( + running_tracker, domain, interpreter, gold_action_name ) - if error_message: - error_messages.append(error_message) + if collect_sources: + self._collect_sources( + running_tracker, predicted_action_name, gold_action_name + ) + else: + # to be able to remove only rules turns from the dialogue history + # for ML policies, + # we need to know which rules were used in ML trackers + if ( + not tracker.is_rule_tracker + and predicted_action_name == gold_action_name + and self._prediction_source not in {DEFAULT_RULES, LOOP_RULES} + ): + rules_used_in_stories.add(self._prediction_source) + + error_messages += self._check_prediction( + running_tracker, predicted_action_name, gold_action_name + ) running_tracker.update(event) - return error_messages + return error_messages, rules_used_in_stories def _find_contradicting_rules( self, @@ -544,7 +608,7 @@ def _find_contradicting_rules( all_trackers: List[TrackerWithCachedStates], domain: Domain, interpreter: NaturalLanguageInterpreter, - ) -> None: + ) -> Set[Text]: logger.debug("Started checking rules and stories for contradictions.") # during training we run `predict_action_probabilities` to check for # contradicting rules. @@ -553,11 +617,11 @@ def _find_contradicting_rules( logger.setLevel(logging.WARNING) # we need to run prediction on rule trackers twice, because we need to collect - # information which rule snippets contributed to the learned rules + # the information about which rule snippets contributed to the learned rules self._run_prediction_on_trackers( rule_trackers, domain, interpreter, collect_sources=True ) - error_messages = self._run_prediction_on_trackers( + error_messages, rules_used_in_stories = self._run_prediction_on_trackers( all_trackers, domain, interpreter, collect_sources=False ) @@ -571,29 +635,14 @@ def _find_contradicting_rules( ) logger.debug("Found no contradicting rules.") + return rules_used_in_stories - def train( + def _create_lookup_from_trackers( self, - training_trackers: List[TrackerWithCachedStates], + rule_trackers: List[TrackerWithCachedStates], + story_trackers: List[TrackerWithCachedStates], domain: Domain, - interpreter: NaturalLanguageInterpreter, - **kwargs: Any, ) -> None: - - # only consider original trackers (no augmented ones) - training_trackers = [ - t - for t in training_trackers - if not hasattr(t, "is_augmented") or not t.is_augmented - ] - # only use trackers from rule-based training data - rule_trackers = [t for t in training_trackers if t.is_rule_tracker] - if self._restrict_rules: - self._check_rule_restriction(rule_trackers) - - if self._check_for_contradictions: - self._check_for_incomplete_rules(rule_trackers, domain) - ( rule_trackers_as_states, rule_trackers_as_actions, @@ -604,12 +653,19 @@ def train( ) self.lookup[RULES] = self._remove_rule_snippet_predictions(rules_lookup) - story_trackers = [t for t in training_trackers if not t.is_rule_tracker] ( story_trackers_as_states, story_trackers_as_actions, ) = self.featurizer.training_states_and_actions(story_trackers, domain) + if self._check_for_contradictions: + ( + self.lookup[ONLY_RULE_SLOTS], + self.lookup[ONLY_RULE_LOOPS], + ) = self._find_only_rule_slots_loops( + rule_trackers_as_states, story_trackers_as_states + ) + # use all trackers to find negative rules in unhappy paths trackers_as_states = rule_trackers_as_states + story_trackers_as_states trackers_as_actions = rule_trackers_as_actions + story_trackers_as_actions @@ -621,13 +677,43 @@ def train( trackers_as_states, trackers_as_actions ) + def train( + self, + training_trackers: List[TrackerWithCachedStates], + domain: Domain, + interpreter: NaturalLanguageInterpreter, + **kwargs: Any, + ) -> None: + + # only consider original trackers (no augmented ones) + training_trackers = [ + t + for t in training_trackers + if not hasattr(t, "is_augmented") or not t.is_augmented + ] + # trackers from rule-based training data + rule_trackers = [t for t in training_trackers if t.is_rule_tracker] + if self._restrict_rules: + self._check_rule_restriction(rule_trackers) + if self._check_for_contradictions: + self._check_for_incomplete_rules(rule_trackers, domain) + + # trackers from ML-based training data + story_trackers = [t for t in training_trackers if not t.is_rule_tracker] + + self._create_lookup_from_trackers(rule_trackers, story_trackers, domain) + # make this configurable because checking might take a lot of time if self._check_for_contradictions: # using trackers here might not be the most efficient way, however # it allows us to directly test `predict_action_probabilities` method - self._find_contradicting_rules( + rules_used_in_stories = self._find_contradicting_rules( rule_trackers, training_trackers, domain, interpreter ) + # set is not json serializable, so convert to list + self.lookup[RULES_NOT_IN_STORIES] = list( + set(self.lookup[RULES].keys()) - rules_used_in_stories + ) logger.debug(f"Memorized '{len(self.lookup[RULES])}' unique rules.") @@ -959,9 +1045,17 @@ def _prediction_with_unhappy_path( returning_from_unhappy_path: bool, is_end_to_end_prediction: bool, ) -> "PolicyPrediction": + optional_events = [] + if self._prediction_source in self.lookup.get(RULES_NOT_IN_STORIES, []): + # the prediction is based on rules that are not present in the stories + optional_events = [ + HideRuleTurn(self.lookup[ONLY_RULE_SLOTS], self.lookup[ONLY_RULE_LOOPS]) + ] + return self._prediction( probabilities, events=[LoopInterrupted(True)] if returning_from_unhappy_path else [], + optional_events=optional_events, is_end_to_end_prediction=is_end_to_end_prediction, ) diff --git a/rasa/core/policies/ted_policy.py b/rasa/core/policies/ted_policy.py index f06530b26a54..3d25ddb230ac 100644 --- a/rasa/core/policies/ted_policy.py +++ b/rasa/core/policies/ted_policy.py @@ -539,7 +539,11 @@ def _featurize_tracker_for_e2e( # the first example in the constructed batch either does not contain user input # or uses intent or text based on whether TED is e2e only. tracker_state_features = self.featurizer.create_state_features( - [tracker], domain, interpreter, use_text_for_last_user_input=self.only_e2e + [tracker], + domain, + interpreter, + use_text_for_last_user_input=self.only_e2e, + for_only_ml_policy=True, ) # the second - text, but only after user utterance and if not only e2e if ( @@ -548,12 +552,16 @@ def _featurize_tracker_for_e2e( and not self.only_e2e ): tracker_state_features += self.featurizer.create_state_features( - [tracker], domain, interpreter, use_text_for_last_user_input=True + [tracker], + domain, + interpreter, + use_text_for_last_user_input=True, + for_only_ml_policy=True, ) return tracker_state_features def _pick_confidence( - self, confidences: np.ndarray, similarities: np.ndarray + self, confidences: np.ndarray, similarities: np.ndarray, domain: Domain ) -> Tuple[np.ndarray, bool]: # the confidences and similarities have shape (batch-size x number of actions) # batch-size can only be 1 or 2; @@ -568,16 +576,30 @@ def _pick_confidence( # we use similarities to pick appropriate input, # since it seems to be more accurate measure, # policy is trained to maximize the similarity not the confidence + non_e2e_action_name = domain.action_names_or_texts[ + np.argmax(confidences[0]) + ] + e2e_action_name = domain.action_names_or_texts[np.argmax(confidences[1])] + logger.debug(f"User intent lead to '{non_e2e_action_name}'.") + logger.debug(f"User text lead to '{e2e_action_name}'.") if ( np.max(confidences[1]) > self.config[E2E_CONFIDENCE_THRESHOLD] # TODO maybe compare confidences is better and np.max(similarities[1]) > np.max(similarities[0]) ): + logger.debug(f"TED predicted '{e2e_action_name}' based on user text.") return confidences[1], True + logger.debug(f"TED predicted '{non_e2e_action_name}' based on user intent.") return confidences[0], False # by default the first example in a batch is the one to use for prediction + predicted_action_name = domain.action_names_or_texts[np.argmax(confidences[0])] + basis_for_prediction = "text" if self.only_e2e else "intent" + logger.debug( + f"TED predicted '{predicted_action_name}' " + f"based on user {basis_for_prediction}." + ) return confidences[0], self.only_e2e def predict_action_probabilities( @@ -606,7 +628,9 @@ def predict_action_probabilities( similarities = output["similarities"].numpy()[:, -1, :] confidences = output["action_scores"].numpy()[:, -1, :] # take correct prediction from batch - confidence, is_e2e_prediction = self._pick_confidence(confidences, similarities) + confidence, is_e2e_prediction = self._pick_confidence( + confidences, similarities, domain + ) if self.config[LOSS_TYPE] == SOFTMAX and self.config[RANKING_LENGTH] > 0: confidence = rasa.utils.train_utils.normalize( @@ -1212,7 +1236,7 @@ def _encode_real_features_per_attribute( Args: tf_batch_data: dictionary mapping every attribute to its features and masks attribute: the attribute we will encode features for - (e.g., ACTION_NAME, INTENT) + (e.g., ACTION_NAME, INTENT) Returns: A tensor combining all features for `attribute` @@ -1312,10 +1336,9 @@ def _convert_to_original_shape( Args: attribute_features: the "real" features to convert - attribute_mask: the tensor containing the position of "real" features - in the dialogue, shape is (batch-size x dialogue_len x 1) - dialogue_lengths: the tensor containing the actual dialogue length, - shape is (batch-size,) + tf_batch_data: dictionary mapping every attribute to its features and masks + attribute: the attribute we will encode features for + (e.g., ACTION_NAME, INTENT) Returns: The converted attribute features diff --git a/rasa/shared/core/domain.py b/rasa/shared/core/domain.py index afffdf79fed1..8a9358acd801 100644 --- a/rasa/shared/core/domain.py +++ b/rasa/shared/core/domain.py @@ -1068,13 +1068,82 @@ def get_active_states(self, tracker: "DialogueStateTracker") -> State: } return self._clean_state(state) + @staticmethod + def _remove_only_rule_features( + state: State, tracker: "DialogueStateTracker" + ) -> None: + # remove only rule slots + for slot in tracker.only_rule_slots: + if state.get(rasa.shared.core.constants.SLOTS, {}).get(slot): + del state[rasa.shared.core.constants.SLOTS][slot] + # remove active loop feature if it is only rule loop + if ( + state.get(rasa.shared.core.constants.ACTIVE_LOOP, {}).get( + rasa.shared.core.constants.LOOP_NAME + ) + in tracker.only_rule_loops + ): + del state[rasa.shared.core.constants.ACTIVE_LOOP][ + rasa.shared.core.constants.LOOP_NAME + ] + + @staticmethod + def _substitute_only_rule_user_input(state: State, last_ml_state: State) -> None: + if not rasa.shared.core.trackers.is_prev_action_listen_in_state(state): + if not last_ml_state.get(rasa.shared.core.constants.USER) and state.get( + rasa.shared.core.constants.USER + ): + del state[rasa.shared.core.constants.USER] + elif last_ml_state.get(rasa.shared.core.constants.USER): + state[rasa.shared.core.constants.USER] = last_ml_state[ + rasa.shared.core.constants.USER + ] + def states_for_tracker_history( - self, tracker: "DialogueStateTracker" + self, tracker: "DialogueStateTracker", for_only_ml_policy: bool = False ) -> List[State]: - """Array of states for each state of the trackers history.""" - return [ - self.get_active_states(tr) for tr in tracker.generate_all_prior_trackers() - ] + """Array of states for each state of the trackers history. + + Args: + tracker: Instance of `DialogueStateTracker` to featurize. + for_only_ml_policy: If True ignore dialogue turns that are present + only in rules. + + Return: + A list of states. + """ + states = [] + last_ml_action_sub_state = None + last_action_is_ml_action = True + for tr in tracker.generate_all_prior_trackers(): + if for_only_ml_policy: + # remember previous ml action based on the last non hidden turn + # we need this to override previous action in the ml state + if last_action_is_ml_action: + last_ml_action_sub_state = self._get_prev_action_sub_state(tr) + last_action_is_ml_action = not tr.hide_rule_turn + + if tr.hide_rule_turn: + continue + + state = self.get_active_states(tr) + + if for_only_ml_policy: + # clean state from only rule features + self._remove_only_rule_features(state, tracker) + # make sure user input is the same as for previous state + # for non action_listen turns + if states: + self._substitute_only_rule_user_input(state, states[-1]) + # substitute previous rule action with last_ml_action_sub_state + if last_ml_action_sub_state: + state[ + rasa.shared.core.constants.PREVIOUS_ACTION + ] = last_ml_action_sub_state + + states.append(state) + + return states def slots_for_entities(self, entities: List[Dict[Text, Any]]) -> List[SlotSet]: if self.store_entities_as_slots: diff --git a/rasa/shared/core/events.py b/rasa/shared/core/events.py index be081e2038ad..e1f39cd990a9 100644 --- a/rasa/shared/core/events.py +++ b/rasa/shared/core/events.py @@ -723,7 +723,10 @@ def __hash__(self) -> int: def __eq__(self, other: Any) -> bool: """Compares this event with another event.""" - return isinstance(other, EntitiesAdded) + if not isinstance(other, EntitiesAdded): + return NotImplemented + + return self.entities == other.entities @classmethod def _from_parameters(cls, parameters: Dict[Text, Any]) -> "EntitiesAdded": @@ -1540,6 +1543,9 @@ def apply_to(self, tracker: "DialogueStateTracker") -> None: """Applies event to current conversation state.""" tracker.set_latest_action(self.as_sub_state()) tracker.clear_followup_action() + # HideRuleTurn event is added by RulePolicy before actual action is executed, + # so we can safely reset it on each action executed + tracker.hide_rule_turn = False class AgentUttered(SkipEventInMDStoryMixin): @@ -1875,3 +1881,72 @@ def apply_to(self, tracker: "DialogueStateTracker") -> None: """Applies event to current conversation state.""" # noinspection PyProtectedMember tracker._reset() + + +class HideRuleTurn(SkipEventInMDStoryMixin, AlwaysEqualEventMixin): + """Emulate undoing of the last dialogue turn. + + The bot reverts everything until before the most recent action. + This includes the action itself, as well as any events that + action created, like set slot events, unless they are in a list + to reapply them. + """ + + type_name = "hide_rule_turn" + + def __init__( + self, + only_rule_slots: List[Text], + only_rule_loops: List[Text], + timestamp: Optional[float] = None, + metadata: Optional[Dict[Text, Any]] = None, + ) -> None: + """Initializes event. + + Args: + only_rule_slots: The list of slot names, + SlotSet events for which shouldn't be hidden + only_rule_loops: The list of loop names, + ActiveLoop events for which shouldn't be hidden + timestamp: the timestamp + metadata: some optional metadata + """ + super().__init__(timestamp, metadata) + self.only_rule_slots = only_rule_slots + self.only_rule_loops = only_rule_loops + + def __hash__(self) -> int: + """Returns unique hash for event.""" + return hash(32143124321) + + @classmethod + def _from_parameters(cls, parameters: Dict[Text, Any]) -> "HideRuleTurn": + return HideRuleTurn( + parameters.get("only_rule_slots"), + parameters.get("only_rule_loops"), + parameters.get("timestamp"), + parameters.get("metadata"), + ) + + def as_dict(self) -> Dict[Text, Any]: + """Converts the event into a dict. + + Returns: + A dict that represents this event. + """ + d = super().as_dict() + d.update( + { + "only_rule_slots": self.only_rule_slots, + "only_rule_loops": self.only_rule_loops, + } + ) + return d + + def apply_to(self, tracker: "DialogueStateTracker") -> None: + # HideRuleTurn event is added by RulePolicy before actual action is executed, + # we will reset it on each action executed + tracker.hide_rule_turn = True + # only rule slots and loops are always the same for all the trackers + tracker.only_rule_slots = self.only_rule_slots + tracker.only_rule_loops = self.only_rule_loops diff --git a/rasa/shared/core/generator.py b/rasa/shared/core/generator.py index 994ee52fedaf..10248cb700cc 100644 --- a/rasa/shared/core/generator.py +++ b/rasa/shared/core/generator.py @@ -107,7 +107,19 @@ def _unfreeze_states(frozen_states: Deque[FrozenState]) -> List[State]: for frozen_state in frozen_states ] - def past_states(self, domain: Domain) -> List[State]: + def past_states( + self, domain: Domain, for_only_ml_policy: bool = False + ) -> List[State]: + """Generate the past states of this tracker based on the history. + + Args: + domain: a :class:`rasa.shared.core.domain.Domain`. + for_only_ml_policy: If True ignore dialogue turns that are present + only in rules. + + Returns: + a list of states + """ states_for_hashing = self.past_states_for_hashing(domain) return self._unfreeze_states(states_for_hashing) diff --git a/rasa/shared/core/trackers.py b/rasa/shared/core/trackers.py index 96988d3e7498..4b68c7d610c3 100644 --- a/rasa/shared/core/trackers.py +++ b/rasa/shared/core/trackers.py @@ -59,6 +59,7 @@ SessionStarted, ActionExecutionRejected, EntitiesAdded, + HideRuleTurn, ) from rasa.shared.core.domain import Domain, State from rasa.shared.core.slots import Slot @@ -205,6 +206,9 @@ def __init__( self.latest_bot_utterance = None self._reset() self.active_loop: Dict[Text, Union[Text, bool, Dict, None]] = {} + self.hide_rule_turn = False + self.only_rule_slots = [] + self.only_rule_loops = [] ### # Public tracker interface @@ -273,16 +277,20 @@ def freeze_current_state(state: State) -> FrozenState: }.items() ) - def past_states(self, domain: Domain) -> List[State]: + def past_states( + self, domain: Domain, for_only_ml_policy: bool = False + ) -> List[State]: """Generate the past states of this tracker based on the history. Args: - domain: a :class:`rasa.shared.core.domain.Domain` + domain: a :class:`rasa.shared.core.domain.Domain`. + for_only_ml_policy: If True ignore dialogue turns that are present + only in rules. Returns: a list of states """ - return domain.states_for_tracker_history(self) + return domain.states_for_tracker_history(self, for_only_ml_policy) def change_loop_to(self, loop_name: Optional[Text]) -> None: """Set the currently active loop. From d957f5dafc6c5158d44394a8cea7db956a92649c Mon Sep 17 00:00:00 2001 From: Vova Vv Date: Fri, 8 Jan 2021 14:22:03 +0100 Subject: [PATCH 02/68] add missing docstrings --- rasa/core/policies/rule_policy.py | 7 +++++++ rasa/shared/core/domain.py | 8 ++++++++ rasa/shared/core/events.py | 5 +++++ 3 files changed, 20 insertions(+) diff --git a/rasa/core/policies/rule_policy.py b/rasa/core/policies/rule_policy.py index 15fbc991ac0e..77969a5bc7c8 100644 --- a/rasa/core/policies/rule_policy.py +++ b/rasa/core/policies/rule_policy.py @@ -684,7 +684,14 @@ def train( interpreter: NaturalLanguageInterpreter, **kwargs: Any, ) -> None: + """Trains the policy on given training trackers. + Args: + training_trackers: + the list of the :class:`rasa.core.trackers.DialogueStateTracker` + domain: the :class:`rasa.shared.core.domain.Domain` + interpreter: Interpreter which can be used by the polices for featurization. + """ # only consider original trackers (no augmented ones) training_trackers = [ t diff --git a/rasa/shared/core/domain.py b/rasa/shared/core/domain.py index 8a9358acd801..3b894c07a713 100644 --- a/rasa/shared/core/domain.py +++ b/rasa/shared/core/domain.py @@ -1146,6 +1146,14 @@ def states_for_tracker_history( return states def slots_for_entities(self, entities: List[Dict[Text, Any]]) -> List[SlotSet]: + """Find which slots should be auto fill with entities. + + Args: + entities: The list of entities. + + Returns: + A list of `SlotSet` events. + """ if self.store_entities_as_slots: slot_events = [] for s in self.slots: diff --git a/rasa/shared/core/events.py b/rasa/shared/core/events.py index e1f39cd990a9..df39dd2ec584 100644 --- a/rasa/shared/core/events.py +++ b/rasa/shared/core/events.py @@ -1944,6 +1944,11 @@ def as_dict(self) -> Dict[Text, Any]: return d def apply_to(self, tracker: "DialogueStateTracker") -> None: + """Applies event to current conversation state. + + Args: + tracker: The current conversation state. + """ # HideRuleTurn event is added by RulePolicy before actual action is executed, # we will reset it on each action executed tracker.hide_rule_turn = True From c7fed2664ec681ec135c1f7d1fafa4e9c44a2eb7 Mon Sep 17 00:00:00 2001 From: Vova Vv Date: Fri, 8 Jan 2021 15:39:12 +0100 Subject: [PATCH 03/68] fix docstrings --- rasa/core/policies/rule_policy.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/rasa/core/policies/rule_policy.py b/rasa/core/policies/rule_policy.py index 77969a5bc7c8..d38cf2049e78 100644 --- a/rasa/core/policies/rule_policy.py +++ b/rasa/core/policies/rule_policy.py @@ -687,9 +687,8 @@ def train( """Trains the policy on given training trackers. Args: - training_trackers: - the list of the :class:`rasa.core.trackers.DialogueStateTracker` - domain: the :class:`rasa.shared.core.domain.Domain` + training_trackers: The list of the trackers. + domain: The domain. interpreter: Interpreter which can be used by the polices for featurization. """ # only consider original trackers (no augmented ones) From ff5014a191ea78975fc4960eb1f01247da3930a9 Mon Sep 17 00:00:00 2001 From: Vova Vv Date: Mon, 11 Jan 2021 10:34:25 +0100 Subject: [PATCH 04/68] fix hiding rules --- rasa/core/policies/form_policy.py | 4 +- rasa/core/policies/memoization.py | 6 +- rasa/core/policies/rule_policy.py | 96 +++++++++++++------- rasa/core/policies/sklearn_policy.py | 4 +- rasa/shared/core/domain.py | 4 +- tests/core/policies/test_rule_policy.py | 112 ++++++++++++++++++++++++ tests/core/test_policies.py | 6 +- 7 files changed, 191 insertions(+), 41 deletions(-) diff --git a/rasa/core/policies/form_policy.py b/rasa/core/policies/form_policy.py index 5fcb2f7a2db1..d68a0ad36257 100644 --- a/rasa/core/policies/form_policy.py +++ b/rasa/core/policies/form_policy.py @@ -112,7 +112,9 @@ def state_is_unhappy(self, tracker: DialogueStateTracker, domain: Domain) -> boo # since it is assumed that training stories contain # only unhappy paths, notify the form that # it should not be validated if predicted by other policy - tracker_as_states = self.featurizer.prediction_states([tracker], domain) + tracker_as_states = self.featurizer.prediction_states( + [tracker], domain, for_only_ml_policy=True + ) states = tracker_as_states[0] memorized_form = self.recall(states, tracker, domain) diff --git a/rasa/core/policies/memoization.py b/rasa/core/policies/memoization.py index 849678e3f4a1..b84e6de4b1a6 100644 --- a/rasa/core/policies/memoization.py +++ b/rasa/core/policies/memoization.py @@ -224,7 +224,9 @@ def predict_action_probabilities( ) -> PolicyPrediction: result = self._default_predictions(domain) - tracker_as_states = self.featurizer.prediction_states([tracker], domain) + tracker_as_states = self.featurizer.prediction_states( + [tracker], domain, for_only_ml_policy=True + ) states = tracker_as_states[0] logger.debug(f"Current tracker state:{self.format_tracker_states(states)}") predicted_action_name = self.recall(states, tracker, domain) @@ -310,7 +312,7 @@ def _recall_using_delorean(self, old_states, tracker, domain) -> Optional[Text]: mcfly_tracker = self._back_to_the_future(tracker) while mcfly_tracker is not None: tracker_as_states = self.featurizer.prediction_states( - [mcfly_tracker], domain + [mcfly_tracker], domain, for_only_ml_policy=True ) states = tracker_as_states[0] diff --git a/rasa/core/policies/rule_policy.py b/rasa/core/policies/rule_policy.py index d38cf2049e78..ce7566c224a9 100644 --- a/rasa/core/policies/rule_policy.py +++ b/rasa/core/policies/rule_policy.py @@ -47,7 +47,6 @@ from rasa.shared.nlu.constants import ACTION_NAME, INTENT_NAME_KEY import rasa.core.test import rasa.core.training.training -from rasa.shared.core.events import Event if TYPE_CHECKING: @@ -71,8 +70,8 @@ LOOP_WAS_INTERRUPTED = "loop_was_interrupted" DO_NOT_PREDICT_LOOP_ACTION = "do_not_predict_loop_action" -DEFAULT_RULES = "predicting default action" -LOOP_RULES = "handling active loops and forms" +DEFAULT_RULES = "predicting default action with intent " +LOOP_RULES = "handling active loops and forms - " class InvalidRule(RasaException): @@ -499,7 +498,9 @@ def _collect_sources( # we need to remember which action should be predicted by the rule # in order to correctly output the names of the contradicting rules rule_name = tracker.sender_id - if self._prediction_source in {DEFAULT_RULES, LOOP_RULES}: + if self._prediction_source.startswith( + DEFAULT_RULES + ) or self._prediction_source.startswith(LOOP_RULES): # the real gold action contradict the one in the rules in this case gold_action_name = predicted_action_name rule_name = self._prediction_source @@ -508,6 +509,21 @@ def _collect_sources( (rule_name, gold_action_name) ) + @staticmethod + def _default_sources() -> Set[Text]: + return { + DEFAULT_RULES + default_intent + for default_intent in DEFAULT_ACTION_MAPPINGS.keys() + } + + @staticmethod + def _handling_loop_sources(domain: Domain) -> Set[Text]: + loop_sources = set() + for loop_name in domain.form_names: + loop_sources.add(LOOP_RULES + loop_name) + loop_sources.add(LOOP_RULES + loop_name + " - " + ACTION_LISTEN_NAME) + return loop_sources + def _check_prediction( self, tracker: TrackerWithCachedStates, @@ -590,7 +606,6 @@ def _run_prediction_on_trackers( if ( not tracker.is_rule_tracker and predicted_action_name == gold_action_name - and self._prediction_source not in {DEFAULT_RULES, LOOP_RULES} ): rules_used_in_stories.add(self._prediction_source) @@ -608,7 +623,7 @@ def _find_contradicting_rules( all_trackers: List[TrackerWithCachedStates], domain: Domain, interpreter: NaturalLanguageInterpreter, - ) -> Set[Text]: + ) -> List[Text]: logger.debug("Started checking rules and stories for contradictions.") # during training we run `predict_action_probabilities` to check for # contradicting rules. @@ -635,7 +650,13 @@ def _find_contradicting_rules( ) logger.debug("Found no contradicting rules.") - return rules_used_in_stories + all_rules = ( + set(self._rules_sources.keys()) + | self._default_sources() + | self._handling_loop_sources(domain) + ) + # set is not json serializable, so convert to list + return list(all_rules - rules_used_in_stories) def _create_lookup_from_trackers( self, @@ -713,13 +734,9 @@ def train( if self._check_for_contradictions: # using trackers here might not be the most efficient way, however # it allows us to directly test `predict_action_probabilities` method - rules_used_in_stories = self._find_contradicting_rules( + self.lookup[RULES_NOT_IN_STORIES] = self._find_contradicting_rules( rule_trackers, training_trackers, domain, interpreter ) - # set is not json serializable, so convert to list - self.lookup[RULES_NOT_IN_STORIES] = list( - set(self.lookup[RULES].keys()) - rules_used_in_stories - ) logger.debug(f"Memorized '{len(self.lookup[RULES])}' unique rules.") @@ -817,12 +834,12 @@ def _get_possible_keys( @staticmethod def _find_action_from_default_actions( tracker: DialogueStateTracker, - ) -> Optional[Text]: + ) -> Tuple[Optional[Text], Optional[Text]]: if ( not tracker.latest_action_name == ACTION_LISTEN_NAME or not tracker.latest_message ): - return None + return None, None default_action_name = DEFAULT_ACTION_MAPPINGS.get( tracker.latest_message.intent.get(INTENT_NAME_KEY) @@ -830,13 +847,17 @@ def _find_action_from_default_actions( if default_action_name: logger.debug(f"Predicted default action '{default_action_name}'.") + return ( + default_action_name, + DEFAULT_RULES + tracker.latest_message.intent.get(INTENT_NAME_KEY), + ) - return default_action_name + return None, None @staticmethod def _find_action_from_loop_happy_path( tracker: DialogueStateTracker, - ) -> Optional[Text]: + ) -> Tuple[Optional[Text], Optional[Text]]: active_loop_name = tracker.active_loop_name active_loop_rejected = tracker.active_loop.get(LOOP_REJECTED) @@ -853,14 +874,19 @@ def _find_action_from_loop_happy_path( if should_predict_loop: logger.debug(f"Predicted loop '{active_loop_name}'.") - return active_loop_name + return active_loop_name, LOOP_RULES + active_loop_name # predict `action_listen` if loop action was run successfully if should_predict_listen: logger.debug( f"Predicted '{ACTION_LISTEN_NAME}' after loop '{active_loop_name}'." ) - return ACTION_LISTEN_NAME + return ( + ACTION_LISTEN_NAME, + LOOP_RULES + active_loop_name + " - " + ACTION_LISTEN_NAME, + ) + + return None, None def _find_action_from_rules( self, @@ -891,7 +917,7 @@ def _find_action_from_rules( return None, None, False tracker_as_states = self.featurizer.prediction_states( - [tracker], domain, use_text_for_last_user_input + [tracker], domain, use_text_for_last_user_input, for_only_ml_policy=False ) states = tracker_as_states[0] @@ -942,7 +968,11 @@ def _find_action_from_rules( f"Predicted loop '{active_loop_name}' by overwriting " f"'{ACTION_LISTEN_NAME}' predicted by general rule." ) - return active_loop_name, LOOP_RULES, returning_from_unhappy_path + return ( + active_loop_name, + LOOP_RULES + active_loop_name, + returning_from_unhappy_path, + ) # do not predict anything predicted_action_name = None @@ -988,13 +1018,15 @@ def predict_action_probabilities( # Rasa Open Source default actions overrule anything. If users want to achieve # the same, they need to write a rule or make sure that their loop rejects # accordingly. - default_action_name = self._find_action_from_default_actions(tracker) + ( + default_action_name, + self._prediction_source, + ) = self._find_action_from_default_actions(tracker) # text has priority over intents including default, # however loop happy path has priority over rules prediction if default_action_name and not rules_action_name_from_text: - self._prediction_source = DEFAULT_RULES - return self._prediction( + return self._rule_prediction( self._prediction_result(default_action_name, tracker, domain) ) @@ -1002,18 +1034,20 @@ def predict_action_probabilities( # The rules or any other prediction will be applied only if a loop was rejected. # If we are in a loop, and the loop didn't run previously or rejected, we can # simply force predict the loop. - loop_happy_path_action_name = self._find_action_from_loop_happy_path(tracker) + ( + loop_happy_path_action_name, + self._prediction_source, + ) = self._find_action_from_loop_happy_path(tracker) if loop_happy_path_action_name: - self._prediction_source = LOOP_RULES # this prediction doesn't use user input # and happy user input anyhow should be ignored during featurization - return self._prediction( + return self._rule_prediction( self._prediction_result(loop_happy_path_action_name, tracker, domain) ) # predict rules from text first if rules_action_name_from_text: - return self._prediction_with_unhappy_path( + return self._rule_prediction( self._prediction_result(rules_action_name_from_text, tracker, domain), returning_from_unhappy_path=returning_from_unhappy_path_from_text, is_end_to_end_prediction=True, @@ -1034,7 +1068,7 @@ def predict_action_probabilities( else: probabilities = self._default_predictions(domain) - return self._prediction_with_unhappy_path( + return self._rule_prediction( probabilities, returning_from_unhappy_path=( # returning_from_unhappy_path is a negative condition, @@ -1045,11 +1079,11 @@ def predict_action_probabilities( is_end_to_end_prediction=False, ) - def _prediction_with_unhappy_path( + def _rule_prediction( self, probabilities: List[float], - returning_from_unhappy_path: bool, - is_end_to_end_prediction: bool, + returning_from_unhappy_path: bool = False, + is_end_to_end_prediction: bool = False, ) -> "PolicyPrediction": optional_events = [] if self._prediction_source in self.lookup.get(RULES_NOT_IN_STORIES, []): diff --git a/rasa/core/policies/sklearn_policy.py b/rasa/core/policies/sklearn_policy.py index 2efcab236ce3..2e39656d19a7 100644 --- a/rasa/core/policies/sklearn_policy.py +++ b/rasa/core/policies/sklearn_policy.py @@ -284,7 +284,9 @@ def predict_action_probabilities( interpreter: NaturalLanguageInterpreter, **kwargs: Any, ) -> PolicyPrediction: - X = self.featurizer.create_state_features([tracker], domain, interpreter) + X = self.featurizer.create_state_features( + [tracker], domain, interpreter, for_only_ml_policy=True + ) training_data, _ = model_data_utils.convert_to_data_format( X, self.zero_state_features ) diff --git a/rasa/shared/core/domain.py b/rasa/shared/core/domain.py index 3b894c07a713..efeba3b48b74 100644 --- a/rasa/shared/core/domain.py +++ b/rasa/shared/core/domain.py @@ -1083,9 +1083,7 @@ def _remove_only_rule_features( ) in tracker.only_rule_loops ): - del state[rasa.shared.core.constants.ACTIVE_LOOP][ - rasa.shared.core.constants.LOOP_NAME - ] + del state[rasa.shared.core.constants.ACTIVE_LOOP] @staticmethod def _substitute_only_rule_user_input(state: State, last_ml_state: State) -> None: diff --git a/tests/core/policies/test_rule_policy.py b/tests/core/policies/test_rule_policy.py index 2add4ead31ac..6237a8db3c13 100644 --- a/tests/core/policies/test_rule_policy.py +++ b/tests/core/policies/test_rule_policy.py @@ -20,7 +20,10 @@ ACTION_BACK_NAME, RULE_SNIPPET_ACTION_NAME, REQUESTED_SLOT, + USER, + PREVIOUS_ACTION, ) +from rasa.shared.nlu.constants import TEXT, INTENT, ACTION_NAME from rasa.shared.core.domain import Domain from rasa.shared.core.events import ( ActionExecuted, @@ -29,6 +32,7 @@ SlotSet, ActionExecutionRejected, LoopInterrupted, + HideRuleTurn, ) from rasa.shared.nlu.interpreter import RegexInterpreter from rasa.core.nlg import TemplatedNaturalLanguageGenerator @@ -1928,3 +1932,111 @@ def test_predict_nothing_if_fallback_disabled(): ) assert prediction.max_confidence == 0 + + +def test_hide_rule_turn_with_loops(): + form_name = "some_form" + another_form_name = "another_form" + activate_form = "activate_form" + activate_another_form = "activate_another_form" + chitchat = "chitchat" + action_chitchat = "action_chitchat" + domain = Domain.from_yaml( + f""" + intents: + - {GREET_INTENT_NAME} + - {activate_form} + - {chitchat} + - {activate_another_form} + actions: + - {UTTER_GREET_ACTION} + - {action_chitchat} + slots: + {REQUESTED_SLOT}: + type: unfeaturized + forms: + - {form_name} + - {another_form_name} + """ + ) + + form_activation_rule = _form_activation_rule(domain, form_name, activate_form) + + another_form_activation_rule = _form_activation_rule( + domain, another_form_name, activate_another_form + ) + another_form_activation_story = another_form_activation_rule.copy() + another_form_activation_story.is_rule_tracker = False + + chitchat_story = TrackerWithCachedStates.from_events( + "chitchat story", + domain=domain, + slots=domain.slots, + evts=[ + ActionExecuted(ACTION_LISTEN_NAME), + UserUttered(intent={"name": chitchat}), + ActionExecuted(action_chitchat), + ActionExecuted(ACTION_LISTEN_NAME), + ], + ) + policy = RulePolicy() + policy.train( + [ + form_activation_rule, + chitchat_story, + another_form_activation_rule, + another_form_activation_story, + ], + domain, + RegexInterpreter(), + ) + + conversation_events = [ + ActionExecuted(ACTION_LISTEN_NAME), + UserUttered("haha", {"name": activate_form}), + ] + prediction = policy.predict_action_probabilities( + DialogueStateTracker.from_events( + "casd", evts=conversation_events, slots=domain.slots + ), + domain, + RegexInterpreter(), + ) + assert_predicted_action(prediction, domain, form_name) + assert isinstance(prediction.optional_events[0], HideRuleTurn) + assert form_name in prediction.optional_events[0].only_rule_loops + assert another_form_name not in prediction.optional_events[0].only_rule_loops + + conversation_events += prediction.optional_events + conversation_events += [ + ActionExecuted(form_name), + ActiveLoop(form_name), + ] + prediction = policy.predict_action_probabilities( + DialogueStateTracker.from_events( + "casd", evts=conversation_events, slots=domain.slots + ), + domain, + RegexInterpreter(), + ) + assert_predicted_action(prediction, domain, ACTION_LISTEN_NAME) + assert isinstance(prediction.optional_events[0], HideRuleTurn) + assert form_name in prediction.optional_events[0].only_rule_loops + assert another_form_name not in prediction.optional_events[0].only_rule_loops + + conversation_events += prediction.optional_events + conversation_events += [ + ActionExecuted(ACTION_LISTEN_NAME), + UserUttered("haha", {"name": chitchat}), + ] + tracker = DialogueStateTracker.from_events( + "casd", evts=conversation_events, slots=domain.slots + ) + states = tracker.past_states(domain, for_only_ml_policy=True) + assert states == [ + {}, + { + USER: {TEXT: "haha", INTENT: chitchat}, + PREVIOUS_ACTION: {ACTION_NAME: ACTION_LISTEN_NAME}, + }, + ] diff --git a/tests/core/test_policies.py b/tests/core/test_policies.py index 997eb5228352..756c9c6e4feb 100644 --- a/tests/core/test_policies.py +++ b/tests/core/test_policies.py @@ -793,9 +793,9 @@ def test_memorise_with_nlu( tracker = DialogueStateTracker(dialogue.name, default_domain.slots) tracker.recreate_from_dialogue(dialogue) - states = trained_policy.featurizer.prediction_states([tracker], default_domain)[ - 0 - ] + states = trained_policy.featurizer.prediction_states( + [tracker], default_domain, for_only_ml_policy=True + )[0] recalled = trained_policy.recall(states, tracker, default_domain) assert recalled is not None From ab647821d9858d1361eca13b9aaec39a5260cf2a Mon Sep 17 00:00:00 2001 From: Vova Vv Date: Mon, 11 Jan 2021 11:05:32 +0100 Subject: [PATCH 05/68] add tests --- tests/core/policies/test_rule_policy.py | 166 ++++++++++++++++++++++++ 1 file changed, 166 insertions(+) diff --git a/tests/core/policies/test_rule_policy.py b/tests/core/policies/test_rule_policy.py index 6237a8db3c13..bb3e770a1a27 100644 --- a/tests/core/policies/test_rule_policy.py +++ b/tests/core/policies/test_rule_policy.py @@ -1934,6 +1934,172 @@ def test_predict_nothing_if_fallback_disabled(): assert prediction.max_confidence == 0 +def test_hide_rule_turn(): + chitchat = "chitchat" + action_chitchat = "action_chitchat" + domain = Domain.from_yaml( + f""" + intents: + - {GREET_INTENT_NAME} + - {chitchat} + actions: + - {UTTER_GREET_ACTION} + - {action_chitchat} + """ + ) + chitchat_story = TrackerWithCachedStates.from_events( + "chitchat story", + domain=domain, + slots=domain.slots, + evts=[ + ActionExecuted(ACTION_LISTEN_NAME), + UserUttered(intent={"name": chitchat}), + ActionExecuted(action_chitchat), + ActionExecuted(ACTION_LISTEN_NAME), + ], + ) + policy = RulePolicy() + policy.train( + [GREET_RULE, chitchat_story,], domain, RegexInterpreter(), + ) + + conversation_events = [ + ActionExecuted(ACTION_LISTEN_NAME), + UserUttered("haha", {"name": GREET_INTENT_NAME}), + ] + prediction = policy.predict_action_probabilities( + DialogueStateTracker.from_events( + "casd", evts=conversation_events, slots=domain.slots + ), + domain, + RegexInterpreter(), + ) + assert_predicted_action(prediction, domain, UTTER_GREET_ACTION) + assert isinstance(prediction.optional_events[0], HideRuleTurn) + + conversation_events += prediction.optional_events + conversation_events += [ + ActionExecuted(UTTER_GREET_ACTION), + ] + prediction = policy.predict_action_probabilities( + DialogueStateTracker.from_events( + "casd", evts=conversation_events, slots=domain.slots + ), + domain, + RegexInterpreter(), + ) + assert_predicted_action(prediction, domain, ACTION_LISTEN_NAME) + assert isinstance(prediction.optional_events[0], HideRuleTurn) + + conversation_events += prediction.optional_events + conversation_events += [ + ActionExecuted(ACTION_LISTEN_NAME), + UserUttered("haha", {"name": chitchat}), + ] + tracker = DialogueStateTracker.from_events( + "casd", evts=conversation_events, slots=domain.slots + ) + states = tracker.past_states(domain, for_only_ml_policy=True) + assert states == [ + {}, + { + USER: {TEXT: "haha", INTENT: chitchat}, + PREVIOUS_ACTION: {ACTION_NAME: ACTION_LISTEN_NAME}, + }, + ] + + +def test_hide_rule_turn_with_slots(): + some_action = "some_action" + some_other_action = "some_other_action" + some_intent = "some_intent" + some_other_intent = "some_other_intent" + some_slot = "some_slot" + some_slot_value = "value1" + some_other_slot = "some_other_slot" + some_other_slot_value = "value2" + domain = Domain.from_yaml( + f""" + intents: + - {some_intent} + - {some_other_intent} + actions: + - {some_action} + - {some_other_action} + slots: + {some_slot}: + type: text + {some_other_slot}: + type: text + """ + ) + + simple_rule = TrackerWithCachedStates.from_events( + "simple rule with an action that sets 1 slot", + domain=domain, + slots=domain.slots, + evts=[ + ActionExecuted(RULE_SNIPPET_ACTION_NAME), + ActionExecuted(ACTION_LISTEN_NAME), + UserUttered(intent={"name": some_intent}), + ActionExecuted(some_action), + SlotSet(some_slot, some_slot_value), + ActionExecuted(ACTION_LISTEN_NAME), + ], + is_rule_tracker=True, + ) + simple_rule_with_slot_set = TrackerWithCachedStates.from_events( + "simple rule with an additional slot set before it starts", + domain=domain, + slots=domain.slots, + evts=[ + SlotSet(some_other_slot, some_other_slot_value), + ActionExecuted(RULE_SNIPPET_ACTION_NAME), + ActionExecuted(ACTION_LISTEN_NAME), + UserUttered(intent={"name": some_intent}), + ActionExecuted(some_action), + SlotSet(some_slot, some_slot_value), + ActionExecuted(ACTION_LISTEN_NAME), + ], + is_rule_tracker=True, + ) + simple_story_with_other_slot_set = TrackerWithCachedStates.from_events( + "simple rule with an additional slot set before it starts", + domain=domain, + slots=domain.slots, + evts=[ + ActionExecuted(ACTION_LISTEN_NAME), + UserUttered(intent={"name": some_other_action}), + ActionExecuted(some_other_action), + SlotSet(some_other_slot, some_other_slot_value), + ActionExecuted(ACTION_LISTEN_NAME), + ], + ) + + policy = RulePolicy() + policy.train( + [simple_rule, simple_rule_with_slot_set, simple_story_with_other_slot_set], + domain, + RegexInterpreter(), + ) + + conversation_events = [ + ActionExecuted(ACTION_LISTEN_NAME), + UserUttered("haha", {"name": some_intent}), + ] + prediction = policy.predict_action_probabilities( + DialogueStateTracker.from_events( + "casd", evts=conversation_events, slots=domain.slots + ), + domain, + RegexInterpreter(), + ) + assert_predicted_action(prediction, domain, some_action) + assert isinstance(prediction.optional_events[0], HideRuleTurn) + assert some_slot in prediction.optional_events[0].only_rule_slots + assert some_other_slot not in prediction.optional_events[0].only_rule_slots + + def test_hide_rule_turn_with_loops(): form_name = "some_form" another_form_name = "another_form" From e55933ec00b00235424c310641fd47457474b41d Mon Sep 17 00:00:00 2001 From: Vova Vv Date: Mon, 11 Jan 2021 11:13:40 +0100 Subject: [PATCH 06/68] add changelog --- changelog/7701.improvement.md | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 changelog/7701.improvement.md diff --git a/changelog/7701.improvement.md b/changelog/7701.improvement.md new file mode 100644 index 000000000000..1562b789c8c0 --- /dev/null +++ b/changelog/7701.improvement.md @@ -0,0 +1,3 @@ +Hide dialogue turns predicted by `RulePolicy` in the tracker states +for ML only policies like `TEDPolicy` +if dialogue turns only appear as a rule in the training data and does not appear in stories. \ No newline at end of file From f7d083b96f9c4a9b9ddcccd1e22b282a25bf133e Mon Sep 17 00:00:00 2001 From: Vova Vv Date: Mon, 11 Jan 2021 11:20:46 +0100 Subject: [PATCH 07/68] add docs --- docs/docs/rules.mdx | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/docs/rules.mdx b/docs/docs/rules.mdx index 2af748bd1b9e..e37d890e5737 100644 --- a/docs/docs/rules.mdx +++ b/docs/docs/rules.mdx @@ -47,6 +47,14 @@ rules: This example rule applies at the start of conversation as well as when the user decides to a send a message with an intent `greet` in the middle of an ongoing conversation. +Dialogue turns that only appear as a rule in the training data and do not appear in stories +will be hidden in the tracker states for ML only policies like `TEDPolicy`. +For example if you have a greet rule above and don't add it to any of your stories then +when during prediction `RulePolicy` predicts `utter_greet`, afterwards `TEDPolicy` will +see dialogue turns as if user never input intent `greet`. + + + ### Rules for the Conversation Start To write a rule which only applies at the beginning of a conversation, add a From ee525c6ee47cf90e32c2045a64af6b09967f58d7 Mon Sep 17 00:00:00 2001 From: Vladimir Vlasov Date: Mon, 11 Jan 2021 16:28:46 +0100 Subject: [PATCH 08/68] Update docs/docs/rules.mdx Co-authored-by: Ella Rohm-Ensing --- docs/docs/rules.mdx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/docs/rules.mdx b/docs/docs/rules.mdx index e37d890e5737..38e4a7bf198b 100644 --- a/docs/docs/rules.mdx +++ b/docs/docs/rules.mdx @@ -47,8 +47,8 @@ rules: This example rule applies at the start of conversation as well as when the user decides to a send a message with an intent `greet` in the middle of an ongoing conversation. -Dialogue turns that only appear as a rule in the training data and do not appear in stories -will be hidden in the tracker states for ML only policies like `TEDPolicy`. +Dialogue turns that only appear as rules in the training data and do not appear in stories +will be ignored by ML-only policies like `TEDPolicy` at prediction time. For example if you have a greet rule above and don't add it to any of your stories then when during prediction `RulePolicy` predicts `utter_greet`, afterwards `TEDPolicy` will see dialogue turns as if user never input intent `greet`. From bcca5774cd8033a0e8601c83fd2a5168232ea201 Mon Sep 17 00:00:00 2001 From: Vladimir Vlasov Date: Mon, 11 Jan 2021 16:29:21 +0100 Subject: [PATCH 09/68] Update changelog/7701.improvement.md Co-authored-by: Ella Rohm-Ensing --- changelog/7701.improvement.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/changelog/7701.improvement.md b/changelog/7701.improvement.md index 1562b789c8c0..13e4d85285fd 100644 --- a/changelog/7701.improvement.md +++ b/changelog/7701.improvement.md @@ -1,3 +1,3 @@ Hide dialogue turns predicted by `RulePolicy` in the tracker states -for ML only policies like `TEDPolicy` -if dialogue turns only appear as a rule in the training data and does not appear in stories. \ No newline at end of file +for ML-only policies like `TEDPolicy` +if those dialogue turns only appear as rules in the training data and do not appear in stories. From cba4f698cc982a0ca1b00d21e123cc7c9b768565 Mon Sep 17 00:00:00 2001 From: Vladimir Vlasov Date: Mon, 11 Jan 2021 16:29:43 +0100 Subject: [PATCH 10/68] Update docs/docs/rules.mdx Co-authored-by: Ella Rohm-Ensing --- docs/docs/rules.mdx | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/docs/docs/rules.mdx b/docs/docs/rules.mdx index 38e4a7bf198b..b72c3f084e3c 100644 --- a/docs/docs/rules.mdx +++ b/docs/docs/rules.mdx @@ -49,10 +49,9 @@ to a send a message with an intent `greet` in the middle of an ongoing conversat Dialogue turns that only appear as rules in the training data and do not appear in stories will be ignored by ML-only policies like `TEDPolicy` at prediction time. -For example if you have a greet rule above and don't add it to any of your stories then -when during prediction `RulePolicy` predicts `utter_greet`, afterwards `TEDPolicy` will -see dialogue turns as if user never input intent `greet`. - +For example if you define the greeting rule as above and don't add it to any of your stories, +after `RulePolicy` predicts `utter_greet`, `TEDPolicy` will +make predictions as if the `greet, utter_greet` turn did not occur. ### Rules for the Conversation Start From 75516ce09bcb69753696b3de9d851cbcc416bba7 Mon Sep 17 00:00:00 2001 From: Vova Vv Date: Mon, 11 Jan 2021 18:08:09 +0100 Subject: [PATCH 11/68] create featurize for prediction helpers to automatically set for_only_ml_policy --- rasa/core/policies/form_policy.py | 5 +-- rasa/core/policies/memoization.py | 10 +---- rasa/core/policies/policy.py | 59 +++++++++++++++++++++++++++- rasa/core/policies/rule_policy.py | 5 +-- rasa/core/policies/sklearn_policy.py | 4 +- rasa/core/policies/ted_policy.py | 16 ++------ tests/core/test_policies.py | 4 +- 7 files changed, 67 insertions(+), 36 deletions(-) diff --git a/rasa/core/policies/form_policy.py b/rasa/core/policies/form_policy.py index d68a0ad36257..95fd234608c0 100644 --- a/rasa/core/policies/form_policy.py +++ b/rasa/core/policies/form_policy.py @@ -112,10 +112,7 @@ def state_is_unhappy(self, tracker: DialogueStateTracker, domain: Domain) -> boo # since it is assumed that training stories contain # only unhappy paths, notify the form that # it should not be validated if predicted by other policy - tracker_as_states = self.featurizer.prediction_states( - [tracker], domain, for_only_ml_policy=True - ) - states = tracker_as_states[0] + states = self.prediction_states(tracker, domain) memorized_form = self.recall(states, tracker, domain) diff --git a/rasa/core/policies/memoization.py b/rasa/core/policies/memoization.py index b84e6de4b1a6..55fa21efeb7f 100644 --- a/rasa/core/policies/memoization.py +++ b/rasa/core/policies/memoization.py @@ -224,10 +224,7 @@ def predict_action_probabilities( ) -> PolicyPrediction: result = self._default_predictions(domain) - tracker_as_states = self.featurizer.prediction_states( - [tracker], domain, for_only_ml_policy=True - ) - states = tracker_as_states[0] + states = self.prediction_states(tracker, domain) logger.debug(f"Current tracker state:{self.format_tracker_states(states)}") predicted_action_name = self.recall(states, tracker, domain) if predicted_action_name is not None: @@ -311,10 +308,7 @@ def _recall_using_delorean(self, old_states, tracker, domain) -> Optional[Text]: mcfly_tracker = self._back_to_the_future(tracker) while mcfly_tracker is not None: - tracker_as_states = self.featurizer.prediction_states( - [mcfly_tracker], domain, for_only_ml_policy=True - ) - states = tracker_as_states[0] + states = self.prediction_states(mcfly_tracker, domain) if old_states != states: # check if we like new futures diff --git a/rasa/core/policies/policy.py b/rasa/core/policies/policy.py index 0294705baa33..11dc0eabdfa5 100644 --- a/rasa/core/policies/policy.py +++ b/rasa/core/policies/policy.py @@ -23,7 +23,7 @@ import rasa.shared.utils.common import rasa.utils.common import rasa.shared.utils.io -from rasa.shared.core.domain import Domain +from rasa.shared.core.domain import Domain, State from rasa.core.featurizers.single_state_featurizer import SingleStateFeaturizer from rasa.core.featurizers.tracker_featurizers import ( TrackerFeaturizer, @@ -36,7 +36,6 @@ from rasa.core.constants import DEFAULT_POLICY_PRIORITY from rasa.shared.core.constants import USER, SLOTS, PREVIOUS_ACTION, ACTIVE_LOOP from rasa.shared.nlu.constants import ENTITIES, INTENT, TEXT, ACTION_TEXT, ACTION_NAME -from rasa.utils.tensorflow.constants import EPOCHS if TYPE_CHECKING: from rasa.shared.nlu.training_data.features import Features @@ -193,6 +192,62 @@ def featurize_for_training( return state_features, label_ids, entity_tags + def prediction_states( + self, + tracker: DialogueStateTracker, + domain: Domain, + use_text_for_last_user_input: bool = False, + ) -> List[State]: + """Transforms list of trackers to lists of states for prediction. + + Args: + tracker: The tracker to be featurized. + domain: The Domain. + use_text_for_last_user_input: Indicates whether to use text or intent label + for featurizing last user input. + + Returns: + A list of states. + """ + return self.featurizer.prediction_states( + [tracker], + domain, + use_text_for_last_user_input=use_text_for_last_user_input, + for_only_ml_policy=self.supported_data() == SupportedData.ML_DATA, + )[0] + + def featurize_for_prediction( + self, + tracker: DialogueStateTracker, + domain: Domain, + interpreter: NaturalLanguageInterpreter, + use_text_for_last_user_input: bool = False, + ) -> List[List[Dict[Text, List["Features"]]]]: + """Transform training trackers into a vector representation. + + The trackers, consisting of multiple turns, will be transformed + into a float vector which can be used by a ML model. + + Args: + tracker: The tracker to be featurized. + domain: The Domain. + interpreter: The NLU interpreter. + use_text_for_last_user_input: Indicates whether to use text or intent label + for featurizing last user input. + + Returns: + A dictionary of attribute (INTENT, TEXT, ACTION_NAME, ACTION_TEXT, + ENTITIES, SLOTS, FORM) to a list of features for all dialogue turns in + the tracker. + """ + return self.featurizer.create_state_features( + [tracker], + domain, + interpreter, + use_text_for_last_user_input=use_text_for_last_user_input, + for_only_ml_policy=self.supported_data() == SupportedData.ML_DATA, + ) + def train( self, training_trackers: List[TrackerWithCachedStates], diff --git a/rasa/core/policies/rule_policy.py b/rasa/core/policies/rule_policy.py index ce7566c224a9..10ddd509f04c 100644 --- a/rasa/core/policies/rule_policy.py +++ b/rasa/core/policies/rule_policy.py @@ -916,10 +916,7 @@ def _find_action_from_rules( # the text or the intent return None, None, False - tracker_as_states = self.featurizer.prediction_states( - [tracker], domain, use_text_for_last_user_input, for_only_ml_policy=False - ) - states = tracker_as_states[0] + states = self.prediction_states(tracker, domain, use_text_for_last_user_input) current_states = self.format_tracker_states(states) logger.debug(f"Current tracker state:{current_states}") diff --git a/rasa/core/policies/sklearn_policy.py b/rasa/core/policies/sklearn_policy.py index 2e39656d19a7..9f9acf98e9ff 100644 --- a/rasa/core/policies/sklearn_policy.py +++ b/rasa/core/policies/sklearn_policy.py @@ -284,9 +284,7 @@ def predict_action_probabilities( interpreter: NaturalLanguageInterpreter, **kwargs: Any, ) -> PolicyPrediction: - X = self.featurizer.create_state_features( - [tracker], domain, interpreter, for_only_ml_policy=True - ) + X = self.featurize_for_prediction(tracker, domain, interpreter) training_data, _ = model_data_utils.convert_to_data_format( X, self.zero_state_features ) diff --git a/rasa/core/policies/ted_policy.py b/rasa/core/policies/ted_policy.py index 3d25ddb230ac..3bf8f62f0e7f 100644 --- a/rasa/core/policies/ted_policy.py +++ b/rasa/core/policies/ted_policy.py @@ -538,12 +538,8 @@ def _featurize_tracker_for_e2e( # and second - an optional one (see conditions below), # the first example in the constructed batch either does not contain user input # or uses intent or text based on whether TED is e2e only. - tracker_state_features = self.featurizer.create_state_features( - [tracker], - domain, - interpreter, - use_text_for_last_user_input=self.only_e2e, - for_only_ml_policy=True, + tracker_state_features = self.featurize_for_prediction( + tracker, domain, interpreter, use_text_for_last_user_input=self.only_e2e, ) # the second - text, but only after user utterance and if not only e2e if ( @@ -551,12 +547,8 @@ def _featurize_tracker_for_e2e( and TEXT in self.fake_features and not self.only_e2e ): - tracker_state_features += self.featurizer.create_state_features( - [tracker], - domain, - interpreter, - use_text_for_last_user_input=True, - for_only_ml_policy=True, + tracker_state_features += self.featurize_for_prediction( + tracker, domain, interpreter, use_text_for_last_user_input=True, ) return tracker_state_features diff --git a/tests/core/test_policies.py b/tests/core/test_policies.py index 756c9c6e4feb..71a10eafb433 100644 --- a/tests/core/test_policies.py +++ b/tests/core/test_policies.py @@ -793,9 +793,7 @@ def test_memorise_with_nlu( tracker = DialogueStateTracker(dialogue.name, default_domain.slots) tracker.recreate_from_dialogue(dialogue) - states = trained_policy.featurizer.prediction_states( - [tracker], default_domain, for_only_ml_policy=True - )[0] + states = trained_policy.prediction_states(tracker, default_domain) recalled = trained_policy.recall(states, tracker, default_domain) assert recalled is not None From fa77daa2fb7d8309138b6c0a4fa73e0f9562260e Mon Sep 17 00:00:00 2001 From: Vova Vv Date: Tue, 12 Jan 2021 13:10:29 +0100 Subject: [PATCH 12/68] additionaly check states in the test --- tests/core/policies/test_rule_policy.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tests/core/policies/test_rule_policy.py b/tests/core/policies/test_rule_policy.py index bb3e770a1a27..cb10a54cf7a9 100644 --- a/tests/core/policies/test_rule_policy.py +++ b/tests/core/policies/test_rule_policy.py @@ -2099,6 +2099,23 @@ def test_hide_rule_turn_with_slots(): assert some_slot in prediction.optional_events[0].only_rule_slots assert some_other_slot not in prediction.optional_events[0].only_rule_slots + conversation_events += prediction.optional_events + conversation_events += [ + ActionExecuted(ACTION_LISTEN_NAME), + UserUttered("haha", {"name": some_other_action}), + ] + tracker = DialogueStateTracker.from_events( + "casd", evts=conversation_events, slots=domain.slots + ) + states = tracker.past_states(domain, for_only_ml_policy=True) + assert states == [ + {}, + { + USER: {TEXT: "haha", INTENT: some_other_action}, + PREVIOUS_ACTION: {ACTION_NAME: ACTION_LISTEN_NAME}, + }, + ] + def test_hide_rule_turn_with_loops(): form_name = "some_form" From fc92ab2a704b16488c247115a7eefd2af4999670 Mon Sep 17 00:00:00 2001 From: Vova Vv Date: Thu, 14 Jan 2021 10:10:00 +0100 Subject: [PATCH 13/68] add a test for a rule without action listen in the end --- rasa/shared/core/domain.py | 2 +- tests/core/policies/test_rule_policy.py | 80 ++++++++++++++++++++++++- 2 files changed, 80 insertions(+), 2 deletions(-) diff --git a/rasa/shared/core/domain.py b/rasa/shared/core/domain.py index efeba3b48b74..0f094a274658 100644 --- a/rasa/shared/core/domain.py +++ b/rasa/shared/core/domain.py @@ -1139,7 +1139,7 @@ def states_for_tracker_history( rasa.shared.core.constants.PREVIOUS_ACTION ] = last_ml_action_sub_state - states.append(state) + states.append(self._clean_state(state)) return states diff --git a/tests/core/policies/test_rule_policy.py b/tests/core/policies/test_rule_policy.py index cb10a54cf7a9..c0fb3e6064e6 100644 --- a/tests/core/policies/test_rule_policy.py +++ b/tests/core/policies/test_rule_policy.py @@ -1960,7 +1960,7 @@ def test_hide_rule_turn(): ) policy = RulePolicy() policy.train( - [GREET_RULE, chitchat_story,], domain, RegexInterpreter(), + [GREET_RULE, chitchat_story], domain, RegexInterpreter(), ) conversation_events = [ @@ -2117,6 +2117,84 @@ def test_hide_rule_turn_with_slots(): ] +def test_hide_rule_turn_no_last_action_listen(): + action_after_chitchat = "action_after_chitchat" + chitchat = "chitchat" + action_chitchat = "action_chitchat" + followup_on_chitchat = "followup_on_chitchat" + domain = Domain.from_yaml( + f""" + intents: + - {chitchat} + actions: + - {action_chitchat} + - {action_after_chitchat} + slots: + {followup_on_chitchat}: + type: bool + """ + ) + simple_rule_no_last_action_listen = TrackerWithCachedStates.from_events( + "simple rule without action listen in the end", + domain=domain, + slots=domain.slots, + evts=[ + ActionExecuted(RULE_SNIPPET_ACTION_NAME), + ActionExecuted(action_chitchat), + SlotSet(followup_on_chitchat, True), + ActionExecuted(action_after_chitchat), + ActionExecuted(RULE_SNIPPET_ACTION_NAME), + ], + is_rule_tracker=True, + ) + chitchat_story = TrackerWithCachedStates.from_events( + "chitchat story", + domain=domain, + slots=domain.slots, + evts=[ + ActionExecuted(ACTION_LISTEN_NAME), + UserUttered(intent={"name": chitchat}), + ActionExecuted(action_chitchat), + ActionExecuted(ACTION_LISTEN_NAME), + ], + ) + policy = RulePolicy() + policy.train( + [simple_rule_no_last_action_listen, chitchat_story], domain, RegexInterpreter(), + ) + + conversation_events = [ + ActionExecuted(ACTION_LISTEN_NAME), + UserUttered(intent={"name": chitchat}), + ActionExecuted(action_chitchat), + SlotSet(followup_on_chitchat, True), + ] + prediction = policy.predict_action_probabilities( + DialogueStateTracker.from_events( + "casd", evts=conversation_events, slots=domain.slots + ), + domain, + RegexInterpreter(), + ) + assert_predicted_action(prediction, domain, action_after_chitchat) + assert isinstance(prediction.optional_events[0], HideRuleTurn) + assert followup_on_chitchat in prediction.optional_events[0].only_rule_slots + + conversation_events += prediction.optional_events + conversation_events += [ + ActionExecuted(action_after_chitchat), + ] + tracker = DialogueStateTracker.from_events( + "casd", evts=conversation_events, slots=domain.slots + ) + states = tracker.past_states(domain, for_only_ml_policy=True) + assert states == [ + {}, + {USER: {INTENT: chitchat}, PREVIOUS_ACTION: {ACTION_NAME: ACTION_LISTEN_NAME}}, + {USER: {INTENT: chitchat}, PREVIOUS_ACTION: {ACTION_NAME: action_chitchat}}, + ] + + def test_hide_rule_turn_with_loops(): form_name = "some_form" another_form_name = "another_form" From 7bc11dec70635196efa1ea5dfe4af14f01b7850c Mon Sep 17 00:00:00 2001 From: Vladimir Vlasov Date: Wed, 20 Jan 2021 19:53:58 +0100 Subject: [PATCH 14/68] Update rasa/core/policies/policy.py Co-authored-by: Tobias Wochinger --- rasa/core/policies/policy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rasa/core/policies/policy.py b/rasa/core/policies/policy.py index 11dc0eabdfa5..50f2dcccc210 100644 --- a/rasa/core/policies/policy.py +++ b/rasa/core/policies/policy.py @@ -223,7 +223,7 @@ def featurize_for_prediction( interpreter: NaturalLanguageInterpreter, use_text_for_last_user_input: bool = False, ) -> List[List[Dict[Text, List["Features"]]]]: - """Transform training trackers into a vector representation. + """Transforms training tracker into a vector representation. The trackers, consisting of multiple turns, will be transformed into a float vector which can be used by a ML model. From 0d43cd4d6cc0242857e623178f5e5a1163548d33 Mon Sep 17 00:00:00 2001 From: Vladimir Vlasov Date: Wed, 20 Jan 2021 19:54:43 +0100 Subject: [PATCH 15/68] Update rasa/core/policies/ted_policy.py Co-authored-by: Tobias Wochinger --- rasa/core/policies/ted_policy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rasa/core/policies/ted_policy.py b/rasa/core/policies/ted_policy.py index 3bf8f62f0e7f..43b329509c1b 100644 --- a/rasa/core/policies/ted_policy.py +++ b/rasa/core/policies/ted_policy.py @@ -571,8 +571,8 @@ def _pick_confidence( non_e2e_action_name = domain.action_names_or_texts[ np.argmax(confidences[0]) ] - e2e_action_name = domain.action_names_or_texts[np.argmax(confidences[1])] logger.debug(f"User intent lead to '{non_e2e_action_name}'.") + e2e_action_name = domain.action_names_or_texts[np.argmax(confidences[1])] logger.debug(f"User text lead to '{e2e_action_name}'.") if ( np.max(confidences[1]) > self.config[E2E_CONFIDENCE_THRESHOLD] From f4d5d21382a3a6c0993cd8be8ea21b2735a3a241 Mon Sep 17 00:00:00 2001 From: Vladimir Vlasov Date: Thu, 21 Jan 2021 10:27:57 +0100 Subject: [PATCH 16/68] Update rasa/core/policies/policy.py Co-authored-by: Tobias Wochinger --- rasa/core/policies/policy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rasa/core/policies/policy.py b/rasa/core/policies/policy.py index 50f2dcccc210..42715ceab794 100644 --- a/rasa/core/policies/policy.py +++ b/rasa/core/policies/policy.py @@ -198,7 +198,7 @@ def prediction_states( domain: Domain, use_text_for_last_user_input: bool = False, ) -> List[State]: - """Transforms list of trackers to lists of states for prediction. + """Transforms tracker to states for prediction. Args: tracker: The tracker to be featurized. From 02078c9a2cdf1b6703262fb81044057ad7a7d66e Mon Sep 17 00:00:00 2001 From: Vladimir Vlasov Date: Thu, 21 Jan 2021 10:31:56 +0100 Subject: [PATCH 17/68] Update rasa/core/featurizers/tracker_featurizers.py Co-authored-by: Tobias Wochinger --- rasa/core/featurizers/tracker_featurizers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rasa/core/featurizers/tracker_featurizers.py b/rasa/core/featurizers/tracker_featurizers.py index b4ba43180923..4f712c465e36 100644 --- a/rasa/core/featurizers/tracker_featurizers.py +++ b/rasa/core/featurizers/tracker_featurizers.py @@ -59,7 +59,7 @@ def _create_states( Args: tracker: a :class:`rasa.core.trackers.DialogueStateTracker` domain: a :class:`rasa.shared.core.domain.Domain` - for_only_ml_policy: If True ignore dialogue turns that are present + ignore_rule_only_turns: If `True` ignore dialogue turns that are present only in rules. Returns: From 2d3b0935596fbc154591867cd14a859f90770606 Mon Sep 17 00:00:00 2001 From: Vova Vv Date: Thu, 21 Jan 2021 10:38:32 +0100 Subject: [PATCH 18/68] rename for_only_ml to ignore_rule_only ... --- rasa/core/featurizers/tracker_featurizers.py | 28 +++++++++++--------- rasa/core/policies/policy.py | 4 +-- rasa/core/policies/rule_policy.py | 14 +++++----- rasa/shared/core/constants.py | 3 +++ rasa/shared/core/domain.py | 20 +++++++------- rasa/shared/core/events.py | 26 +++++++++--------- rasa/shared/core/generator.py | 4 +-- rasa/shared/core/trackers.py | 10 +++---- tests/core/policies/test_rule_policy.py | 22 +++++++-------- 9 files changed, 69 insertions(+), 62 deletions(-) diff --git a/rasa/core/featurizers/tracker_featurizers.py b/rasa/core/featurizers/tracker_featurizers.py index 4f712c465e36..3ba039995787 100644 --- a/rasa/core/featurizers/tracker_featurizers.py +++ b/rasa/core/featurizers/tracker_featurizers.py @@ -52,7 +52,9 @@ def __init__( @staticmethod def _create_states( - tracker: DialogueStateTracker, domain: Domain, for_only_ml_policy: bool = False + tracker: DialogueStateTracker, + domain: Domain, + ignore_rule_only_turns: bool = False, ) -> List[State]: """Create states for the given tracker. @@ -65,7 +67,7 @@ def _create_states( Returns: a list of states """ - return tracker.past_states(domain, for_only_ml_policy) + return tracker.past_states(domain, ignore_rule_only_turns) def _featurize_states( self, @@ -239,7 +241,7 @@ def prediction_states( trackers: List[DialogueStateTracker], domain: Domain, use_text_for_last_user_input: bool = False, - for_only_ml_policy: bool = False, + ignore_rule_only_turns: bool = False, ) -> List[List[State]]: """Transforms list of trackers to lists of states for prediction. @@ -248,7 +250,7 @@ def prediction_states( domain: The domain. use_text_for_last_user_input: Indicates whether to use text or intent label for featurizing last user input. - for_only_ml_policy: If True ignore dialogue turns that are present + ignore_rule_only_turns: If True ignore dialogue turns that are present only in rules. Returns: @@ -264,7 +266,7 @@ def create_state_features( domain: Domain, interpreter: NaturalLanguageInterpreter, use_text_for_last_user_input: bool = False, - for_only_ml_policy: bool = False, + ignore_rule_only_turns: bool = False, ) -> List[List[Dict[Text, List["Features"]]]]: """Create state features for prediction. @@ -274,7 +276,7 @@ def create_state_features( interpreter: The interpreter use_text_for_last_user_input: Indicates whether to use text or intent label for featurizing last user input. - for_only_ml_policy: If True ignore dialogue turns that are present + ignore_rule_only_turns: If True ignore dialogue turns that are present only in rules. Returns: @@ -283,7 +285,7 @@ def create_state_features( turns in all trackers. """ trackers_as_states = self.prediction_states( - trackers, domain, use_text_for_last_user_input, for_only_ml_policy + trackers, domain, use_text_for_last_user_input, ignore_rule_only_turns ) return self._featurize_states(trackers_as_states, interpreter) @@ -403,7 +405,7 @@ def prediction_states( trackers: List[DialogueStateTracker], domain: Domain, use_text_for_last_user_input: bool = False, - for_only_ml_policy: bool = False, + ignore_rule_only_turns: bool = False, ) -> List[List[State]]: """Transforms list of trackers to lists of states for prediction. @@ -412,14 +414,14 @@ def prediction_states( domain: The domain. use_text_for_last_user_input: Indicates whether to use text or intent label for featurizing last user input. - for_only_ml_policy: If True ignore dialogue turns that are present + ignore_rule_only_turns: If True ignore dialogue turns that are present only in rules. Returns: A list of states. """ trackers_as_states = [ - self._create_states(tracker, domain, for_only_ml_policy) + self._create_states(tracker, domain, ignore_rule_only_turns) for tracker in trackers ] self._choose_last_user_input(trackers_as_states, use_text_for_last_user_input) @@ -562,7 +564,7 @@ def prediction_states( trackers: List[DialogueStateTracker], domain: Domain, use_text_for_last_user_input: bool = False, - for_only_ml_policy: bool = False, + ignore_rule_only_turns: bool = False, ) -> List[List[State]]: """Transforms list of trackers to lists of states for prediction. @@ -571,14 +573,14 @@ def prediction_states( domain: The domain. use_text_for_last_user_input: Indicates whether to use text or intent label for featurizing last user input. - for_only_ml_policy: If True ignore dialogue turns that are present + ignore_rule_only_turns: If True ignore dialogue turns that are present only in rules. Returns: A list of states. """ trackers_as_states = [ - self._create_states(tracker, domain, for_only_ml_policy) + self._create_states(tracker, domain, ignore_rule_only_turns) for tracker in trackers ] trackers_as_states = [ diff --git a/rasa/core/policies/policy.py b/rasa/core/policies/policy.py index 42715ceab794..93e8b6f57dcd 100644 --- a/rasa/core/policies/policy.py +++ b/rasa/core/policies/policy.py @@ -213,7 +213,7 @@ def prediction_states( [tracker], domain, use_text_for_last_user_input=use_text_for_last_user_input, - for_only_ml_policy=self.supported_data() == SupportedData.ML_DATA, + ignore_rule_only_turns=self.supported_data() == SupportedData.ML_DATA, )[0] def featurize_for_prediction( @@ -245,7 +245,7 @@ def featurize_for_prediction( domain, interpreter, use_text_for_last_user_input=use_text_for_last_user_input, - for_only_ml_policy=self.supported_data() == SupportedData.ML_DATA, + ignore_rule_only_turns=self.supported_data() == SupportedData.ML_DATA, ) def train( diff --git a/rasa/core/policies/rule_policy.py b/rasa/core/policies/rule_policy.py index 10ddd509f04c..f5919aed7e55 100644 --- a/rasa/core/policies/rule_policy.py +++ b/rasa/core/policies/rule_policy.py @@ -42,6 +42,8 @@ LOOP_NAME, SLOTS, ACTIVE_LOOP, + RULE_ONLY_SLOTS, + RULE_ONLY_LOOPS, ) from rasa.shared.core.domain import InvalidDomain, State, Domain from rasa.shared.nlu.constants import ACTION_NAME, INTENT_NAME_KEY @@ -64,8 +66,6 @@ RULES = "rules" RULES_FOR_LOOP_UNHAPPY_PATH = "rules_for_loop_unhappy_path" RULES_NOT_IN_STORIES = "rules_not_in_stories" -ONLY_RULE_SLOTS = "only_rule_slots" -ONLY_RULE_LOOPS = "only_rule_loops" LOOP_WAS_INTERRUPTED = "loop_was_interrupted" DO_NOT_PREDICT_LOOP_ACTION = "do_not_predict_loop_action" @@ -426,7 +426,7 @@ def _get_slots_loops_from_states( loops.add(active_loop) return slots, loops - def _find_only_rule_slots_loops( + def _find_rule_only_slots_loops( self, rule_trackers_as_states: List[List[State]], story_trackers_as_states: List[List[State]], @@ -681,9 +681,9 @@ def _create_lookup_from_trackers( if self._check_for_contradictions: ( - self.lookup[ONLY_RULE_SLOTS], - self.lookup[ONLY_RULE_LOOPS], - ) = self._find_only_rule_slots_loops( + self.lookup[RULE_ONLY_SLOTS], + self.lookup[RULE_ONLY_LOOPS], + ) = self._find_rule_only_slots_loops( rule_trackers_as_states, story_trackers_as_states ) @@ -1086,7 +1086,7 @@ def _rule_prediction( if self._prediction_source in self.lookup.get(RULES_NOT_IN_STORIES, []): # the prediction is based on rules that are not present in the stories optional_events = [ - HideRuleTurn(self.lookup[ONLY_RULE_SLOTS], self.lookup[ONLY_RULE_LOOPS]) + HideRuleTurn(self.lookup[RULE_ONLY_SLOTS], self.lookup[RULE_ONLY_LOOPS]) ] return self._prediction( diff --git a/rasa/shared/core/constants.py b/rasa/shared/core/constants.py index 35dd052770ff..cfd1195a51bd 100644 --- a/rasa/shared/core/constants.py +++ b/rasa/shared/core/constants.py @@ -79,3 +79,6 @@ USE_TEXT_FOR_FEATURIZATION = "use_text_for_featurization" ENTITY_LABEL_SEPARATOR = "#" + +RULE_ONLY_SLOTS = "rule_only_slots" +RULE_ONLY_LOOPS = "rule_only_loops" diff --git a/rasa/shared/core/domain.py b/rasa/shared/core/domain.py index 0f094a274658..32d65bb62d90 100644 --- a/rasa/shared/core/domain.py +++ b/rasa/shared/core/domain.py @@ -1069,11 +1069,11 @@ def get_active_states(self, tracker: "DialogueStateTracker") -> State: return self._clean_state(state) @staticmethod - def _remove_only_rule_features( + def _remove_rule_only_features( state: State, tracker: "DialogueStateTracker" ) -> None: # remove only rule slots - for slot in tracker.only_rule_slots: + for slot in tracker.rule_only_slots: if state.get(rasa.shared.core.constants.SLOTS, {}).get(slot): del state[rasa.shared.core.constants.SLOTS][slot] # remove active loop feature if it is only rule loop @@ -1081,12 +1081,12 @@ def _remove_only_rule_features( state.get(rasa.shared.core.constants.ACTIVE_LOOP, {}).get( rasa.shared.core.constants.LOOP_NAME ) - in tracker.only_rule_loops + in tracker.rule_only_loops ): del state[rasa.shared.core.constants.ACTIVE_LOOP] @staticmethod - def _substitute_only_rule_user_input(state: State, last_ml_state: State) -> None: + def _substitute_rule_only_user_input(state: State, last_ml_state: State) -> None: if not rasa.shared.core.trackers.is_prev_action_listen_in_state(state): if not last_ml_state.get(rasa.shared.core.constants.USER) and state.get( rasa.shared.core.constants.USER @@ -1098,13 +1098,13 @@ def _substitute_only_rule_user_input(state: State, last_ml_state: State) -> None ] def states_for_tracker_history( - self, tracker: "DialogueStateTracker", for_only_ml_policy: bool = False + self, tracker: "DialogueStateTracker", ignore_rule_only_turns: bool = False ) -> List[State]: """Array of states for each state of the trackers history. Args: tracker: Instance of `DialogueStateTracker` to featurize. - for_only_ml_policy: If True ignore dialogue turns that are present + ignore_rule_only_turns: If True ignore dialogue turns that are present only in rules. Return: @@ -1114,7 +1114,7 @@ def states_for_tracker_history( last_ml_action_sub_state = None last_action_is_ml_action = True for tr in tracker.generate_all_prior_trackers(): - if for_only_ml_policy: + if ignore_rule_only_turns: # remember previous ml action based on the last non hidden turn # we need this to override previous action in the ml state if last_action_is_ml_action: @@ -1126,13 +1126,13 @@ def states_for_tracker_history( state = self.get_active_states(tr) - if for_only_ml_policy: + if ignore_rule_only_turns: # clean state from only rule features - self._remove_only_rule_features(state, tracker) + self._remove_rule_only_features(state, tracker) # make sure user input is the same as for previous state # for non action_listen turns if states: - self._substitute_only_rule_user_input(state, states[-1]) + self._substitute_rule_only_user_input(state, states[-1]) # substitute previous rule action with last_ml_action_sub_state if last_ml_action_sub_state: state[ diff --git a/rasa/shared/core/events.py b/rasa/shared/core/events.py index df39dd2ec584..150859bebc7b 100644 --- a/rasa/shared/core/events.py +++ b/rasa/shared/core/events.py @@ -25,6 +25,8 @@ ENTITY_LABEL_SEPARATOR, ACTION_SESSION_START_NAME, ACTION_LISTEN_NAME, + RULE_ONLY_SLOTS, + RULE_ONLY_LOOPS, ) from rasa.shared.exceptions import UnsupportedFeatureException from rasa.shared.nlu.constants import ( @@ -1896,24 +1898,24 @@ class HideRuleTurn(SkipEventInMDStoryMixin, AlwaysEqualEventMixin): def __init__( self, - only_rule_slots: List[Text], - only_rule_loops: List[Text], + rule_only_slots: List[Text], + rule_only_loops: List[Text], timestamp: Optional[float] = None, metadata: Optional[Dict[Text, Any]] = None, ) -> None: """Initializes event. Args: - only_rule_slots: The list of slot names, + rule_only_slots: The list of slot names, SlotSet events for which shouldn't be hidden - only_rule_loops: The list of loop names, + rule_only_loops: The list of loop names, ActiveLoop events for which shouldn't be hidden timestamp: the timestamp metadata: some optional metadata """ super().__init__(timestamp, metadata) - self.only_rule_slots = only_rule_slots - self.only_rule_loops = only_rule_loops + self.rule_only_slots = rule_only_slots + self.rule_only_loops = rule_only_loops def __hash__(self) -> int: """Returns unique hash for event.""" @@ -1922,8 +1924,8 @@ def __hash__(self) -> int: @classmethod def _from_parameters(cls, parameters: Dict[Text, Any]) -> "HideRuleTurn": return HideRuleTurn( - parameters.get("only_rule_slots"), - parameters.get("only_rule_loops"), + parameters.get(RULE_ONLY_SLOTS), + parameters.get(RULE_ONLY_LOOPS), parameters.get("timestamp"), parameters.get("metadata"), ) @@ -1937,8 +1939,8 @@ def as_dict(self) -> Dict[Text, Any]: d = super().as_dict() d.update( { - "only_rule_slots": self.only_rule_slots, - "only_rule_loops": self.only_rule_loops, + RULE_ONLY_SLOTS: self.rule_only_slots, + RULE_ONLY_LOOPS: self.rule_only_loops, } ) return d @@ -1953,5 +1955,5 @@ def apply_to(self, tracker: "DialogueStateTracker") -> None: # we will reset it on each action executed tracker.hide_rule_turn = True # only rule slots and loops are always the same for all the trackers - tracker.only_rule_slots = self.only_rule_slots - tracker.only_rule_loops = self.only_rule_loops + tracker.rule_only_slots = self.rule_only_slots + tracker.rule_only_loops = self.rule_only_loops diff --git a/rasa/shared/core/generator.py b/rasa/shared/core/generator.py index 10248cb700cc..8eb0384b0173 100644 --- a/rasa/shared/core/generator.py +++ b/rasa/shared/core/generator.py @@ -108,13 +108,13 @@ def _unfreeze_states(frozen_states: Deque[FrozenState]) -> List[State]: ] def past_states( - self, domain: Domain, for_only_ml_policy: bool = False + self, domain: Domain, ignore_rule_only_turns: bool = False ) -> List[State]: """Generate the past states of this tracker based on the history. Args: domain: a :class:`rasa.shared.core.domain.Domain`. - for_only_ml_policy: If True ignore dialogue turns that are present + ignore_rule_only_turns: If True ignore dialogue turns that are present only in rules. Returns: diff --git a/rasa/shared/core/trackers.py b/rasa/shared/core/trackers.py index 4b68c7d610c3..a220ccbfc773 100644 --- a/rasa/shared/core/trackers.py +++ b/rasa/shared/core/trackers.py @@ -207,8 +207,8 @@ def __init__( self._reset() self.active_loop: Dict[Text, Union[Text, bool, Dict, None]] = {} self.hide_rule_turn = False - self.only_rule_slots = [] - self.only_rule_loops = [] + self.rule_only_slots = [] + self.rule_only_loops = [] ### # Public tracker interface @@ -278,19 +278,19 @@ def freeze_current_state(state: State) -> FrozenState: ) def past_states( - self, domain: Domain, for_only_ml_policy: bool = False + self, domain: Domain, ignore_rule_only_turns: bool = False ) -> List[State]: """Generate the past states of this tracker based on the history. Args: domain: a :class:`rasa.shared.core.domain.Domain`. - for_only_ml_policy: If True ignore dialogue turns that are present + ignore_rule_only_turns: If True ignore dialogue turns that are present only in rules. Returns: a list of states """ - return domain.states_for_tracker_history(self, for_only_ml_policy) + return domain.states_for_tracker_history(self, ignore_rule_only_turns) def change_loop_to(self, loop_name: Optional[Text]) -> None: """Set the currently active loop. diff --git a/tests/core/policies/test_rule_policy.py b/tests/core/policies/test_rule_policy.py index c0fb3e6064e6..5c32c6404af1 100644 --- a/tests/core/policies/test_rule_policy.py +++ b/tests/core/policies/test_rule_policy.py @@ -1999,7 +1999,7 @@ def test_hide_rule_turn(): tracker = DialogueStateTracker.from_events( "casd", evts=conversation_events, slots=domain.slots ) - states = tracker.past_states(domain, for_only_ml_policy=True) + states = tracker.past_states(domain, ignore_rule_only_turns=True) assert states == [ {}, { @@ -2096,8 +2096,8 @@ def test_hide_rule_turn_with_slots(): ) assert_predicted_action(prediction, domain, some_action) assert isinstance(prediction.optional_events[0], HideRuleTurn) - assert some_slot in prediction.optional_events[0].only_rule_slots - assert some_other_slot not in prediction.optional_events[0].only_rule_slots + assert some_slot in prediction.optional_events[0].rule_only_slots + assert some_other_slot not in prediction.optional_events[0].rule_only_slots conversation_events += prediction.optional_events conversation_events += [ @@ -2107,7 +2107,7 @@ def test_hide_rule_turn_with_slots(): tracker = DialogueStateTracker.from_events( "casd", evts=conversation_events, slots=domain.slots ) - states = tracker.past_states(domain, for_only_ml_policy=True) + states = tracker.past_states(domain, ignore_rule_only_turns=True) assert states == [ {}, { @@ -2178,7 +2178,7 @@ def test_hide_rule_turn_no_last_action_listen(): ) assert_predicted_action(prediction, domain, action_after_chitchat) assert isinstance(prediction.optional_events[0], HideRuleTurn) - assert followup_on_chitchat in prediction.optional_events[0].only_rule_slots + assert followup_on_chitchat in prediction.optional_events[0].rule_only_slots conversation_events += prediction.optional_events conversation_events += [ @@ -2187,7 +2187,7 @@ def test_hide_rule_turn_no_last_action_listen(): tracker = DialogueStateTracker.from_events( "casd", evts=conversation_events, slots=domain.slots ) - states = tracker.past_states(domain, for_only_ml_policy=True) + states = tracker.past_states(domain, ignore_rule_only_turns=True) assert states == [ {}, {USER: {INTENT: chitchat}, PREVIOUS_ACTION: {ACTION_NAME: ACTION_LISTEN_NAME}}, @@ -2265,8 +2265,8 @@ def test_hide_rule_turn_with_loops(): ) assert_predicted_action(prediction, domain, form_name) assert isinstance(prediction.optional_events[0], HideRuleTurn) - assert form_name in prediction.optional_events[0].only_rule_loops - assert another_form_name not in prediction.optional_events[0].only_rule_loops + assert form_name in prediction.optional_events[0].rule_only_loops + assert another_form_name not in prediction.optional_events[0].rule_only_loops conversation_events += prediction.optional_events conversation_events += [ @@ -2282,8 +2282,8 @@ def test_hide_rule_turn_with_loops(): ) assert_predicted_action(prediction, domain, ACTION_LISTEN_NAME) assert isinstance(prediction.optional_events[0], HideRuleTurn) - assert form_name in prediction.optional_events[0].only_rule_loops - assert another_form_name not in prediction.optional_events[0].only_rule_loops + assert form_name in prediction.optional_events[0].rule_only_loops + assert another_form_name not in prediction.optional_events[0].rule_only_loops conversation_events += prediction.optional_events conversation_events += [ @@ -2293,7 +2293,7 @@ def test_hide_rule_turn_with_loops(): tracker = DialogueStateTracker.from_events( "casd", evts=conversation_events, slots=domain.slots ) - states = tracker.past_states(domain, for_only_ml_policy=True) + states = tracker.past_states(domain, ignore_rule_only_turns=True) assert states == [ {}, { From edd0ae771c926b8447e4dfa21c2fccdd11f1ee31 Mon Sep 17 00:00:00 2001 From: Vova Vv Date: Fri, 22 Jan 2021 17:59:26 +0100 Subject: [PATCH 19/68] fix overriding self._prediction_source --- rasa/core/policies/rule_policy.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/rasa/core/policies/rule_policy.py b/rasa/core/policies/rule_policy.py index f5919aed7e55..b386474e1cf6 100644 --- a/rasa/core/policies/rule_policy.py +++ b/rasa/core/policies/rule_policy.py @@ -1006,7 +1006,7 @@ def predict_action_probabilities( """Predicts the next action (see parent class for more information).""" ( rules_action_name_from_text, - self._prediction_source, + prediction_source, returning_from_unhappy_path_from_text, ) = self._find_action_from_rules( tracker, domain, use_text_for_last_user_input=True @@ -1017,12 +1017,13 @@ def predict_action_probabilities( # accordingly. ( default_action_name, - self._prediction_source, + prediction_source, ) = self._find_action_from_default_actions(tracker) # text has priority over intents including default, # however loop happy path has priority over rules prediction if default_action_name and not rules_action_name_from_text: + self._prediction_source = prediction_source return self._rule_prediction( self._prediction_result(default_action_name, tracker, domain) ) @@ -1033,9 +1034,10 @@ def predict_action_probabilities( # simply force predict the loop. ( loop_happy_path_action_name, - self._prediction_source, + prediction_source, ) = self._find_action_from_loop_happy_path(tracker) if loop_happy_path_action_name: + self._prediction_source = prediction_source # this prediction doesn't use user input # and happy user input anyhow should be ignored during featurization return self._rule_prediction( @@ -1044,6 +1046,7 @@ def predict_action_probabilities( # predict rules from text first if rules_action_name_from_text: + self._prediction_source = prediction_source return self._rule_prediction( self._prediction_result(rules_action_name_from_text, tracker, domain), returning_from_unhappy_path=returning_from_unhappy_path_from_text, From 753152152b5ccf596256561e3b295d6ba8a35e75 Mon Sep 17 00:00:00 2001 From: Vova Vv Date: Mon, 25 Jan 2021 10:30:25 +0100 Subject: [PATCH 20/68] fix prediction source from text --- rasa/core/policies/rule_policy.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/rasa/core/policies/rule_policy.py b/rasa/core/policies/rule_policy.py index b386474e1cf6..f4e5ebe40223 100644 --- a/rasa/core/policies/rule_policy.py +++ b/rasa/core/policies/rule_policy.py @@ -1006,7 +1006,7 @@ def predict_action_probabilities( """Predicts the next action (see parent class for more information).""" ( rules_action_name_from_text, - prediction_source, + prediction_source_from_text, returning_from_unhappy_path_from_text, ) = self._find_action_from_rules( tracker, domain, use_text_for_last_user_input=True @@ -1017,13 +1017,12 @@ def predict_action_probabilities( # accordingly. ( default_action_name, - prediction_source, + self._prediction_source, ) = self._find_action_from_default_actions(tracker) # text has priority over intents including default, # however loop happy path has priority over rules prediction if default_action_name and not rules_action_name_from_text: - self._prediction_source = prediction_source return self._rule_prediction( self._prediction_result(default_action_name, tracker, domain) ) @@ -1034,10 +1033,9 @@ def predict_action_probabilities( # simply force predict the loop. ( loop_happy_path_action_name, - prediction_source, + self._prediction_source, ) = self._find_action_from_loop_happy_path(tracker) if loop_happy_path_action_name: - self._prediction_source = prediction_source # this prediction doesn't use user input # and happy user input anyhow should be ignored during featurization return self._rule_prediction( @@ -1046,7 +1044,7 @@ def predict_action_probabilities( # predict rules from text first if rules_action_name_from_text: - self._prediction_source = prediction_source + self._prediction_source = prediction_source_from_text return self._rule_prediction( self._prediction_result(rules_action_name_from_text, tracker, domain), returning_from_unhappy_path=returning_from_unhappy_path_from_text, From b651ace0f151ec038415c432cbacdfd2c77f4fdd Mon Sep 17 00:00:00 2001 From: Vova Vv Date: Mon, 25 Jan 2021 11:35:34 +0100 Subject: [PATCH 21/68] remove in-loop events --- rasa/core/policies/ted_policy.py | 4 ++-- rasa/shared/core/trackers.py | 11 ++++++++++- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/rasa/core/policies/ted_policy.py b/rasa/core/policies/ted_policy.py index 43b329509c1b..8b5ae08098fa 100644 --- a/rasa/core/policies/ted_policy.py +++ b/rasa/core/policies/ted_policy.py @@ -539,7 +539,7 @@ def _featurize_tracker_for_e2e( # the first example in the constructed batch either does not contain user input # or uses intent or text based on whether TED is e2e only. tracker_state_features = self.featurize_for_prediction( - tracker, domain, interpreter, use_text_for_last_user_input=self.only_e2e, + tracker, domain, interpreter, use_text_for_last_user_input=self.only_e2e ) # the second - text, but only after user utterance and if not only e2e if ( @@ -548,7 +548,7 @@ def _featurize_tracker_for_e2e( and not self.only_e2e ): tracker_state_features += self.featurize_for_prediction( - tracker, domain, interpreter, use_text_for_last_user_input=True, + tracker, domain, interpreter, use_text_for_last_user_input=True ) return tracker_state_features diff --git a/rasa/shared/core/trackers.py b/rasa/shared/core/trackers.py index a220ccbfc773..0bec338655b3 100644 --- a/rasa/shared/core/trackers.py +++ b/rasa/shared/core/trackers.py @@ -60,6 +60,7 @@ ActionExecutionRejected, EntitiesAdded, HideRuleTurn, + DefinePrevUserUtteredFeaturization, ) from rasa.shared.core.domain import Domain, State from rasa.shared.core.slots import Slot @@ -564,7 +565,15 @@ def _undo_till_previous_loop_execution( if isinstance(e, ActionExecuted) and e.action_name == loop_action_name: break - if isinstance(e, (ActionExecuted, UserUttered)): + if isinstance( + e, + ( + ActionExecuted, + UserUttered, + HideRuleTurn, + DefinePrevUserUtteredFeaturization, + ), + ): del done_events[-1 - offset] else: # Remember events which aren't unfeaturized to get the index right From fde0e7b9b5645ecd2dab3f85596cfd67a79304ef Mon Sep 17 00:00:00 2001 From: Vova Vv Date: Mon, 25 Jan 2021 12:58:54 +0100 Subject: [PATCH 22/68] fix test --- tests/core/test_processor.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/core/test_processor.py b/tests/core/test_processor.py index 553edcf66774..9d38caac2403 100644 --- a/tests/core/test_processor.py +++ b/tests/core/test_processor.py @@ -42,6 +42,7 @@ DefinePrevUserUtteredFeaturization, ActionExecutionRejected, LoopInterrupted, + HideRuleTurn, ) from rasa.core.interpreter import RasaNLUHttpInterpreter from rasa.shared.nlu.interpreter import NaturalLanguageInterpreter, RegexInterpreter @@ -907,6 +908,7 @@ async def test_restart_triggers_session_start( BotUttered("hey there name1!", metadata={"template_name": "utter_greet"}), ActionExecuted(ACTION_LISTEN_NAME), UserUttered("/restart", {INTENT_NAME_KEY: "restart", "confidence": 1.0}), + HideRuleTurn([], []), DefinePrevUserUtteredFeaturization(use_text_for_featurization=False), ActionExecuted(ACTION_RESTART_NAME), Restarted(), From 58c8b9c516506bf190d92e24590deb2ade72ff12 Mon Sep 17 00:00:00 2001 From: Vova Vv Date: Wed, 3 Feb 2021 15:50:40 +0100 Subject: [PATCH 23/68] fix test --- examples/e2ebot/config.yml | 1 + tests/core/policies/test_rule_policy.py | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/examples/e2ebot/config.yml b/examples/e2ebot/config.yml index f38558adb0ad..50eb460d8c7e 100644 --- a/examples/e2ebot/config.yml +++ b/examples/e2ebot/config.yml @@ -2,6 +2,7 @@ language: en pipeline: - name: WhitespaceTokenizer intent_tokenization_flag: True + - name: LanguageModelFeaturizer - name: RegexFeaturizer - name: LexicalSyntacticFeaturizer - name: CountVectorsFeaturizer diff --git a/tests/core/policies/test_rule_policy.py b/tests/core/policies/test_rule_policy.py index 6b731d26de55..d35bd5b513ad 100644 --- a/tests/core/policies/test_rule_policy.py +++ b/tests/core/policies/test_rule_policy.py @@ -2288,7 +2288,9 @@ def test_hide_rule_turn_with_loops(): domain, RegexInterpreter(), ) - assert_predicted_action(prediction, domain, ACTION_LISTEN_NAME) + assert_predicted_action( + prediction, domain, ACTION_LISTEN_NAME, is_no_user_prediction=True + ) assert isinstance(prediction.optional_events[0], HideRuleTurn) assert form_name in prediction.optional_events[0].rule_only_loops assert another_form_name not in prediction.optional_events[0].rule_only_loops From 23c37fa11707584b29824bda73f008ceb489cff8 Mon Sep 17 00:00:00 2001 From: Vladimir Vlasov Date: Mon, 8 Feb 2021 16:34:56 +0100 Subject: [PATCH 24/68] Update rasa/shared/core/generator.py Co-authored-by: Tobias Wochinger --- rasa/shared/core/generator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rasa/shared/core/generator.py b/rasa/shared/core/generator.py index 8eb0384b0173..3f5ffeee01db 100644 --- a/rasa/shared/core/generator.py +++ b/rasa/shared/core/generator.py @@ -110,7 +110,7 @@ def _unfreeze_states(frozen_states: Deque[FrozenState]) -> List[State]: def past_states( self, domain: Domain, ignore_rule_only_turns: bool = False ) -> List[State]: - """Generate the past states of this tracker based on the history. + """Generates the past states of this tracker based on the history. Args: domain: a :class:`rasa.shared.core.domain.Domain`. From 631d73cc1ce3f721e2ccff13275df9ca0435c106 Mon Sep 17 00:00:00 2001 From: Vladimir Vlasov Date: Mon, 8 Feb 2021 16:35:06 +0100 Subject: [PATCH 25/68] Update rasa/shared/core/events.py Co-authored-by: Tobias Wochinger --- rasa/shared/core/events.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rasa/shared/core/events.py b/rasa/shared/core/events.py index 150859bebc7b..0ae0ae8403de 100644 --- a/rasa/shared/core/events.py +++ b/rasa/shared/core/events.py @@ -1886,7 +1886,7 @@ def apply_to(self, tracker: "DialogueStateTracker") -> None: class HideRuleTurn(SkipEventInMDStoryMixin, AlwaysEqualEventMixin): - """Emulate undoing of the last dialogue turn. + """Emulates undoing of the last dialogue turn. The bot reverts everything until before the most recent action. This includes the action itself, as well as any events that From 4b70aa69e1be8f9afacc465898aa43f3ddd710c3 Mon Sep 17 00:00:00 2001 From: Vova Vv Date: Tue, 9 Feb 2021 19:31:55 +0100 Subject: [PATCH 26/68] fix hiding followup action and add test --- rasa/core/policies/rule_policy.py | 2 +- rasa/shared/core/domain.py | 15 +++- rasa/shared/core/events.py | 5 +- tests/core/policies/test_rule_policy.py | 113 +++++++++++++++++++++++- 4 files changed, 128 insertions(+), 7 deletions(-) diff --git a/rasa/core/policies/rule_policy.py b/rasa/core/policies/rule_policy.py index 3be886610b08..a0928ed69d79 100644 --- a/rasa/core/policies/rule_policy.py +++ b/rasa/core/policies/rule_policy.py @@ -967,7 +967,7 @@ def _find_action_from_rules( ) return ( active_loop_name, - LOOP_RULES + active_loop_name, + best_rule_key, returning_from_unhappy_path, ) diff --git a/rasa/shared/core/domain.py b/rasa/shared/core/domain.py index f714819e267f..5875c7aa1265 100644 --- a/rasa/shared/core/domain.py +++ b/rasa/shared/core/domain.py @@ -1112,16 +1112,23 @@ def states_for_tracker_history( """ states = [] last_ml_action_sub_state = None - last_action_is_ml_action = True + turn_was_hidden = False for tr in tracker.generate_all_prior_trackers(): if ignore_rule_only_turns: # remember previous ml action based on the last non hidden turn # we need this to override previous action in the ml state - if last_action_is_ml_action: + if not turn_was_hidden: last_ml_action_sub_state = self._get_prev_action_sub_state(tr) - last_action_is_ml_action = not tr.hide_rule_turn - if tr.hide_rule_turn: + # followup action or happy path loop prediction + # don't change the fact whether dialogue turn should be hidden + if ( + not tr.followup_action + and not tr.latest_action_name == tr.active_loop_name + ): + turn_was_hidden = tr.hide_rule_turn + + if turn_was_hidden: continue state = self.get_active_states(tr) diff --git a/rasa/shared/core/events.py b/rasa/shared/core/events.py index 150859bebc7b..0566ab84f4ad 100644 --- a/rasa/shared/core/events.py +++ b/rasa/shared/core/events.py @@ -1545,9 +1545,12 @@ def apply_to(self, tracker: "DialogueStateTracker") -> None: """Applies event to current conversation state.""" tracker.set_latest_action(self.as_sub_state()) tracker.clear_followup_action() + # HideRuleTurn event is added by RulePolicy before actual action is executed, # so we can safely reset it on each action executed - tracker.hide_rule_turn = False + # unless it is active_loop prediction + if not self.action_name == tracker.active_loop_name: + tracker.hide_rule_turn = False class AgentUttered(SkipEventInMDStoryMixin): diff --git a/tests/core/policies/test_rule_policy.py b/tests/core/policies/test_rule_policy.py index d35bd5b513ad..c53ace95456a 100644 --- a/tests/core/policies/test_rule_policy.py +++ b/tests/core/policies/test_rule_policy.py @@ -1,5 +1,5 @@ from pathlib import Path -from typing import Text, Optional +from typing import Text import pytest @@ -22,6 +22,8 @@ REQUESTED_SLOT, USER, PREVIOUS_ACTION, + ACTIVE_LOOP, + LOOP_NAME, ) from rasa.shared.nlu.constants import TEXT, INTENT, ACTION_NAME from rasa.shared.core.domain import Domain @@ -33,6 +35,7 @@ ActionExecutionRejected, LoopInterrupted, HideRuleTurn, + FollowupAction, ) from rasa.shared.nlu.interpreter import RegexInterpreter from rasa.core.nlg import TemplatedNaturalLanguageGenerator @@ -2311,3 +2314,111 @@ def test_hide_rule_turn_with_loops(): PREVIOUS_ACTION: {ACTION_NAME: ACTION_LISTEN_NAME}, }, ] + + +def test_hide_rule_turn_with_loops_as_followup_action(): + form_name = "some_form" + activate_form = "activate_form" + domain = Domain.from_yaml( + f""" + intents: + - {GREET_INTENT_NAME} + - {activate_form} + actions: + - {UTTER_GREET_ACTION} + slots: + {REQUESTED_SLOT}: + type: unfeaturized + forms: + - {form_name} + """ + ) + + form_activation_rule = _form_activation_rule(domain, form_name, activate_form) + form_activation_story = form_activation_rule.copy() + form_activation_story.is_rule_tracker = False + + policy = RulePolicy() + policy.train( + [form_activation_rule, GREET_RULE, form_activation_story], + domain, + RegexInterpreter(), + ) + + conversation_events = [ + ActionExecuted(ACTION_LISTEN_NAME), + UserUttered("haha", {"name": activate_form}), + ] + prediction = policy.predict_action_probabilities( + DialogueStateTracker.from_events( + "casd", evts=conversation_events, slots=domain.slots + ), + domain, + RegexInterpreter(), + ) + assert_predicted_action(prediction, domain, form_name) + assert not prediction.optional_events + + conversation_events += [ + ActionExecuted(form_name), + ActiveLoop(form_name), + ] + prediction = policy.predict_action_probabilities( + DialogueStateTracker.from_events( + "casd", evts=conversation_events, slots=domain.slots + ), + domain, + RegexInterpreter(), + ) + assert_predicted_action( + prediction, domain, ACTION_LISTEN_NAME, is_no_user_prediction=True + ) + assert not prediction.optional_events + + conversation_events += [ + ActionExecuted(ACTION_LISTEN_NAME), + UserUttered("haha", {"name": GREET_INTENT_NAME}), + ActionExecutionRejected(form_name), + ] + prediction = policy.predict_action_probabilities( + DialogueStateTracker.from_events( + "casd", evts=conversation_events, slots=domain.slots + ), + domain, + RegexInterpreter(), + ) + assert_predicted_action(prediction, domain, UTTER_GREET_ACTION) + assert isinstance(prediction.optional_events[0], HideRuleTurn) + + conversation_events += prediction.optional_events + conversation_events += [ + ActionExecuted(UTTER_GREET_ACTION), + FollowupAction(form_name), + ActionExecuted(form_name), + ] + prediction = policy.predict_action_probabilities( + DialogueStateTracker.from_events( + "casd", evts=conversation_events, slots=domain.slots + ), + domain, + RegexInterpreter(), + ) + assert_predicted_action( + prediction, domain, ACTION_LISTEN_NAME, is_no_user_prediction=True + ) + tracker = DialogueStateTracker.from_events( + "casd", evts=conversation_events, slots=domain.slots + ) + states = tracker.past_states(domain, ignore_rule_only_turns=True) + assert states == [ + {}, + { + USER: {TEXT: "haha", INTENT: activate_form}, + PREVIOUS_ACTION: {ACTION_NAME: ACTION_LISTEN_NAME}, + }, + { + USER: {TEXT: "haha", INTENT: activate_form}, + PREVIOUS_ACTION: {ACTION_NAME: form_name}, + ACTIVE_LOOP: {LOOP_NAME: form_name}, + }, + ] From 06b93600d391074a2ca48a90a48b4df72e9d0e0f Mon Sep 17 00:00:00 2001 From: Vladimir Vlasov Date: Wed, 10 Feb 2021 14:13:48 +0100 Subject: [PATCH 27/68] Update tests/core/policies/test_rule_policy.py Co-authored-by: Tobias Wochinger --- tests/core/policies/test_rule_policy.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/core/policies/test_rule_policy.py b/tests/core/policies/test_rule_policy.py index c53ace95456a..03453ed5b33c 100644 --- a/tests/core/policies/test_rule_policy.py +++ b/tests/core/policies/test_rule_policy.py @@ -2107,8 +2107,7 @@ def test_hide_rule_turn_with_slots(): ) assert_predicted_action(prediction, domain, some_action) assert isinstance(prediction.optional_events[0], HideRuleTurn) - assert some_slot in prediction.optional_events[0].rule_only_slots - assert some_other_slot not in prediction.optional_events[0].rule_only_slots + assert prediction.optional_events[0].rule_only_slots == [some_slot] conversation_events += prediction.optional_events conversation_events += [ From 0a1c24401386d830202a17c3ada48544ce3de2d6 Mon Sep 17 00:00:00 2001 From: Vladimir Vlasov Date: Wed, 10 Feb 2021 14:16:51 +0100 Subject: [PATCH 28/68] Update rasa/shared/core/domain.py Co-authored-by: Tobias Wochinger --- rasa/shared/core/domain.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rasa/shared/core/domain.py b/rasa/shared/core/domain.py index 5875c7aa1265..4fa64209b441 100644 --- a/rasa/shared/core/domain.py +++ b/rasa/shared/core/domain.py @@ -1072,7 +1072,7 @@ def get_active_states(self, tracker: "DialogueStateTracker") -> State: def _remove_rule_only_features( state: State, tracker: "DialogueStateTracker" ) -> None: - # remove only rule slots + # remove slots which only occur in rules but not in stories for slot in tracker.rule_only_slots: if state.get(rasa.shared.core.constants.SLOTS, {}).get(slot): del state[rasa.shared.core.constants.SLOTS][slot] From 33ec86362cd8b34728a95b0ea7adac0cd02ebcbf Mon Sep 17 00:00:00 2001 From: Vladimir Vlasov Date: Wed, 10 Feb 2021 14:17:43 +0100 Subject: [PATCH 29/68] Update rasa/shared/core/domain.py Co-authored-by: Tobias Wochinger --- rasa/shared/core/domain.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rasa/shared/core/domain.py b/rasa/shared/core/domain.py index 4fa64209b441..1afb02c5b84c 100644 --- a/rasa/shared/core/domain.py +++ b/rasa/shared/core/domain.py @@ -1076,7 +1076,7 @@ def _remove_rule_only_features( for slot in tracker.rule_only_slots: if state.get(rasa.shared.core.constants.SLOTS, {}).get(slot): del state[rasa.shared.core.constants.SLOTS][slot] - # remove active loop feature if it is only rule loop + # remove active loop which only occur in rules but not in stories if ( state.get(rasa.shared.core.constants.ACTIVE_LOOP, {}).get( rasa.shared.core.constants.LOOP_NAME From c34d61a9caa911b1f3eba88d41a40384acbe89e0 Mon Sep 17 00:00:00 2001 From: Vova Vv Date: Wed, 10 Feb 2021 14:55:41 +0100 Subject: [PATCH 30/68] get rid of self._prediction_source as any policy needs to be stateless --- rasa/core/policies/rule_policy.py | 126 ++++++++++++++++++++++-------- 1 file changed, 95 insertions(+), 31 deletions(-) diff --git a/rasa/core/policies/rule_policy.py b/rasa/core/policies/rule_policy.py index a0928ed69d79..d74b48a35413 100644 --- a/rasa/core/policies/rule_policy.py +++ b/rasa/core/policies/rule_policy.py @@ -14,6 +14,7 @@ UserUttered, ActionExecuted, HideRuleTurn, + Event, ) from rasa.core.featurizers.tracker_featurizers import TrackerFeaturizer from rasa.shared.nlu.interpreter import NaturalLanguageInterpreter @@ -144,7 +145,6 @@ def __init__( self._restrict_rules = restrict_rules self._check_for_contradictions = check_for_contradictions - self._prediction_source = None self._rules_sources = None # max history is set to `None` in order to capture any lengths of rule stories @@ -449,10 +449,9 @@ def _predict_next_action( tracker: TrackerWithCachedStates, domain: Domain, interpreter: NaturalLanguageInterpreter, - ) -> Optional[Text]: - probabilities = self.predict_action_probabilities( - tracker, domain, interpreter - ).probabilities + ) -> Tuple[Optional[Text], Optional[Text]]: + prediction = self.predict_action_probabilities(tracker, domain, interpreter) + probabilities = prediction.probabilities # do not raise an error if RulePolicy didn't predict anything for stories; # however for rules RulePolicy should always predict an action predicted_action_name = None @@ -464,7 +463,7 @@ def _predict_next_action( np.argmax(probabilities) ] - return predicted_action_name + return predicted_action_name, prediction.prediction_source def _predicted_action_name( self, @@ -472,8 +471,10 @@ def _predicted_action_name( domain: Domain, interpreter: NaturalLanguageInterpreter, gold_action_name: Text, - ) -> Optional[Text]: - predicted_action_name = self._predict_next_action(tracker, domain, interpreter) + ) -> Tuple[Optional[Text], Optional[Text]]: + predicted_action_name, prediction_source = self._predict_next_action( + tracker, domain, interpreter + ) # if there is an active_loop, # RulePolicy will always predict active_loop first, # but inside loop unhappy path there might be another action @@ -483,31 +484,30 @@ def _predicted_action_name( and predicted_action_name == tracker.active_loop_name ): rasa.core.test.emulate_loop_rejection(tracker) - predicted_action_name = self._predict_next_action( + predicted_action_name, prediction_source = self._predict_next_action( tracker, domain, interpreter ) - return predicted_action_name + return predicted_action_name, prediction_source def _collect_sources( self, tracker: TrackerWithCachedStates, predicted_action_name: Optional[Text], gold_action_name: Text, + prediction_source: Optional[Text], ) -> None: # we need to remember which action should be predicted by the rule # in order to correctly output the names of the contradicting rules rule_name = tracker.sender_id - if self._prediction_source.startswith( - DEFAULT_RULES - ) or self._prediction_source.startswith(LOOP_RULES): + if prediction_source.startswith(DEFAULT_RULES) or prediction_source.startswith( + LOOP_RULES + ): # the real gold action contradict the one in the rules in this case gold_action_name = predicted_action_name - rule_name = self._prediction_source + rule_name = prediction_source - self._rules_sources[self._prediction_source].append( - (rule_name, gold_action_name) - ) + self._rules_sources[prediction_source].append((rule_name, gold_action_name)) @staticmethod def _default_sources() -> Set[Text]: @@ -529,6 +529,7 @@ def _check_prediction( tracker: TrackerWithCachedStates, predicted_action_name: Optional[Text], gold_action_name: Text, + prediction_source: Optional[Text], ) -> List[Text]: if not predicted_action_name or predicted_action_name == gold_action_name: return [] @@ -536,7 +537,7 @@ def _check_prediction( tracker_type = "rule" if tracker.is_rule_tracker else "story" contradicting_rules = { rule_name - for rule_name, action_name in self._rules_sources[self._prediction_source] + for rule_name, action_name in self._rules_sources[prediction_source] if action_name != gold_action_name } @@ -592,12 +593,15 @@ def _run_prediction_on_trackers( continue gold_action_name = event.action_name or event.action_text - predicted_action_name = self._predicted_action_name( + predicted_action_name, prediction_source = self._predicted_action_name( running_tracker, domain, interpreter, gold_action_name ) if collect_sources: self._collect_sources( - running_tracker, predicted_action_name, gold_action_name + running_tracker, + predicted_action_name, + gold_action_name, + prediction_source, ) else: # to be able to remove only rules turns from the dialogue history @@ -607,10 +611,13 @@ def _run_prediction_on_trackers( not tracker.is_rule_tracker and predicted_action_name == gold_action_name ): - rules_used_in_stories.add(self._prediction_source) + rules_used_in_stories.add(prediction_source) error_messages += self._check_prediction( - running_tracker, predicted_action_name, gold_action_name + running_tracker, + predicted_action_name, + gold_action_name, + prediction_source, ) running_tracker.update(event) @@ -1002,7 +1009,7 @@ def predict_action_probabilities( domain: Domain, interpreter: NaturalLanguageInterpreter, **kwargs: Any, - ) -> PolicyPrediction: + ) -> "RulePolicyPrediction": """Predicts the next action (see parent class for more information).""" ( rules_action_name_from_text, @@ -1017,14 +1024,15 @@ def predict_action_probabilities( # accordingly. ( default_action_name, - self._prediction_source, + default_prediction_source, ) = self._find_action_from_default_actions(tracker) # text has priority over intents including default, # however loop happy path has priority over rules prediction if default_action_name and not rules_action_name_from_text: return self._rule_prediction( - self._prediction_result(default_action_name, tracker, domain) + self._prediction_result(default_action_name, tracker, domain), + default_prediction_source, ) # A loop has priority over any other rule except defaults. @@ -1033,21 +1041,22 @@ def predict_action_probabilities( # simply force predict the loop. ( loop_happy_path_action_name, - self._prediction_source, + loop_happy_path_prediction_source, ) = self._find_action_from_loop_happy_path(tracker) if loop_happy_path_action_name: # this prediction doesn't use user input # and happy user input anyhow should be ignored during featurization return self._rule_prediction( self._prediction_result(loop_happy_path_action_name, tracker, domain), + loop_happy_path_prediction_source, is_no_user_prediction=True, ) # predict rules from text first if rules_action_name_from_text: - self._prediction_source = prediction_source_from_text return self._rule_prediction( self._prediction_result(rules_action_name_from_text, tracker, domain), + prediction_source_from_text, returning_from_unhappy_path=returning_from_unhappy_path_from_text, is_end_to_end_prediction=True, ) @@ -1055,7 +1064,7 @@ def predict_action_probabilities( ( rules_action_name_from_intent, # we want to remember the source even if rules didn't predict any action - self._prediction_source, + prediction_source_from_intent, returning_from_unhappy_path_from_intent, ) = self._find_action_from_rules( tracker, domain, use_text_for_last_user_input=False @@ -1069,6 +1078,7 @@ def predict_action_probabilities( return self._rule_prediction( probabilities, + prediction_source_from_intent, returning_from_unhappy_path=( # returning_from_unhappy_path is a negative condition, # so `or` should be applied @@ -1081,23 +1091,27 @@ def predict_action_probabilities( def _rule_prediction( self, probabilities: List[float], + prediction_source: Text, returning_from_unhappy_path: bool = False, is_end_to_end_prediction: bool = False, is_no_user_prediction: bool = False, - ) -> "PolicyPrediction": + ) -> "RulePolicyPrediction": optional_events = [] - if self._prediction_source in self.lookup.get(RULES_NOT_IN_STORIES, []): + if prediction_source in self.lookup.get(RULES_NOT_IN_STORIES, []): # the prediction is based on rules that are not present in the stories optional_events = [ HideRuleTurn(self.lookup[RULE_ONLY_SLOTS], self.lookup[RULE_ONLY_LOOPS]) ] - return self._prediction( + return RulePolicyPrediction( probabilities, + self.__class__.__name__, + self.priority, events=[LoopInterrupted(True)] if returning_from_unhappy_path else [], optional_events=optional_events, is_end_to_end_prediction=is_end_to_end_prediction, is_no_user_prediction=is_no_user_prediction, + prediction_source=prediction_source, ) def _default_predictions(self, domain: Domain) -> List[float]: @@ -1121,3 +1135,53 @@ def _metadata(self) -> Dict[Text, Any]: @classmethod def _metadata_filename(cls) -> Text: return "rule_policy.json" + + +class RulePolicyPrediction(PolicyPrediction): + def __init__( + self, + probabilities: List[float], + policy_name: Optional[Text], + policy_priority: int = 1, + events: Optional[List[Event]] = None, + optional_events: Optional[List[Event]] = None, + is_end_to_end_prediction: bool = False, + is_no_user_prediction: bool = False, + diagnostic_data: Optional[Dict[Text, Any]] = None, + prediction_source: Optional[Text] = None, + ) -> None: + """Creates a `RulePolicyPrediction`. + + Args: + probabilities: The probabilities for each action. + policy_name: Name of the policy which made the prediction. + policy_priority: The priority of the policy which made the prediction. + events: Events which the `Policy` needs to have applied to the tracker + after the prediction. These events are applied independent of whether + the policy wins against other policies or not. Be careful which events + you return as they can potentially influence the conversation flow. + optional_events: Events which the `Policy` needs to have applied to the + tracker after the prediction in case it wins. These events are only + applied in case the policy's prediction wins. Be careful which events + you return as they can potentially influence the conversation flow. + is_end_to_end_prediction: `True` if the prediction used the text of the + user message instead of the intent. + is_no_user_prediction: `True` if the prediction uses neither the text + of the user message nor the intent. This is for the example the case + for happy loop paths. + diagnostic_data: Intermediate results or other information that is not + necessary for Rasa to function, but intended for debugging and + fine-tuning purposes. + prediction_source: A description of the matching rule. + """ + self.prediction_source = prediction_source + super().__init__( + probabilities, + policy_name, + policy_priority, + events, + optional_events, + is_end_to_end_prediction, + is_no_user_prediction, + diagnostic_data, + ) From 3f79e385fcd046cf14de74618e680e1ade7fd37f Mon Sep 17 00:00:00 2001 From: Vova Vv Date: Wed, 10 Feb 2021 15:24:44 +0100 Subject: [PATCH 31/68] refactor find contradicting rules --- rasa/core/policies/rule_policy.py | 49 ++++++++++++++++++++++++++----- 1 file changed, 42 insertions(+), 7 deletions(-) diff --git a/rasa/core/policies/rule_policy.py b/rasa/core/policies/rule_policy.py index d74b48a35413..95d4d6ca1b33 100644 --- a/rasa/core/policies/rule_policy.py +++ b/rasa/core/policies/rule_policy.py @@ -624,13 +624,47 @@ def _run_prediction_on_trackers( return error_messages, rules_used_in_stories - def _find_contradicting_rules( + def _collect_rule_sources( + self, + rule_trackers: List[TrackerWithCachedStates], + domain: Domain, + interpreter: NaturalLanguageInterpreter, + ) -> None: + self._run_prediction_on_trackers( + rule_trackers, domain, interpreter, collect_sources=True + ) + + def _find_contradicting_and_used_in_stories_rules( + self, + trackers: List[TrackerWithCachedStates], + domain: Domain, + interpreter: NaturalLanguageInterpreter, + ) -> Tuple[List[Text], Set[Text]]: + return self._run_prediction_on_trackers( + trackers, domain, interpreter, collect_sources=False + ) + + def _analyze_rules( self, rule_trackers: List[TrackerWithCachedStates], all_trackers: List[TrackerWithCachedStates], domain: Domain, interpreter: NaturalLanguageInterpreter, ) -> List[Text]: + """Analyze learned rules by running prediction on training trackers. + + This method collects error messages for contradicting rules + and creates the lookup for rules that are not present in the stories. + + Args: + rule_trackers: The list of the rule trackers. + all_trackers: The list of all trackers. + domain: The domain. + interpreter: Interpreter which can be used by the polices for featurization. + + Returns: + The list of rules that are not present in the stories. + """ logger.debug("Started checking rules and stories for contradictions.") # during training we run `predict_action_probabilities` to check for # contradicting rules. @@ -640,11 +674,12 @@ def _find_contradicting_rules( # we need to run prediction on rule trackers twice, because we need to collect # the information about which rule snippets contributed to the learned rules - self._run_prediction_on_trackers( - rule_trackers, domain, interpreter, collect_sources=True - ) - error_messages, rules_used_in_stories = self._run_prediction_on_trackers( - all_trackers, domain, interpreter, collect_sources=False + self._collect_rule_sources(rule_trackers, domain, interpreter) + ( + error_messages, + rules_used_in_stories, + ) = self._find_contradicting_and_used_in_stories_rules( + all_trackers, domain, interpreter ) logger.setLevel(logger_level) # reset logger level @@ -741,7 +776,7 @@ def train( if self._check_for_contradictions: # using trackers here might not be the most efficient way, however # it allows us to directly test `predict_action_probabilities` method - self.lookup[RULES_NOT_IN_STORIES] = self._find_contradicting_rules( + self.lookup[RULES_NOT_IN_STORIES] = self._analyze_rules( rule_trackers, training_trackers, domain, interpreter ) From e263efb918590c9c0b0053b83cb43e704a72eb5b Mon Sep 17 00:00:00 2001 From: Vova Vv Date: Wed, 10 Feb 2021 15:27:25 +0100 Subject: [PATCH 32/68] add comment about default sources --- rasa/core/policies/rule_policy.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/rasa/core/policies/rule_policy.py b/rasa/core/policies/rule_policy.py index 95d4d6ca1b33..84c45224d4e3 100644 --- a/rasa/core/policies/rule_policy.py +++ b/rasa/core/policies/rule_policy.py @@ -891,6 +891,8 @@ def _find_action_from_default_actions( logger.debug(f"Predicted default action '{default_action_name}'.") return ( default_action_name, + # create prediction source that corresponds to one of + # default prediction sources in `_default_sources()` DEFAULT_RULES + tracker.latest_message.intent.get(INTENT_NAME_KEY), ) From be8c1904453708ca183fa5054aa1be4b29ebcbdf Mon Sep 17 00:00:00 2001 From: Vova Vv Date: Wed, 10 Feb 2021 15:30:03 +0100 Subject: [PATCH 33/68] update docstring --- rasa/core/featurizers/tracker_featurizers.py | 4 +++- rasa/core/policies/policy.py | 8 +++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/rasa/core/featurizers/tracker_featurizers.py b/rasa/core/featurizers/tracker_featurizers.py index c32b9292d690..f8baa40d4cc9 100644 --- a/rasa/core/featurizers/tracker_featurizers.py +++ b/rasa/core/featurizers/tracker_featurizers.py @@ -288,7 +288,9 @@ def create_state_features( only in rules. Returns: - A dictionary of state type (INTENT, TEXT, ACTION_NAME, ACTION_TEXT, + A list (corresponds to the list of trackers) + of lists (corresponds to all dialogue turns) + of dictionaries of state type (INTENT, TEXT, ACTION_NAME, ACTION_TEXT, ENTITIES, SLOTS, ACTIVE_LOOP) to a list of features for all dialogue turns in all trackers. """ diff --git a/rasa/core/policies/policy.py b/rasa/core/policies/policy.py index dfc5d045e3c9..4f45f1a77dbc 100644 --- a/rasa/core/policies/policy.py +++ b/rasa/core/policies/policy.py @@ -238,9 +238,11 @@ def featurize_for_prediction( for featurizing last user input. Returns: - A dictionary of attribute (INTENT, TEXT, ACTION_NAME, ACTION_TEXT, - ENTITIES, SLOTS, FORM) to a list of features for all dialogue turns in - the tracker. + A list (corresponds to the list of trackers) + of lists (corresponds to all dialogue turns) + of dictionaries of state type (INTENT, TEXT, ACTION_NAME, ACTION_TEXT, + ENTITIES, SLOTS, ACTIVE_LOOP) to a list of features for all dialogue + turns in all trackers. """ return self.featurizer.create_state_features( [tracker], From 5470a64a2874e773957dcbd2b29b6f981163d1dd Mon Sep 17 00:00:00 2001 From: Vova Vv Date: Wed, 10 Feb 2021 15:32:22 +0100 Subject: [PATCH 34/68] update docstring --- rasa/shared/core/events.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/rasa/shared/core/events.py b/rasa/shared/core/events.py index c69d887ef03a..db810a4ef197 100644 --- a/rasa/shared/core/events.py +++ b/rasa/shared/core/events.py @@ -1893,8 +1893,9 @@ class HideRuleTurn(SkipEventInMDStoryMixin, AlwaysEqualEventMixin): The bot reverts everything until before the most recent action. This includes the action itself, as well as any events that - action created, like set slot events, unless they are in a list - to reapply them. + action created, like set slot events (unless they are not in a list + of `rule_only_slots`) and active loop events (unless they are not in a list + of `rule_only_loops`). """ type_name = "hide_rule_turn" From 5e8162c77de33636502a9ca524b8a425d59a3d12 Mon Sep 17 00:00:00 2001 From: Vova Vv Date: Wed, 10 Feb 2021 15:33:44 +0100 Subject: [PATCH 35/68] update docstring --- rasa/shared/core/events.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rasa/shared/core/events.py b/rasa/shared/core/events.py index db810a4ef197..1a59ebd65c1e 100644 --- a/rasa/shared/core/events.py +++ b/rasa/shared/core/events.py @@ -1911,9 +1911,9 @@ def __init__( Args: rule_only_slots: The list of slot names, - SlotSet events for which shouldn't be hidden + which only occur in rules but not in stories. rule_only_loops: The list of loop names, - ActiveLoop events for which shouldn't be hidden + which only occur in rules but not in stories. timestamp: the timestamp metadata: some optional metadata """ From a84c1f65f92e01c7d7c979f45470209c50148dff Mon Sep 17 00:00:00 2001 From: Vova Vv Date: Wed, 10 Feb 2021 15:40:38 +0100 Subject: [PATCH 36/68] add test to not hide loops that are in stories --- tests/core/policies/test_rule_policy.py | 57 +++++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/tests/core/policies/test_rule_policy.py b/tests/core/policies/test_rule_policy.py index 03453ed5b33c..715cef511b27 100644 --- a/tests/core/policies/test_rule_policy.py +++ b/tests/core/policies/test_rule_policy.py @@ -2315,6 +2315,63 @@ def test_hide_rule_turn_with_loops(): ] +def test_do_not_hide_rule_turn_with_loops_in_stories(): + form_name = "some_form" + activate_form = "activate_form" + domain = Domain.from_yaml( + f""" + intents: + - {activate_form} + slots: + {REQUESTED_SLOT}: + type: unfeaturized + forms: + - {form_name} + """ + ) + + form_activation_rule = _form_activation_rule(domain, form_name, activate_form) + form_activation_story = form_activation_rule.copy() + form_activation_story.is_rule_tracker = False + + policy = RulePolicy() + policy.train( + [form_activation_rule, form_activation_story], + domain, + RegexInterpreter(), + ) + + conversation_events = [ + ActionExecuted(ACTION_LISTEN_NAME), + UserUttered("haha", {"name": activate_form}), + ] + prediction = policy.predict_action_probabilities( + DialogueStateTracker.from_events( + "casd", evts=conversation_events, slots=domain.slots + ), + domain, + RegexInterpreter(), + ) + assert_predicted_action(prediction, domain, form_name) + assert not prediction.optional_events + + conversation_events += [ + ActionExecuted(form_name), + ActiveLoop(form_name), + ] + prediction = policy.predict_action_probabilities( + DialogueStateTracker.from_events( + "casd", evts=conversation_events, slots=domain.slots + ), + domain, + RegexInterpreter(), + ) + assert_predicted_action( + prediction, domain, ACTION_LISTEN_NAME, is_no_user_prediction=True + ) + assert not prediction.optional_events + + def test_hide_rule_turn_with_loops_as_followup_action(): form_name = "some_form" activate_form = "activate_form" From 9e19c584b93ea0297d6e66353bdb6cc8a54861ac Mon Sep 17 00:00:00 2001 From: Vova Vv Date: Wed, 10 Feb 2021 15:42:41 +0100 Subject: [PATCH 37/68] use loop rules separator constant --- rasa/core/policies/rule_policy.py | 10 ++++++++-- tests/core/policies/test_rule_policy.py | 4 +--- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/rasa/core/policies/rule_policy.py b/rasa/core/policies/rule_policy.py index 84c45224d4e3..b371d5767ba9 100644 --- a/rasa/core/policies/rule_policy.py +++ b/rasa/core/policies/rule_policy.py @@ -73,6 +73,7 @@ DEFAULT_RULES = "predicting default action with intent " LOOP_RULES = "handling active loops and forms - " +LOOP_RULES_SEPARATOR = " - " class InvalidRule(RasaException): @@ -521,7 +522,9 @@ def _handling_loop_sources(domain: Domain) -> Set[Text]: loop_sources = set() for loop_name in domain.form_names: loop_sources.add(LOOP_RULES + loop_name) - loop_sources.add(LOOP_RULES + loop_name + " - " + ACTION_LISTEN_NAME) + loop_sources.add( + LOOP_RULES + loop_name + LOOP_RULES_SEPARATOR + ACTION_LISTEN_NAME + ) return loop_sources def _check_prediction( @@ -927,7 +930,10 @@ def _find_action_from_loop_happy_path( ) return ( ACTION_LISTEN_NAME, - LOOP_RULES + active_loop_name + " - " + ACTION_LISTEN_NAME, + LOOP_RULES + + active_loop_name + + LOOP_RULES_SEPARATOR + + ACTION_LISTEN_NAME, ) return None, None diff --git a/tests/core/policies/test_rule_policy.py b/tests/core/policies/test_rule_policy.py index 715cef511b27..77c8222a5bf9 100644 --- a/tests/core/policies/test_rule_policy.py +++ b/tests/core/policies/test_rule_policy.py @@ -2336,9 +2336,7 @@ def test_do_not_hide_rule_turn_with_loops_in_stories(): policy = RulePolicy() policy.train( - [form_activation_rule, form_activation_story], - domain, - RegexInterpreter(), + [form_activation_rule, form_activation_story], domain, RegexInterpreter(), ) conversation_events = [ From fbfef6c5a4852a9aa9b1b7d46f5268e58a0d84b1 Mon Sep 17 00:00:00 2001 From: Vova Vv Date: Wed, 10 Feb 2021 15:44:05 +0100 Subject: [PATCH 38/68] update docstring --- rasa/core/policies/rule_policy.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/rasa/core/policies/rule_policy.py b/rasa/core/policies/rule_policy.py index b371d5767ba9..62028e3f2d59 100644 --- a/rasa/core/policies/rule_policy.py +++ b/rasa/core/policies/rule_policy.py @@ -1181,6 +1181,8 @@ def _metadata_filename(cls) -> Text: class RulePolicyPrediction(PolicyPrediction): + """Stores information about the prediction of a `RulePolicy`.""" + def __init__( self, probabilities: List[float], From 71e68d03865d1ca23e5ee564d0e5e6eb10b53186 Mon Sep 17 00:00:00 2001 From: Vova Vv Date: Mon, 15 Feb 2021 15:52:24 +0100 Subject: [PATCH 39/68] remove HideRuleTurnEvent --- rasa/core/actions/action.py | 6 +- rasa/core/featurizers/tracker_featurizers.py | 60 ++++++++++- rasa/core/policies/ensemble.py | 32 +++++- rasa/core/policies/form_policy.py | 6 +- rasa/core/policies/memoization.py | 24 +++-- rasa/core/policies/policy.py | 20 +++- rasa/core/policies/rule_policy.py | 18 ++-- rasa/core/policies/sklearn_policy.py | 2 +- rasa/core/policies/ted_policy.py | 15 ++- rasa/core/processor.py | 1 - rasa/shared/core/domain.py | 32 ++++-- rasa/shared/core/events.py | 85 +--------------- rasa/shared/core/generator.py | 6 +- rasa/shared/core/trackers.py | 32 +++--- tests/core/policies/test_rule_policy.py | 101 +++++++++++-------- tests/core/test_processor.py | 2 - 16 files changed, 254 insertions(+), 188 deletions(-) diff --git a/rasa/core/actions/action.py b/rasa/core/actions/action.py index fc5d3057e3f1..dc229f68d9d0 100644 --- a/rasa/core/actions/action.py +++ b/rasa/core/actions/action.py @@ -249,7 +249,10 @@ def event_for_successful_execution( Event which should be logged onto the tracker. """ return ActionExecuted( - self.name(), prediction.policy_name, prediction.max_confidence + self.name(), + prediction.policy_name, + prediction.max_confidence, + hide_rule_turn=prediction.hide_rule_turn, ) @@ -340,6 +343,7 @@ def event_for_successful_execution( policy=prediction.policy_name, confidence=prediction.max_confidence, action_text=self.action_text, + hide_rule_turn=prediction.hide_rule_turn, ) diff --git a/rasa/core/featurizers/tracker_featurizers.py b/rasa/core/featurizers/tracker_featurizers.py index f8baa40d4cc9..dbe1d5f5ca40 100644 --- a/rasa/core/featurizers/tracker_featurizers.py +++ b/rasa/core/featurizers/tracker_featurizers.py @@ -55,6 +55,8 @@ def _create_states( tracker: DialogueStateTracker, domain: Domain, ignore_rule_only_turns: bool = False, + rule_only_slots: Optional[List[Text]] = None, + rule_only_loops: Optional[List[Text]] = None, ) -> List[State]: """Create states for the given tracker. @@ -63,11 +65,20 @@ def _create_states( domain: a :class:`rasa.shared.core.domain.Domain` ignore_rule_only_turns: If `True` ignore dialogue turns that are present only in rules. + rule_only_slots: The list of slot names, + which only occur in rules but not in stories. + rule_only_loops: The list of loop names, + which only occur in rules but not in stories. Returns: a list of states """ - return tracker.past_states(domain, ignore_rule_only_turns) + return tracker.past_states( + domain, + ignore_rule_only_turns=ignore_rule_only_turns, + rule_only_slots=rule_only_slots, + rule_only_loops=rule_only_loops, + ) def _featurize_states( self, @@ -250,6 +261,8 @@ def prediction_states( domain: Domain, use_text_for_last_user_input: bool = False, ignore_rule_only_turns: bool = False, + rule_only_slots: Optional[List[Text]] = None, + rule_only_loops: Optional[List[Text]] = None, ) -> List[List[State]]: """Transforms list of trackers to lists of states for prediction. @@ -260,6 +273,10 @@ def prediction_states( for featurizing last user input. ignore_rule_only_turns: If True ignore dialogue turns that are present only in rules. + rule_only_slots: The list of slot names, + which only occur in rules but not in stories. + rule_only_loops: The list of loop names, + which only occur in rules but not in stories. Returns: A list of states. @@ -275,6 +292,8 @@ def create_state_features( interpreter: NaturalLanguageInterpreter, use_text_for_last_user_input: bool = False, ignore_rule_only_turns: bool = False, + rule_only_slots: Optional[List[Text]] = None, + rule_only_loops: Optional[List[Text]] = None, ) -> List[List[Dict[Text, List["Features"]]]]: """Create state features for prediction. @@ -286,6 +305,10 @@ def create_state_features( for featurizing last user input. ignore_rule_only_turns: If True ignore dialogue turns that are present only in rules. + rule_only_slots: The list of slot names, + which only occur in rules but not in stories. + rule_only_loops: The list of loop names, + which only occur in rules but not in stories. Returns: A list (corresponds to the list of trackers) @@ -295,7 +318,12 @@ def create_state_features( turns in all trackers. """ trackers_as_states = self.prediction_states( - trackers, domain, use_text_for_last_user_input, ignore_rule_only_turns + trackers, + domain, + use_text_for_last_user_input, + ignore_rule_only_turns, + rule_only_slots, + rule_only_loops, ) return self._featurize_states(trackers_as_states, interpreter) @@ -420,6 +448,8 @@ def prediction_states( domain: Domain, use_text_for_last_user_input: bool = False, ignore_rule_only_turns: bool = False, + rule_only_slots: Optional[List[Text]] = None, + rule_only_loops: Optional[List[Text]] = None, ) -> List[List[State]]: """Transforms list of trackers to lists of states for prediction. @@ -430,12 +460,22 @@ def prediction_states( for featurizing last user input. ignore_rule_only_turns: If True ignore dialogue turns that are present only in rules. + rule_only_slots: The list of slot names, + which only occur in rules but not in stories. + rule_only_loops: The list of loop names, + which only occur in rules but not in stories. Returns: A list of states. """ trackers_as_states = [ - self._create_states(tracker, domain, ignore_rule_only_turns) + self._create_states( + tracker, + domain, + ignore_rule_only_turns, + rule_only_slots, + rule_only_loops, + ) for tracker in trackers ] self._choose_last_user_input(trackers_as_states, use_text_for_last_user_input) @@ -579,6 +619,8 @@ def prediction_states( domain: Domain, use_text_for_last_user_input: bool = False, ignore_rule_only_turns: bool = False, + rule_only_slots: Optional[List[Text]] = None, + rule_only_loops: Optional[List[Text]] = None, ) -> List[List[State]]: """Transforms list of trackers to lists of states for prediction. @@ -589,12 +631,22 @@ def prediction_states( for featurizing last user input. ignore_rule_only_turns: If True ignore dialogue turns that are present only in rules. + rule_only_slots: The list of slot names, + which only occur in rules but not in stories. + rule_only_loops: The list of loop names, + which only occur in rules but not in stories. Returns: A list of states. """ trackers_as_states = [ - self._create_states(tracker, domain, ignore_rule_only_turns) + self._create_states( + tracker, + domain, + ignore_rule_only_turns, + rule_only_slots, + rule_only_loops, + ) for tracker in trackers ] trackers_as_states = [ diff --git a/rasa/core/policies/ensemble.py b/rasa/core/policies/ensemble.py index 96610febf061..44c1a88e8a96 100644 --- a/rasa/core/policies/ensemble.py +++ b/rasa/core/policies/ensemble.py @@ -22,7 +22,6 @@ DOCS_URL_POLICIES, DOCS_URL_MIGRATION_GUIDE, DEFAULT_CONFIG_PATH, - DOCS_URL_ACTIONS, DOCS_URL_DEFAULT_ACTIONS, ) from rasa.shared.core.constants import ( @@ -31,6 +30,8 @@ ACTION_LISTEN_NAME, ACTION_RESTART_NAME, ACTION_BACK_NAME, + RULE_ONLY_SLOTS, + RULE_ONLY_LOOPS, ) from rasa.shared.core.domain import InvalidDomain, Domain from rasa.shared.core.events import ( @@ -69,6 +70,20 @@ def __init__( self._check_priorities() self._check_for_important_policies() + rule_policy = self._get_rule_policy() + self._rule_only_slots = ( + rule_policy.lookup.get(RULE_ONLY_SLOTS, []) if rule_policy else [] + ) + self._rule_only_loops = ( + rule_policy.lookup.get(RULE_ONLY_LOOPS, []) if rule_policy else [] + ) + + def _get_rule_policy(self) -> Optional[RulePolicy]: + return next( + (policy for policy in self.policies if isinstance(policy, RulePolicy)), + None, + ) + def _check_for_important_policies(self) -> None: from rasa.core.policies.mapping_policy import MappingPolicy @@ -204,6 +219,13 @@ def train( self.action_fingerprints = rasa.core.training.training.create_action_fingerprints( training_trackers, domain ) + rule_policy = self._get_rule_policy() + self._rule_only_slots = ( + rule_policy.lookup.get(RULE_ONLY_SLOTS, []) if rule_policy else [] + ) + self._rule_only_loops = ( + rule_policy.lookup.get(RULE_ONLY_LOOPS, []) if rule_policy else [] + ) else: logger.info("Skipped training, because there are no training samples.") @@ -688,8 +710,8 @@ def _best_policy_prediction( return self._pick_best_policy(predictions) - @staticmethod def _get_prediction( + self, policy: Policy, tracker: DialogueStateTracker, domain: Domain, @@ -705,7 +727,11 @@ def _get_prediction( and "interpreter" in arguments ): prediction = policy.predict_action_probabilities( - tracker, domain, interpreter + tracker, + domain, + interpreter, + rule_only_slots=self._rule_only_slots, + rule_only_loops=self._rule_only_loops, ) else: rasa.shared.utils.io.raise_warning( diff --git a/rasa/core/policies/form_policy.py b/rasa/core/policies/form_policy.py index 95fd234608c0..ee1e9771d1cd 100644 --- a/rasa/core/policies/form_policy.py +++ b/rasa/core/policies/form_policy.py @@ -103,7 +103,11 @@ def _create_lookup_from_states( return lookup def recall( - self, states: List[State], tracker: DialogueStateTracker, domain: Domain + self, + states: List[State], + tracker: DialogueStateTracker, + domain: Domain, + **kwargs: Any, ) -> Optional[Text]: # modify the states return self._recall_states(self._modified_states(states)) diff --git a/rasa/core/policies/memoization.py b/rasa/core/policies/memoization.py index 55fa21efeb7f..4e96183b4fc5 100644 --- a/rasa/core/policies/memoization.py +++ b/rasa/core/policies/memoization.py @@ -195,7 +195,11 @@ def _recall_states(self, states: List[State]) -> Optional[Text]: return self.lookup.get(self._create_feature_key(states)) def recall( - self, states: List[State], tracker: DialogueStateTracker, domain: Domain + self, + states: List[State], + tracker: DialogueStateTracker, + domain: Domain, + **kwargs: Any, ) -> Optional[Text]: return self._recall_states(states) @@ -224,9 +228,9 @@ def predict_action_probabilities( ) -> PolicyPrediction: result = self._default_predictions(domain) - states = self.prediction_states(tracker, domain) + states = self.prediction_states(tracker, domain, **kwargs) logger.debug(f"Current tracker state:{self.format_tracker_states(states)}") - predicted_action_name = self.recall(states, tracker, domain) + predicted_action_name = self.recall(states, tracker, domain, **kwargs) if predicted_action_name is not None: logger.debug(f"There is a memorised next action '{predicted_action_name}'") result = self._prediction_result(predicted_action_name, tracker, domain) @@ -300,7 +304,9 @@ def _back_to_the_future( return mcfly_tracker - def _recall_using_delorean(self, old_states, tracker, domain) -> Optional[Text]: + def _recall_using_delorean( + self, old_states, tracker, domain, **kwargs + ) -> Optional[Text]: """Recursively go to the past to correctly forget slots, and then back to the future to recall.""" @@ -308,7 +314,7 @@ def _recall_using_delorean(self, old_states, tracker, domain) -> Optional[Text]: mcfly_tracker = self._back_to_the_future(tracker) while mcfly_tracker is not None: - states = self.prediction_states(mcfly_tracker, domain) + states = self.prediction_states(mcfly_tracker, domain, **kwargs) if old_states != states: # check if we like new futures @@ -326,12 +332,16 @@ def _recall_using_delorean(self, old_states, tracker, domain) -> Optional[Text]: return None def recall( - self, states: List[State], tracker: DialogueStateTracker, domain: Domain + self, + states: List[State], + tracker: DialogueStateTracker, + domain: Domain, + **kwargs: Any, ) -> Optional[Text]: predicted_action_name = self._recall_states(states) if predicted_action_name is None: # let's try a different method to recall that tracker - return self._recall_using_delorean(states, tracker, domain) + return self._recall_using_delorean(states, tracker, domain, **kwargs) else: return predicted_action_name diff --git a/rasa/core/policies/policy.py b/rasa/core/policies/policy.py index 4f45f1a77dbc..270bdd2790b1 100644 --- a/rasa/core/policies/policy.py +++ b/rasa/core/policies/policy.py @@ -34,7 +34,14 @@ from rasa.shared.core.trackers import DialogueStateTracker from rasa.shared.core.generator import TrackerWithCachedStates from rasa.core.constants import DEFAULT_POLICY_PRIORITY -from rasa.shared.core.constants import USER, SLOTS, PREVIOUS_ACTION, ACTIVE_LOOP +from rasa.shared.core.constants import ( + USER, + SLOTS, + PREVIOUS_ACTION, + ACTIVE_LOOP, + RULE_ONLY_SLOTS, + RULE_ONLY_LOOPS, +) from rasa.shared.nlu.constants import ENTITIES, INTENT, TEXT, ACTION_TEXT, ACTION_NAME if TYPE_CHECKING: @@ -199,6 +206,7 @@ def prediction_states( tracker: DialogueStateTracker, domain: Domain, use_text_for_last_user_input: bool = False, + **kwargs: Any, ) -> List[State]: """Transforms tracker to states for prediction. @@ -216,6 +224,8 @@ def prediction_states( domain, use_text_for_last_user_input=use_text_for_last_user_input, ignore_rule_only_turns=self.supported_data() == SupportedData.ML_DATA, + rule_only_slots=kwargs.get("rule_only_slots"), + rule_only_loops=kwargs.get("rule_only_loops"), )[0] def featurize_for_prediction( @@ -224,6 +234,7 @@ def featurize_for_prediction( domain: Domain, interpreter: NaturalLanguageInterpreter, use_text_for_last_user_input: bool = False, + **kwargs: Any, ) -> List[List[Dict[Text, List["Features"]]]]: """Transforms training tracker into a vector representation. @@ -250,6 +261,8 @@ def featurize_for_prediction( interpreter, use_text_for_last_user_input=use_text_for_last_user_input, ignore_rule_only_turns=self.supported_data() == SupportedData.ML_DATA, + rule_only_slots=kwargs.get("rule_only_slots"), + rule_only_loops=kwargs.get("rule_only_loops"), ) def train( @@ -466,6 +479,7 @@ def __init__( is_end_to_end_prediction: bool = False, is_no_user_prediction: bool = False, diagnostic_data: Optional[Dict[Text, Any]] = None, + hide_rule_turn: bool = False, ) -> None: """Creates a `PolicyPrediction`. @@ -489,6 +503,8 @@ def __init__( diagnostic_data: Intermediate results or other information that is not necessary for Rasa to function, but intended for debugging and fine-tuning purposes. + hide_rule_turn: `True` if the prediction was made by the rules which + do not appear in the stories """ self.probabilities = probabilities self.policy_name = policy_name @@ -498,6 +514,7 @@ def __init__( self.is_end_to_end_prediction = is_end_to_end_prediction self.is_no_user_prediction = is_no_user_prediction self.diagnostic_data = diagnostic_data or {} + self.hide_rule_turn = hide_rule_turn @staticmethod def for_action_name( @@ -541,6 +558,7 @@ def __eq__(self, other: Any) -> bool: and self.optional_events == other.events and self.is_end_to_end_prediction == other.is_end_to_end_prediction and self.is_no_user_prediction == other.is_no_user_prediction + and self.hide_rule_turn == other.hide_rule_turn # We do not compare `diagnostic_data`, because it has no effect on the # action prediction. ) diff --git a/rasa/core/policies/rule_policy.py b/rasa/core/policies/rule_policy.py index 62028e3f2d59..304787f6d817 100644 --- a/rasa/core/policies/rule_policy.py +++ b/rasa/core/policies/rule_policy.py @@ -13,7 +13,6 @@ LoopInterrupted, UserUttered, ActionExecuted, - HideRuleTurn, Event, ) from rasa.core.featurizers.tracker_featurizers import TrackerFeaturizer @@ -1139,21 +1138,18 @@ def _rule_prediction( is_end_to_end_prediction: bool = False, is_no_user_prediction: bool = False, ) -> "RulePolicyPrediction": - optional_events = [] - if prediction_source in self.lookup.get(RULES_NOT_IN_STORIES, []): - # the prediction is based on rules that are not present in the stories - optional_events = [ - HideRuleTurn(self.lookup[RULE_ONLY_SLOTS], self.lookup[RULE_ONLY_LOOPS]) - ] - return RulePolicyPrediction( probabilities, self.__class__.__name__, self.priority, events=[LoopInterrupted(True)] if returning_from_unhappy_path else [], - optional_events=optional_events, is_end_to_end_prediction=is_end_to_end_prediction, is_no_user_prediction=is_no_user_prediction, + hide_rule_turn=( + True + if prediction_source in self.lookup.get(RULES_NOT_IN_STORIES, []) + else False + ), prediction_source=prediction_source, ) @@ -1193,6 +1189,7 @@ def __init__( is_end_to_end_prediction: bool = False, is_no_user_prediction: bool = False, diagnostic_data: Optional[Dict[Text, Any]] = None, + hide_rule_turn: bool = False, prediction_source: Optional[Text] = None, ) -> None: """Creates a `RulePolicyPrediction`. @@ -1217,6 +1214,8 @@ def __init__( diagnostic_data: Intermediate results or other information that is not necessary for Rasa to function, but intended for debugging and fine-tuning purposes. + hide_rule_turn: `True` if the prediction was made by the rules which + do not appear in the stories prediction_source: A description of the matching rule. """ self.prediction_source = prediction_source @@ -1229,4 +1228,5 @@ def __init__( is_end_to_end_prediction, is_no_user_prediction, diagnostic_data, + hide_rule_turn, ) diff --git a/rasa/core/policies/sklearn_policy.py b/rasa/core/policies/sklearn_policy.py index 17cbd4afd2c2..8cea67c25bc3 100644 --- a/rasa/core/policies/sklearn_policy.py +++ b/rasa/core/policies/sklearn_policy.py @@ -284,7 +284,7 @@ def predict_action_probabilities( interpreter: NaturalLanguageInterpreter, **kwargs: Any, ) -> PolicyPrediction: - X = self.featurize_for_prediction(tracker, domain, interpreter) + X = self.featurize_for_prediction(tracker, domain, interpreter, **kwargs) training_data, _ = model_data_utils.convert_to_data_format( X, self.zero_state_features ) diff --git a/rasa/core/policies/ted_policy.py b/rasa/core/policies/ted_policy.py index 66d7e25af456..9f11c5021c54 100644 --- a/rasa/core/policies/ted_policy.py +++ b/rasa/core/policies/ted_policy.py @@ -547,6 +547,7 @@ def _featurize_tracker_for_e2e( tracker: DialogueStateTracker, domain: Domain, interpreter: NaturalLanguageInterpreter, + **kwargs: Any, ) -> List[List[Dict[Text, List["Features"]]]]: # construct two examples in the batch to be fed to the model - # one by featurizing last user text @@ -554,7 +555,11 @@ def _featurize_tracker_for_e2e( # the first example in the constructed batch either does not contain user input # or uses intent or text based on whether TED is e2e only. tracker_state_features = self.featurize_for_prediction( - tracker, domain, interpreter, use_text_for_last_user_input=self.only_e2e + tracker, + domain, + interpreter, + use_text_for_last_user_input=self.only_e2e, + **kwargs, ) # the second - text, but only after user utterance and if not only e2e if ( @@ -563,7 +568,11 @@ def _featurize_tracker_for_e2e( and not self.only_e2e ): tracker_state_features += self.featurize_for_prediction( - tracker, domain, interpreter, use_text_for_last_user_input=True + tracker, + domain, + interpreter, + use_text_for_last_user_input=True, + **kwargs, ) return tracker_state_features @@ -625,7 +634,7 @@ def predict_action_probabilities( # create model data from tracker tracker_state_features = self._featurize_tracker_for_e2e( - tracker, domain, interpreter + tracker, domain, interpreter, **kwargs ) model_data = self._create_model_data(tracker_state_features) diff --git a/rasa/core/processor.py b/rasa/core/processor.py index 9720dd2321b2..7e337c20a69d 100644 --- a/rasa/core/processor.py +++ b/rasa/core/processor.py @@ -26,7 +26,6 @@ ) from rasa.shared.core.domain import Domain from rasa.shared.core.events import ( - ActionExecuted, ActionExecutionRejected, BotUttered, Event, diff --git a/rasa/shared/core/domain.py b/rasa/shared/core/domain.py index 87ceef1507d7..d5f7665aaf7f 100644 --- a/rasa/shared/core/domain.py +++ b/rasa/shared/core/domain.py @@ -1076,18 +1076,22 @@ def get_active_states(self, tracker: "DialogueStateTracker") -> State: @staticmethod def _remove_rule_only_features( - state: State, tracker: "DialogueStateTracker" + state: State, + rule_only_slots: Optional[List[Text]], + rule_only_loops: Optional[List[Text]], ) -> None: # remove slots which only occur in rules but not in stories - for slot in tracker.rule_only_slots: - if state.get(rasa.shared.core.constants.SLOTS, {}).get(slot): - del state[rasa.shared.core.constants.SLOTS][slot] + if rule_only_slots: + for slot in rule_only_slots: + if state.get(rasa.shared.core.constants.SLOTS, {}).get(slot): + del state[rasa.shared.core.constants.SLOTS][slot] # remove active loop which only occur in rules but not in stories if ( - state.get(rasa.shared.core.constants.ACTIVE_LOOP, {}).get( + rule_only_loops + and state.get(rasa.shared.core.constants.ACTIVE_LOOP, {}).get( rasa.shared.core.constants.LOOP_NAME ) - in tracker.rule_only_loops + in rule_only_loops ): del state[rasa.shared.core.constants.ACTIVE_LOOP] @@ -1104,7 +1108,11 @@ def _substitute_rule_only_user_input(state: State, last_ml_state: State) -> None ] def states_for_tracker_history( - self, tracker: "DialogueStateTracker", ignore_rule_only_turns: bool = False + self, + tracker: "DialogueStateTracker", + ignore_rule_only_turns: bool = False, + rule_only_slots: Optional[List[Text]] = None, + rule_only_loops: Optional[List[Text]] = None, ) -> List[State]: """Array of states for each state of the trackers history. @@ -1112,6 +1120,10 @@ def states_for_tracker_history( tracker: Instance of `DialogueStateTracker` to featurize. ignore_rule_only_turns: If True ignore dialogue turns that are present only in rules. + rule_only_slots: The list of slot names, + which only occur in rules but not in stories. + rule_only_loops: The list of loop names, + which only occur in rules but not in stories. Return: A list of states. @@ -1119,7 +1131,7 @@ def states_for_tracker_history( states = [] last_ml_action_sub_state = None turn_was_hidden = False - for tr in tracker.generate_all_prior_trackers(): + for tr, hide_rule_turn in tracker.generate_all_prior_trackers(): if ignore_rule_only_turns: # remember previous ml action based on the last non hidden turn # we need this to override previous action in the ml state @@ -1132,7 +1144,7 @@ def states_for_tracker_history( not tr.followup_action and not tr.latest_action_name == tr.active_loop_name ): - turn_was_hidden = tr.hide_rule_turn + turn_was_hidden = hide_rule_turn if turn_was_hidden: continue @@ -1141,7 +1153,7 @@ def states_for_tracker_history( if ignore_rule_only_turns: # clean state from only rule features - self._remove_rule_only_features(state, tracker) + self._remove_rule_only_features(state, rule_only_slots, rule_only_loops) # make sure user input is the same as for previous state # for non action_listen turns if states: diff --git a/rasa/shared/core/events.py b/rasa/shared/core/events.py index 1a59ebd65c1e..ecbb794b76a1 100644 --- a/rasa/shared/core/events.py +++ b/rasa/shared/core/events.py @@ -25,8 +25,6 @@ ENTITY_LABEL_SEPARATOR, ACTION_SESSION_START_NAME, ACTION_LISTEN_NAME, - RULE_ONLY_SLOTS, - RULE_ONLY_LOOPS, ) from rasa.shared.exceptions import UnsupportedFeatureException from rasa.shared.nlu.constants import ( @@ -1436,6 +1434,7 @@ def __init__( timestamp: Optional[float] = None, metadata: Optional[Dict] = None, action_text: Optional[Text] = None, + hide_rule_turn: bool = False, ) -> None: """Creates event for a successful event execution. @@ -1454,6 +1453,7 @@ def __init__( self.confidence = confidence self.unpredictable = False self.action_text = action_text + self.hide_rule_turn = hide_rule_turn super().__init__(timestamp, metadata) @@ -1546,12 +1546,6 @@ def apply_to(self, tracker: "DialogueStateTracker") -> None: tracker.set_latest_action(self.as_sub_state()) tracker.clear_followup_action() - # HideRuleTurn event is added by RulePolicy before actual action is executed, - # so we can safely reset it on each action executed - # unless it is active_loop prediction - if not self.action_name == tracker.active_loop_name: - tracker.hide_rule_turn = False - class AgentUttered(SkipEventInMDStoryMixin): """The agent has said something to the user. @@ -1886,78 +1880,3 @@ def apply_to(self, tracker: "DialogueStateTracker") -> None: """Applies event to current conversation state.""" # noinspection PyProtectedMember tracker._reset() - - -class HideRuleTurn(SkipEventInMDStoryMixin, AlwaysEqualEventMixin): - """Emulates undoing of the last dialogue turn. - - The bot reverts everything until before the most recent action. - This includes the action itself, as well as any events that - action created, like set slot events (unless they are not in a list - of `rule_only_slots`) and active loop events (unless they are not in a list - of `rule_only_loops`). - """ - - type_name = "hide_rule_turn" - - def __init__( - self, - rule_only_slots: List[Text], - rule_only_loops: List[Text], - timestamp: Optional[float] = None, - metadata: Optional[Dict[Text, Any]] = None, - ) -> None: - """Initializes event. - - Args: - rule_only_slots: The list of slot names, - which only occur in rules but not in stories. - rule_only_loops: The list of loop names, - which only occur in rules but not in stories. - timestamp: the timestamp - metadata: some optional metadata - """ - super().__init__(timestamp, metadata) - self.rule_only_slots = rule_only_slots - self.rule_only_loops = rule_only_loops - - def __hash__(self) -> int: - """Returns unique hash for event.""" - return hash(32143124321) - - @classmethod - def _from_parameters(cls, parameters: Dict[Text, Any]) -> "HideRuleTurn": - return HideRuleTurn( - parameters.get(RULE_ONLY_SLOTS), - parameters.get(RULE_ONLY_LOOPS), - parameters.get("timestamp"), - parameters.get("metadata"), - ) - - def as_dict(self) -> Dict[Text, Any]: - """Converts the event into a dict. - - Returns: - A dict that represents this event. - """ - d = super().as_dict() - d.update( - { - RULE_ONLY_SLOTS: self.rule_only_slots, - RULE_ONLY_LOOPS: self.rule_only_loops, - } - ) - return d - - def apply_to(self, tracker: "DialogueStateTracker") -> None: - """Applies event to current conversation state. - - Args: - tracker: The current conversation state. - """ - # HideRuleTurn event is added by RulePolicy before actual action is executed, - # we will reset it on each action executed - tracker.hide_rule_turn = True - # only rule slots and loops are always the same for all the trackers - tracker.rule_only_slots = self.rule_only_slots - tracker.rule_only_loops = self.rule_only_loops diff --git a/rasa/shared/core/generator.py b/rasa/shared/core/generator.py index 3f5ffeee01db..48508252da00 100644 --- a/rasa/shared/core/generator.py +++ b/rasa/shared/core/generator.py @@ -107,15 +107,11 @@ def _unfreeze_states(frozen_states: Deque[FrozenState]) -> List[State]: for frozen_state in frozen_states ] - def past_states( - self, domain: Domain, ignore_rule_only_turns: bool = False - ) -> List[State]: + def past_states(self, domain: Domain, **kwargs: Any) -> List[State]: """Generates the past states of this tracker based on the history. Args: domain: a :class:`rasa.shared.core.domain.Domain`. - ignore_rule_only_turns: If True ignore dialogue turns that are present - only in rules. Returns: a list of states diff --git a/rasa/shared/core/trackers.py b/rasa/shared/core/trackers.py index 907a819b020f..b6cad60c156f 100644 --- a/rasa/shared/core/trackers.py +++ b/rasa/shared/core/trackers.py @@ -59,7 +59,6 @@ SessionStarted, ActionExecutionRejected, EntitiesAdded, - HideRuleTurn, DefinePrevUserUtteredFeaturization, ) from rasa.shared.core.domain import Domain, State @@ -207,9 +206,6 @@ def __init__( self.latest_bot_utterance = None self._reset() self.active_loop: Dict[Text, Union[Text, bool, Dict, None]] = {} - self.hide_rule_turn = False - self.rule_only_slots = [] - self.rule_only_loops = [] ### # Public tracker interface @@ -279,7 +275,11 @@ def freeze_current_state(state: State) -> FrozenState: ) def past_states( - self, domain: Domain, ignore_rule_only_turns: bool = False + self, + domain: Domain, + ignore_rule_only_turns: bool = False, + rule_only_slots: Optional[List[Text]] = None, + rule_only_loops: Optional[List[Text]] = None, ) -> List[State]: """Generate the past states of this tracker based on the history. @@ -287,11 +287,17 @@ def past_states( domain: a :class:`rasa.shared.core.domain.Domain`. ignore_rule_only_turns: If True ignore dialogue turns that are present only in rules. + rule_only_slots: The list of slot names, + which only occur in rules but not in stories. + rule_only_loops: The list of loop names, + which only occur in rules but not in stories. Returns: a list of states """ - return domain.states_for_tracker_history(self, ignore_rule_only_turns) + return domain.states_for_tracker_history( + self, ignore_rule_only_turns, rule_only_slots, rule_only_loops + ) def change_loop_to(self, loop_name: Optional[Text]) -> None: """Set the currently active loop. @@ -438,7 +444,7 @@ def init_copy(self) -> "DialogueStateTracker": def generate_all_prior_trackers( self, - ) -> Generator["DialogueStateTracker", None, None]: + ) -> Generator[Tuple["DialogueStateTracker", bool], None, None]: """Returns a generator of the previous trackers of this tracker. The resulting array is representing the trackers before each action.""" @@ -448,11 +454,11 @@ def generate_all_prior_trackers( for event in self.applied_events(): if isinstance(event, ActionExecuted): - yield tracker + yield tracker, event.hide_rule_turn tracker.update(event) - yield tracker + yield tracker, False def applied_events(self) -> List[Event]: """Returns all actions that should be applied - w/o reverted events. @@ -566,13 +572,7 @@ def _undo_till_previous_loop_execution( break if isinstance( - e, - ( - ActionExecuted, - UserUttered, - HideRuleTurn, - DefinePrevUserUtteredFeaturization, - ), + e, (ActionExecuted, UserUttered, DefinePrevUserUtteredFeaturization,), ): del done_events[-1 - offset] else: diff --git a/tests/core/policies/test_rule_policy.py b/tests/core/policies/test_rule_policy.py index 77c8222a5bf9..af626dc2b785 100644 --- a/tests/core/policies/test_rule_policy.py +++ b/tests/core/policies/test_rule_policy.py @@ -24,6 +24,8 @@ PREVIOUS_ACTION, ACTIVE_LOOP, LOOP_NAME, + RULE_ONLY_SLOTS, + RULE_ONLY_LOOPS, ) from rasa.shared.nlu.constants import TEXT, INTENT, ACTION_NAME from rasa.shared.core.domain import Domain @@ -34,7 +36,6 @@ SlotSet, ActionExecutionRejected, LoopInterrupted, - HideRuleTurn, FollowupAction, ) from rasa.shared.nlu.interpreter import RegexInterpreter @@ -1986,11 +1987,10 @@ def test_hide_rule_turn(): RegexInterpreter(), ) assert_predicted_action(prediction, domain, UTTER_GREET_ACTION) - assert isinstance(prediction.optional_events[0], HideRuleTurn) + assert prediction.hide_rule_turn - conversation_events += prediction.optional_events conversation_events += [ - ActionExecuted(UTTER_GREET_ACTION), + ActionExecuted(UTTER_GREET_ACTION, hide_rule_turn=prediction.hide_rule_turn) ] prediction = policy.predict_action_probabilities( DialogueStateTracker.from_events( @@ -2000,11 +2000,10 @@ def test_hide_rule_turn(): RegexInterpreter(), ) assert_predicted_action(prediction, domain, ACTION_LISTEN_NAME) - assert isinstance(prediction.optional_events[0], HideRuleTurn) + assert prediction.hide_rule_turn - conversation_events += prediction.optional_events conversation_events += [ - ActionExecuted(ACTION_LISTEN_NAME), + ActionExecuted(ACTION_LISTEN_NAME, hide_rule_turn=prediction.hide_rule_turn), UserUttered("haha", {"name": chitchat}), ] tracker = DialogueStateTracker.from_events( @@ -2080,7 +2079,7 @@ def test_hide_rule_turn_with_slots(): slots=domain.slots, evts=[ ActionExecuted(ACTION_LISTEN_NAME), - UserUttered(intent={"name": some_other_action}), + UserUttered(intent={"name": some_other_intent}), ActionExecuted(some_other_action), SlotSet(some_other_slot, some_other_slot_value), ActionExecuted(ACTION_LISTEN_NAME), @@ -2093,6 +2092,7 @@ def test_hide_rule_turn_with_slots(): domain, RegexInterpreter(), ) + assert policy.lookup[RULE_ONLY_SLOTS] == [some_slot] conversation_events = [ ActionExecuted(ACTION_LISTEN_NAME), @@ -2106,22 +2106,38 @@ def test_hide_rule_turn_with_slots(): RegexInterpreter(), ) assert_predicted_action(prediction, domain, some_action) - assert isinstance(prediction.optional_events[0], HideRuleTurn) - assert prediction.optional_events[0].rule_only_slots == [some_slot] + assert prediction.hide_rule_turn - conversation_events += prediction.optional_events conversation_events += [ - ActionExecuted(ACTION_LISTEN_NAME), - UserUttered("haha", {"name": some_other_action}), + ActionExecuted(some_action, hide_rule_turn=prediction.hide_rule_turn), + SlotSet(some_slot, some_slot_value), + ] + prediction = policy.predict_action_probabilities( + DialogueStateTracker.from_events( + "casd", evts=conversation_events, slots=domain.slots + ), + domain, + RegexInterpreter(), + ) + assert_predicted_action(prediction, domain, ACTION_LISTEN_NAME) + assert prediction.hide_rule_turn + + conversation_events += [ + ActionExecuted(ACTION_LISTEN_NAME, hide_rule_turn=prediction.hide_rule_turn), + UserUttered("haha", {"name": some_other_intent}), ] tracker = DialogueStateTracker.from_events( "casd", evts=conversation_events, slots=domain.slots ) - states = tracker.past_states(domain, ignore_rule_only_turns=True) + states = tracker.past_states( + domain, + ignore_rule_only_turns=True, + rule_only_slots=policy.lookup[RULE_ONLY_SLOTS], + ) assert states == [ {}, { - USER: {TEXT: "haha", INTENT: some_other_action}, + USER: {TEXT: "haha", INTENT: some_other_intent}, PREVIOUS_ACTION: {ACTION_NAME: ACTION_LISTEN_NAME}, }, ] @@ -2172,6 +2188,7 @@ def test_hide_rule_turn_no_last_action_listen(): policy.train( [simple_rule_no_last_action_listen, chitchat_story], domain, RegexInterpreter(), ) + assert policy.lookup[RULE_ONLY_SLOTS] == [followup_on_chitchat] conversation_events = [ ActionExecuted(ACTION_LISTEN_NAME), @@ -2187,17 +2204,19 @@ def test_hide_rule_turn_no_last_action_listen(): RegexInterpreter(), ) assert_predicted_action(prediction, domain, action_after_chitchat) - assert isinstance(prediction.optional_events[0], HideRuleTurn) - assert followup_on_chitchat in prediction.optional_events[0].rule_only_slots + assert prediction.hide_rule_turn - conversation_events += prediction.optional_events conversation_events += [ - ActionExecuted(action_after_chitchat), + ActionExecuted(action_after_chitchat, hide_rule_turn=prediction.hide_rule_turn), ] tracker = DialogueStateTracker.from_events( "casd", evts=conversation_events, slots=domain.slots ) - states = tracker.past_states(domain, ignore_rule_only_turns=True) + states = tracker.past_states( + domain, + ignore_rule_only_turns=True, + rule_only_slots=policy.lookup[RULE_ONLY_SLOTS], + ) assert states == [ {}, {USER: {INTENT: chitchat}, PREVIOUS_ACTION: {ACTION_NAME: ACTION_LISTEN_NAME}}, @@ -2261,6 +2280,7 @@ def test_hide_rule_turn_with_loops(): domain, RegexInterpreter(), ) + assert policy.lookup[RULE_ONLY_LOOPS] == [form_name] conversation_events = [ ActionExecuted(ACTION_LISTEN_NAME), @@ -2274,13 +2294,10 @@ def test_hide_rule_turn_with_loops(): RegexInterpreter(), ) assert_predicted_action(prediction, domain, form_name) - assert isinstance(prediction.optional_events[0], HideRuleTurn) - assert form_name in prediction.optional_events[0].rule_only_loops - assert another_form_name not in prediction.optional_events[0].rule_only_loops + assert prediction.hide_rule_turn - conversation_events += prediction.optional_events conversation_events += [ - ActionExecuted(form_name), + ActionExecuted(form_name, hide_rule_turn=prediction.hide_rule_turn), ActiveLoop(form_name), ] prediction = policy.predict_action_probabilities( @@ -2293,19 +2310,20 @@ def test_hide_rule_turn_with_loops(): assert_predicted_action( prediction, domain, ACTION_LISTEN_NAME, is_no_user_prediction=True ) - assert isinstance(prediction.optional_events[0], HideRuleTurn) - assert form_name in prediction.optional_events[0].rule_only_loops - assert another_form_name not in prediction.optional_events[0].rule_only_loops + assert prediction.hide_rule_turn - conversation_events += prediction.optional_events conversation_events += [ - ActionExecuted(ACTION_LISTEN_NAME), + ActionExecuted(ACTION_LISTEN_NAME, hide_rule_turn=prediction.hide_rule_turn), UserUttered("haha", {"name": chitchat}), ] tracker = DialogueStateTracker.from_events( "casd", evts=conversation_events, slots=domain.slots ) - states = tracker.past_states(domain, ignore_rule_only_turns=True) + states = tracker.past_states( + domain, + ignore_rule_only_turns=True, + rule_only_loops=policy.lookup[RULE_ONLY_LOOPS], + ) assert states == [ {}, { @@ -2338,6 +2356,7 @@ def test_do_not_hide_rule_turn_with_loops_in_stories(): policy.train( [form_activation_rule, form_activation_story], domain, RegexInterpreter(), ) + assert policy.lookup[RULE_ONLY_LOOPS] == [] conversation_events = [ ActionExecuted(ACTION_LISTEN_NAME), @@ -2351,10 +2370,10 @@ def test_do_not_hide_rule_turn_with_loops_in_stories(): RegexInterpreter(), ) assert_predicted_action(prediction, domain, form_name) - assert not prediction.optional_events + assert not prediction.hide_rule_turn conversation_events += [ - ActionExecuted(form_name), + ActionExecuted(form_name, hide_rule_turn=prediction.hide_rule_turn), ActiveLoop(form_name), ] prediction = policy.predict_action_probabilities( @@ -2367,7 +2386,7 @@ def test_do_not_hide_rule_turn_with_loops_in_stories(): assert_predicted_action( prediction, domain, ACTION_LISTEN_NAME, is_no_user_prediction=True ) - assert not prediction.optional_events + assert not prediction.hide_rule_turn def test_hide_rule_turn_with_loops_as_followup_action(): @@ -2398,6 +2417,7 @@ def test_hide_rule_turn_with_loops_as_followup_action(): domain, RegexInterpreter(), ) + assert policy.lookup[RULE_ONLY_LOOPS] == [] conversation_events = [ ActionExecuted(ACTION_LISTEN_NAME), @@ -2411,10 +2431,10 @@ def test_hide_rule_turn_with_loops_as_followup_action(): RegexInterpreter(), ) assert_predicted_action(prediction, domain, form_name) - assert not prediction.optional_events + assert not prediction.hide_rule_turn conversation_events += [ - ActionExecuted(form_name), + ActionExecuted(form_name, hide_rule_turn=prediction.hide_rule_turn), ActiveLoop(form_name), ] prediction = policy.predict_action_probabilities( @@ -2427,10 +2447,10 @@ def test_hide_rule_turn_with_loops_as_followup_action(): assert_predicted_action( prediction, domain, ACTION_LISTEN_NAME, is_no_user_prediction=True ) - assert not prediction.optional_events + assert not prediction.hide_rule_turn conversation_events += [ - ActionExecuted(ACTION_LISTEN_NAME), + ActionExecuted(ACTION_LISTEN_NAME, hide_rule_turn=prediction.hide_rule_turn), UserUttered("haha", {"name": GREET_INTENT_NAME}), ActionExecutionRejected(form_name), ] @@ -2442,11 +2462,10 @@ def test_hide_rule_turn_with_loops_as_followup_action(): RegexInterpreter(), ) assert_predicted_action(prediction, domain, UTTER_GREET_ACTION) - assert isinstance(prediction.optional_events[0], HideRuleTurn) + assert prediction.hide_rule_turn - conversation_events += prediction.optional_events conversation_events += [ - ActionExecuted(UTTER_GREET_ACTION), + ActionExecuted(UTTER_GREET_ACTION, hide_rule_turn=prediction.hide_rule_turn), FollowupAction(form_name), ActionExecuted(form_name), ] diff --git a/tests/core/test_processor.py b/tests/core/test_processor.py index 2a6bb2ab3bfa..e8b20bb161cd 100644 --- a/tests/core/test_processor.py +++ b/tests/core/test_processor.py @@ -42,7 +42,6 @@ DefinePrevUserUtteredFeaturization, ActionExecutionRejected, LoopInterrupted, - HideRuleTurn, ) from rasa.core.interpreter import RasaNLUHttpInterpreter from rasa.shared.nlu.interpreter import NaturalLanguageInterpreter, RegexInterpreter @@ -944,7 +943,6 @@ async def test_restart_triggers_session_start( BotUttered("hey there name1!", metadata={"template_name": "utter_greet"}), ActionExecuted(ACTION_LISTEN_NAME), UserUttered("/restart", {INTENT_NAME_KEY: "restart", "confidence": 1.0}), - HideRuleTurn([], []), DefinePrevUserUtteredFeaturization(use_text_for_featurization=False), ActionExecuted(ACTION_RESTART_NAME), Restarted(), From fbfc4a3c22fd024becd91d53f079b53a94d9fdba Mon Sep 17 00:00:00 2001 From: Vova Vv Date: Tue, 16 Feb 2021 09:58:30 +0100 Subject: [PATCH 40/68] persist hide_rule_turn --- rasa/core/policies/ensemble.py | 30 ++++++++++++++---------------- rasa/shared/core/events.py | 11 +++++++++-- 2 files changed, 23 insertions(+), 18 deletions(-) diff --git a/rasa/core/policies/ensemble.py b/rasa/core/policies/ensemble.py index 44c1a88e8a96..6b09970e39e7 100644 --- a/rasa/core/policies/ensemble.py +++ b/rasa/core/policies/ensemble.py @@ -70,19 +70,20 @@ def __init__( self._check_priorities() self._check_for_important_policies() - rule_policy = self._get_rule_policy() - self._rule_only_slots = ( - rule_policy.lookup.get(RULE_ONLY_SLOTS, []) if rule_policy else [] - ) - self._rule_only_loops = ( - rule_policy.lookup.get(RULE_ONLY_LOOPS, []) if rule_policy else [] - ) + self._rule_only_slots, self._rule_only_loops = self._get_rule_only_slots_loops() - def _get_rule_policy(self) -> Optional[RulePolicy]: - return next( + def _get_rule_only_slots_loops(self) -> Tuple[List[Text], List[Text]]: + rule_policy = next( (policy for policy in self.policies if isinstance(policy, RulePolicy)), None, ) + rule_only_slots = ( + rule_policy.lookup.get(RULE_ONLY_SLOTS, []) if rule_policy else [] + ) + rule_only_loops = ( + rule_policy.lookup.get(RULE_ONLY_LOOPS, []) if rule_policy else [] + ) + return rule_only_slots, rule_only_loops def _check_for_important_policies(self) -> None: from rasa.core.policies.mapping_policy import MappingPolicy @@ -219,13 +220,10 @@ def train( self.action_fingerprints = rasa.core.training.training.create_action_fingerprints( training_trackers, domain ) - rule_policy = self._get_rule_policy() - self._rule_only_slots = ( - rule_policy.lookup.get(RULE_ONLY_SLOTS, []) if rule_policy else [] - ) - self._rule_only_loops = ( - rule_policy.lookup.get(RULE_ONLY_LOOPS, []) if rule_policy else [] - ) + ( + self._rule_only_slots, + self._rule_only_loops, + ) = self._get_rule_only_slots_loops() else: logger.info("Skipped training, because there are no training samples.") diff --git a/rasa/shared/core/events.py b/rasa/shared/core/events.py index ecbb794b76a1..64c50525149c 100644 --- a/rasa/shared/core/events.py +++ b/rasa/shared/core/events.py @@ -1495,7 +1495,6 @@ def as_story_string(self) -> Text: @classmethod def _from_story_string(cls, parameters: Dict[Text, Any]) -> Optional[List[Event]]: - return [ ActionExecuted( parameters.get("name"), @@ -1504,6 +1503,7 @@ def _from_story_string(cls, parameters: Dict[Text, Any]) -> Optional[List[Event] parameters.get("timestamp"), parameters.get("metadata"), parameters.get("action_text"), + parameters.get("hide_rule_turn"), ) ] @@ -1516,13 +1516,20 @@ def as_dict(self) -> Dict[Text, Any]: confidence = None if hasattr(self, "confidence"): confidence = self.confidence + action_text = None + if hasattr(self, "hide_rule_turn"): + action_text = self.action_text + hide_rule_turn = False + if hasattr(self, "hide_rule_turn"): + hide_rule_turn = self.hide_rule_turn d.update( { "name": self.action_name, "policy": policy, "confidence": confidence, - "action_text": self.action_text, + "action_text": action_text, + "hide_rule_turn": hide_rule_turn, } ) return d From 7154ddd1b688b7c3634fa014aac09789d9340496 Mon Sep 17 00:00:00 2001 From: Vova Vv Date: Tue, 16 Feb 2021 10:32:58 +0100 Subject: [PATCH 41/68] fix typo --- rasa/shared/core/events.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rasa/shared/core/events.py b/rasa/shared/core/events.py index 64c50525149c..ef065b9e0d68 100644 --- a/rasa/shared/core/events.py +++ b/rasa/shared/core/events.py @@ -1517,7 +1517,7 @@ def as_dict(self) -> Dict[Text, Any]: if hasattr(self, "confidence"): confidence = self.confidence action_text = None - if hasattr(self, "hide_rule_turn"): + if hasattr(self, "action_text"): action_text = self.action_text hide_rule_turn = False if hasattr(self, "hide_rule_turn"): From 67553db8fca5c9f12788922126703ce805d35e97 Mon Sep 17 00:00:00 2001 From: Vova Vv Date: Tue, 16 Feb 2021 10:46:13 +0100 Subject: [PATCH 42/68] fix docstrings --- rasa/core/policies/form_policy.py | 10 ++++++++ rasa/core/policies/memoization.py | 42 ++++++++++++++++++++++++++++--- rasa/shared/core/events.py | 2 ++ rasa/shared/core/trackers.py | 7 ++++-- 4 files changed, 56 insertions(+), 5 deletions(-) diff --git a/rasa/core/policies/form_policy.py b/rasa/core/policies/form_policy.py index ee1e9771d1cd..e4a0e095ce9a 100644 --- a/rasa/core/policies/form_policy.py +++ b/rasa/core/policies/form_policy.py @@ -109,6 +109,16 @@ def recall( domain: Domain, **kwargs: Any, ) -> Optional[Text]: + """Finds the action based on the given states. + + Args: + states: List of states. + tracker: The tracker. + domain: The Domain. + + Returns: + The name of the action. + """ # modify the states return self._recall_states(self._modified_states(states)) diff --git a/rasa/core/policies/memoization.py b/rasa/core/policies/memoization.py index 4e96183b4fc5..79adfc6e28ad 100644 --- a/rasa/core/policies/memoization.py +++ b/rasa/core/policies/memoization.py @@ -201,6 +201,16 @@ def recall( domain: Domain, **kwargs: Any, ) -> Optional[Text]: + """Finds the action based on the given states. + + Args: + states: List of states. + tracker: The tracker. + domain: The Domain. + + Returns: + The name of the action. + """ return self._recall_states(states) def _prediction_result( @@ -305,11 +315,25 @@ def _back_to_the_future( return mcfly_tracker def _recall_using_delorean( - self, old_states, tracker, domain, **kwargs + self, + old_states: List[State], + tracker: DialogueStateTracker, + domain: Domain, + **kwargs, ) -> Optional[Text]: - """Recursively go to the past to correctly forget slots, - and then back to the future to recall.""" + """Applies to the future idea to change the past and get the new future. + + Recursively go to the past to correctly forget slots, + and then back to the future to recall. + Args: + old_states: List of states. + tracker: The tracker. + domain: The Domain. + + Returns: + The name of the action. + """ logger.debug("Launch DeLorean...") mcfly_tracker = self._back_to_the_future(tracker) @@ -338,7 +362,19 @@ def recall( domain: Domain, **kwargs: Any, ) -> Optional[Text]: + """Finds the action based on the given states. + + Uses back to the future idea to change the past and check whether the new future + can be used to recall the action. + Args: + states: List of states. + tracker: The tracker. + domain: The Domain. + + Returns: + The name of the action. + """ predicted_action_name = self._recall_states(states) if predicted_action_name is None: # let's try a different method to recall that tracker diff --git a/rasa/shared/core/events.py b/rasa/shared/core/events.py index ef065b9e0d68..cec6fa7048a5 100644 --- a/rasa/shared/core/events.py +++ b/rasa/shared/core/events.py @@ -1447,6 +1447,8 @@ def __init__( metadata: Additional event metadata. action_text: In case it's an end-to-end action prediction, the text which was predicted. + hide_rule_turn: If `True`, this action should be hidden in the dialogue + history created for ML-based policies. """ self.action_name = action_name self.policy = policy diff --git a/rasa/shared/core/trackers.py b/rasa/shared/core/trackers.py index b6cad60c156f..a5244ac4dd5c 100644 --- a/rasa/shared/core/trackers.py +++ b/rasa/shared/core/trackers.py @@ -447,8 +447,11 @@ def generate_all_prior_trackers( ) -> Generator[Tuple["DialogueStateTracker", bool], None, None]: """Returns a generator of the previous trackers of this tracker. - The resulting array is representing the trackers before each action.""" - + Returns: + The tuple with the tracker before each action, + and the boolean flag representing whether this action should be hidden + in the dialogue history created for ML-based policies. + """ tracker = self.init_copy() for event in self.applied_events(): From 46ac8470d406195d5c42504c6213eff0de8fc758 Mon Sep 17 00:00:00 2001 From: Vova Vv Date: Thu, 18 Feb 2021 13:17:13 +0100 Subject: [PATCH 43/68] fix tests --- data/test_dialogues/default.json | 6 +++-- data/test_dialogues/formbot.json | 24 ++++++++++++------- data/test_dialogues/moodbot.json | 21 ++++++++++------ data/test_trackers/tracker_moodbot.json | 15 ++++++++---- .../tracker_moodbot_with_new_utterances.json | 21 ++++++++++------ tests/test_server.py | 2 ++ 6 files changed, 60 insertions(+), 29 deletions(-) diff --git a/data/test_dialogues/default.json b/data/test_dialogues/default.json index 11ac23fda5d5..e915b2c05cc3 100644 --- a/data/test_dialogues/default.json +++ b/data/test_dialogues/default.json @@ -9,7 +9,8 @@ "confidence": null, "policy": null, "timestamp": 1551952977.4850519, - "unpredictable": false + "unpredictable": false, + "hide_rule_turn": false }, { "py/object": "rasa.shared.core.events.UserUttered", @@ -61,7 +62,8 @@ "confidence": null, "policy": null, "timestamp": 1551953040.607782, - "unpredictable": false + "unpredictable": false, + "hide_rule_turn": false }, { "py/object": "rasa.shared.core.events.BotUttered", diff --git a/data/test_dialogues/formbot.json b/data/test_dialogues/formbot.json index 272ec05f8010..5a84c1b3c829 100644 --- a/data/test_dialogues/formbot.json +++ b/data/test_dialogues/formbot.json @@ -9,7 +9,8 @@ "confidence":null, "policy":null, "timestamp":1551884035.892855, - "unpredictable":false + "unpredictable":false, + "hide_rule_turn": false }, { "py/object":"rasa.shared.core.events.UserUttered", @@ -43,7 +44,8 @@ "confidence":1.0, "policy":"policy_2_MemoizationPolicy", "timestamp":1551884060.466681, - "unpredictable":false + "unpredictable":false, + "hide_rule_turn": false }, { "py/object":"rasa.shared.core.events.BotUttered", @@ -62,7 +64,8 @@ "confidence":1.0, "policy":"policy_2_MemoizationPolicy", "timestamp":1551884061.9350882, - "unpredictable":false + "unpredictable":false, + "hide_rule_turn": false }, { "py/object":"rasa.shared.core.events.UserUttered", @@ -118,7 +121,8 @@ "confidence":1.0, "policy":"policy_2_MemoizationPolicy", "timestamp":1551884095.542748, - "unpredictable":false + "unpredictable":false, + "hide_rule_turn": false }, { "py/object":"rasa.shared.core.events.ActionExecuted", @@ -127,7 +131,8 @@ "confidence":1.0, "policy":"policy_2_MemoizationPolicy", "timestamp":1551884097.570883, - "unpredictable":false + "unpredictable":false, + "hide_rule_turn": false }, { "py/object":"rasa.shared.core.events.BotUttered", @@ -146,7 +151,8 @@ "confidence":1.0, "policy":"policy_2_MemoizationPolicy", "timestamp":1551884098.8006358, - "unpredictable":false + "unpredictable":false, + "hide_rule_turn": false }, { "py/object":"rasa.shared.core.events.UserUttered", @@ -180,7 +186,8 @@ "confidence":null, "policy":null, "timestamp":1551884214.951055, - "unpredictable":false + "unpredictable":false, + "hide_rule_turn": false }, { "py/object":"rasa.shared.core.events.ActiveLoop", @@ -200,7 +207,8 @@ "confidence":0.7680902069097734, "policy":"policy_0_TEDPolicy", "timestamp":1551884216.705635, - "unpredictable":false + "unpredictable":false, + "hide_rule_turn": false } ] } \ No newline at end of file diff --git a/data/test_dialogues/moodbot.json b/data/test_dialogues/moodbot.json index f7869cb773b9..e134a07535d1 100644 --- a/data/test_dialogues/moodbot.json +++ b/data/test_dialogues/moodbot.json @@ -9,7 +9,8 @@ "confidence":null, "policy":null, "timestamp":1551883958.346432, - "unpredictable":false + "unpredictable":false, + "hide_rule_turn": false }, { "py/object":"rasa.shared.core.events.UserUttered", @@ -69,7 +70,8 @@ "confidence":1.0, "policy":"policy_2_MemoizationPolicy", "timestamp":1551883975.6456478, - "unpredictable":false + "unpredictable":false, + "hide_rule_turn": false }, { "py/object":"rasa.shared.core.events.BotUttered", @@ -97,7 +99,8 @@ "confidence":1.0, "policy":"policy_2_MemoizationPolicy", "timestamp":1551883979.098331, - "unpredictable":false + "unpredictable":false, + "hide_rule_turn": false }, { "py/object":"rasa.shared.core.events.UserUttered", @@ -157,7 +160,8 @@ "confidence":1.0, "policy":"policy_2_MemoizationPolicy", "timestamp":1551883985.031668, - "unpredictable":false + "unpredictable":false, + "hide_rule_turn": false }, { "py/object":"rasa.shared.core.events.BotUttered", @@ -176,7 +180,8 @@ "confidence":1.0, "policy":"policy_2_MemoizationPolicy", "timestamp":1551883985.940413, - "unpredictable":false + "unpredictable":false, + "hide_rule_turn": false }, { "py/object":"rasa.shared.core.events.BotUttered", @@ -195,7 +200,8 @@ "confidence":1.0, "policy":"policy_2_MemoizationPolicy", "timestamp":1551883986.958556, - "unpredictable":false + "unpredictable":false, + "hide_rule_turn": false }, { "py/object":"rasa.shared.core.events.UserUttered", @@ -255,7 +261,8 @@ "confidence":1.0, "policy":"policy_2_MemoizationPolicy", "timestamp":1551883991.061463, - "unpredictable":false + "unpredictable":false, + "hide_rule_turn": false }, { "py/object":"rasa.shared.core.events.BotUttered", diff --git a/data/test_trackers/tracker_moodbot.json b/data/test_trackers/tracker_moodbot.json index ec1756d331b1..6e206bd97ea4 100644 --- a/data/test_trackers/tracker_moodbot.json +++ b/data/test_trackers/tracker_moodbot.json @@ -39,7 +39,8 @@ "name": "action_listen", "action_text": null, "policy": null, - "confidence": null + "confidence": null, + "hide_rule_turn": false }, { "timestamp": 1517821726.200036, @@ -78,7 +79,8 @@ "name": "utter_greet", "action_text": null, "policy": null, - "confidence": null + "confidence": null, + "hide_rule_turn": false }, { "timestamp": 1517821726.211038, @@ -86,7 +88,8 @@ "name": "action_listen", "action_text": null, "policy": null, - "confidence": null + "confidence": null, + "hide_rule_turn": false }, { "timestamp": 1517821726.209836, @@ -125,7 +128,8 @@ "name": "utter_happy", "action_text": null, "policy": "policy_1_TEDPolicy", - "confidence": 0.8 + "confidence": 0.8, + "hide_rule_turn": false }, { "timestamp": 1517821726.211042, @@ -133,7 +137,8 @@ "name": "action_listen", "action_text": null, "policy": "policy_2_MemoizationPolicy", - "confidence": 1.0 + "confidence": 1.0, + "hide_rule_turn": false } ] } diff --git a/data/test_trackers/tracker_moodbot_with_new_utterances.json b/data/test_trackers/tracker_moodbot_with_new_utterances.json index 4b0bc1d903b0..08f521bb2154 100644 --- a/data/test_trackers/tracker_moodbot_with_new_utterances.json +++ b/data/test_trackers/tracker_moodbot_with_new_utterances.json @@ -38,7 +38,8 @@ "event": "action", "name": "action_listen", "policy": null, - "confidence": null + "confidence": null, + "hide_rule_turn": false }, { "timestamp": 1517821726.200036, @@ -76,14 +77,16 @@ "event": "action", "name": "utter_greet", "policy": null, - "confidence": null + "confidence": null, + "hide_rule_turn": false }, { "timestamp": 1517821726.211038, "event": "action", "name": "action_listen", "policy": null, - "confidence": null + "confidence": null, + "hide_rule_turn": false }, { "timestamp": 1517821726.209836, @@ -121,14 +124,16 @@ "event": "action", "name": "utter_happy", "policy": "policy_1_TEDPolicy", - "confidence": 0.8 + "confidence": 0.8, + "hide_rule_turn": false }, { "timestamp": 1517821726.211042, "event": "action", "name": "action_listen", "policy": "policy_2_MemoizationPolicy", - "confidence": 1.0 + "confidence": 1.0, + "hide_rule_turn": false }, { "timestamp": 1517821730.209836, @@ -166,14 +171,16 @@ "event": "action", "name": "utter_happy", "policy": "policy_1_TEDPolicy", - "confidence": 0.8 + "confidence": 0.8, + "hide_rule_turn": false }, { "timestamp": 1517821750.211042, "event": "action", "name": "action_listen", "policy": "policy_2_MemoizationPolicy", - "confidence": 1.0 + "confidence": 1.0, + "hide_rule_turn": false } ] } diff --git a/tests/test_server.py b/tests/test_server.py index 1837ab17300e..e939f96957e5 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -1110,6 +1110,7 @@ async def test_requesting_non_existent_tracker(rasa_app: SanicASGITestClient): "confidence": 1, "timestamp": 1514764800, "action_text": None, + "hide_rule_turn": False, }, {"event": "session_started", "timestamp": 1514764800}, { @@ -1119,6 +1120,7 @@ async def test_requesting_non_existent_tracker(rasa_app: SanicASGITestClient): "confidence": None, "timestamp": 1514764800, "action_text": None, + "hide_rule_turn": False, }, ] assert content["latest_message"] == { From de0a9c5c7eed5c5d1a0d8d6ef2dba6955e8fec09 Mon Sep 17 00:00:00 2001 From: Vova Vv Date: Fri, 19 Feb 2021 11:22:10 +0100 Subject: [PATCH 44/68] extract rule only staff inside rulepolicy --- examples/moodbot/config.yml | 13 +++++++------ rasa/core/policies/ensemble.py | 16 +++++----------- rasa/core/policies/rule_policy.py | 11 +++++++++++ rasa/shared/core/events.py | 21 ++++----------------- 4 files changed, 27 insertions(+), 34 deletions(-) diff --git a/examples/moodbot/config.yml b/examples/moodbot/config.yml index 5b8a3ffbaa66..bf59507654ea 100644 --- a/examples/moodbot/config.yml +++ b/examples/moodbot/config.yml @@ -1,16 +1,17 @@ language: en pipeline: - - name: "SpacyNLP" - - name: "SpacyTokenizer" - - name: "SpacyFeaturizer" - - name: "DIETClassifier" - entity_recognition: False - epochs: 50 +# - name: "SpacyNLP" + - name: "WhitespaceTokenizer" + - name: "CountVectorsFeaturizer" +# - name: "DIETClassifier" +# entity_recognition: False +# epochs: 50 policies: - name: TEDPolicy max_history: 5 epochs: 100 +# batch_strategy: sequence - name: MemoizationPolicy - name: RulePolicy diff --git a/rasa/core/policies/ensemble.py b/rasa/core/policies/ensemble.py index 6b09970e39e7..a6b1ab31e007 100644 --- a/rasa/core/policies/ensemble.py +++ b/rasa/core/policies/ensemble.py @@ -73,17 +73,11 @@ def __init__( self._rule_only_slots, self._rule_only_loops = self._get_rule_only_slots_loops() def _get_rule_only_slots_loops(self) -> Tuple[List[Text], List[Text]]: - rule_policy = next( - (policy for policy in self.policies if isinstance(policy, RulePolicy)), - None, - ) - rule_only_slots = ( - rule_policy.lookup.get(RULE_ONLY_SLOTS, []) if rule_policy else [] - ) - rule_only_loops = ( - rule_policy.lookup.get(RULE_ONLY_LOOPS, []) if rule_policy else [] - ) - return rule_only_slots, rule_only_loops + for policy in self.policies: + if isinstance(policy, RulePolicy): + return policy.get_rule_only_slots_loops() + + return [], [] def _check_for_important_policies(self) -> None: from rasa.core.policies.mapping_policy import MappingPolicy diff --git a/rasa/core/policies/rule_policy.py b/rasa/core/policies/rule_policy.py index 304787f6d817..771f49fca895 100644 --- a/rasa/core/policies/rule_policy.py +++ b/rasa/core/policies/rule_policy.py @@ -1175,6 +1175,17 @@ def _metadata(self) -> Dict[Text, Any]: def _metadata_filename(cls) -> Text: return "rule_policy.json" + def get_rule_only_slots_loops(self) -> Tuple[List[Text], List[Text]]: + """Gets the lists of slots and loops that are used only in rule data. + + Returns: + A tuple of lists of slots and loops that are used only in rule data. + """ + return ( + self.lookup.get(RULE_ONLY_SLOTS, []), + self.lookup.get(RULE_ONLY_LOOPS, []), + ) + class RulePolicyPrediction(PolicyPrediction): """Stores information about the prediction of a `RulePolicy`.""" diff --git a/rasa/shared/core/events.py b/rasa/shared/core/events.py index cec6fa7048a5..36a8e4433b56 100644 --- a/rasa/shared/core/events.py +++ b/rasa/shared/core/events.py @@ -1512,26 +1512,13 @@ def _from_story_string(cls, parameters: Dict[Text, Any]) -> Optional[List[Event] def as_dict(self) -> Dict[Text, Any]: """Returns serialized event.""" d = super().as_dict() - policy = None # for backwards compatibility (persisted events) - if hasattr(self, "policy"): - policy = self.policy - confidence = None - if hasattr(self, "confidence"): - confidence = self.confidence - action_text = None - if hasattr(self, "action_text"): - action_text = self.action_text - hide_rule_turn = False - if hasattr(self, "hide_rule_turn"): - hide_rule_turn = self.hide_rule_turn - d.update( { "name": self.action_name, - "policy": policy, - "confidence": confidence, - "action_text": action_text, - "hide_rule_turn": hide_rule_turn, + "policy": self.policy, + "confidence": self.confidence, + "action_text": self.action_text, + "hide_rule_turn": self.hide_rule_turn, } ) return d From 9706297866e3b17af842bf42e2c9a931f9737560 Mon Sep 17 00:00:00 2001 From: Vova Vv Date: Mon, 22 Feb 2021 18:46:46 +0100 Subject: [PATCH 45/68] undo changing moodbot config --- examples/moodbot/config.yml | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/examples/moodbot/config.yml b/examples/moodbot/config.yml index bf59507654ea..5b8a3ffbaa66 100644 --- a/examples/moodbot/config.yml +++ b/examples/moodbot/config.yml @@ -1,17 +1,16 @@ language: en pipeline: -# - name: "SpacyNLP" - - name: "WhitespaceTokenizer" - - name: "CountVectorsFeaturizer" -# - name: "DIETClassifier" -# entity_recognition: False -# epochs: 50 + - name: "SpacyNLP" + - name: "SpacyTokenizer" + - name: "SpacyFeaturizer" + - name: "DIETClassifier" + entity_recognition: False + epochs: 50 policies: - name: TEDPolicy max_history: 5 epochs: 100 -# batch_strategy: sequence - name: MemoizationPolicy - name: RulePolicy From d6ced62df06672d1ac492715555f40a924412148 Mon Sep 17 00:00:00 2001 From: Vova Vv Date: Tue, 9 Mar 2021 11:55:13 +0100 Subject: [PATCH 46/68] unwrap kwargs as early as possible --- rasa/core/featurizers/tracker_featurizers.py | 30 ++++-------- rasa/core/policies/form_policy.py | 7 ++- rasa/core/policies/memoization.py | 48 +++++++++++++++++--- rasa/core/policies/policy.py | 27 +++++++---- rasa/core/policies/rule_policy.py | 2 +- rasa/core/policies/sklearn_policy.py | 12 ++++- rasa/core/policies/ted_policy.py | 20 +++++--- rasa/shared/core/domain.py | 6 +-- rasa/shared/core/generator.py | 16 +++++-- rasa/shared/core/trackers.py | 10 ++-- tests/core/policies/test_ted_policy.py | 2 +- tests/core/test_policies.py | 2 +- 12 files changed, 119 insertions(+), 63 deletions(-) diff --git a/rasa/core/featurizers/tracker_featurizers.py b/rasa/core/featurizers/tracker_featurizers.py index dbe1d5f5ca40..a3bc8996ec4e 100644 --- a/rasa/core/featurizers/tracker_featurizers.py +++ b/rasa/core/featurizers/tracker_featurizers.py @@ -65,10 +65,8 @@ def _create_states( domain: a :class:`rasa.shared.core.domain.Domain` ignore_rule_only_turns: If `True` ignore dialogue turns that are present only in rules. - rule_only_slots: The list of slot names, - which only occur in rules but not in stories. - rule_only_loops: The list of loop names, - which only occur in rules but not in stories. + rule_only_slots: Slot names, which only occur in rules but not in stories. + rule_only_loops: Loop names, which only occur in rules but not in stories. Returns: a list of states @@ -273,10 +271,8 @@ def prediction_states( for featurizing last user input. ignore_rule_only_turns: If True ignore dialogue turns that are present only in rules. - rule_only_slots: The list of slot names, - which only occur in rules but not in stories. - rule_only_loops: The list of loop names, - which only occur in rules but not in stories. + rule_only_slots: Slot names, which only occur in rules but not in stories. + rule_only_loops: Loop names, which only occur in rules but not in stories. Returns: A list of states. @@ -305,10 +301,8 @@ def create_state_features( for featurizing last user input. ignore_rule_only_turns: If True ignore dialogue turns that are present only in rules. - rule_only_slots: The list of slot names, - which only occur in rules but not in stories. - rule_only_loops: The list of loop names, - which only occur in rules but not in stories. + rule_only_slots: Slot names, which only occur in rules but not in stories. + rule_only_loops: Loop names, which only occur in rules but not in stories. Returns: A list (corresponds to the list of trackers) @@ -460,10 +454,8 @@ def prediction_states( for featurizing last user input. ignore_rule_only_turns: If True ignore dialogue turns that are present only in rules. - rule_only_slots: The list of slot names, - which only occur in rules but not in stories. - rule_only_loops: The list of loop names, - which only occur in rules but not in stories. + rule_only_slots: Slot names, which only occur in rules but not in stories. + rule_only_loops: Loop names, which only occur in rules but not in stories. Returns: A list of states. @@ -631,10 +623,8 @@ def prediction_states( for featurizing last user input. ignore_rule_only_turns: If True ignore dialogue turns that are present only in rules. - rule_only_slots: The list of slot names, - which only occur in rules but not in stories. - rule_only_loops: The list of loop names, - which only occur in rules but not in stories. + rule_only_slots: Slot names, which only occur in rules but not in stories. + rule_only_loops: Loop names, which only occur in rules but not in stories. Returns: A list of states. diff --git a/rasa/core/policies/form_policy.py b/rasa/core/policies/form_policy.py index e4a0e095ce9a..5c049e87e30d 100644 --- a/rasa/core/policies/form_policy.py +++ b/rasa/core/policies/form_policy.py @@ -107,7 +107,8 @@ def recall( states: List[State], tracker: DialogueStateTracker, domain: Domain, - **kwargs: Any, + rule_only_slots: Optional[List[Text]] = None, + rule_only_loops: Optional[List[Text]] = None, ) -> Optional[Text]: """Finds the action based on the given states. @@ -115,6 +116,8 @@ def recall( states: List of states. tracker: The tracker. domain: The Domain. + rule_only_slots: Slot names, which only occur in rules but not in stories. + rule_only_loops: Loop names, which only occur in rules but not in stories. Returns: The name of the action. @@ -126,7 +129,7 @@ def state_is_unhappy(self, tracker: DialogueStateTracker, domain: Domain) -> boo # since it is assumed that training stories contain # only unhappy paths, notify the form that # it should not be validated if predicted by other policy - states = self.prediction_states(tracker, domain) + states = self._prediction_states(tracker, domain) memorized_form = self.recall(states, tracker, domain) diff --git a/rasa/core/policies/memoization.py b/rasa/core/policies/memoization.py index 79adfc6e28ad..595495e82644 100644 --- a/rasa/core/policies/memoization.py +++ b/rasa/core/policies/memoization.py @@ -199,7 +199,8 @@ def recall( states: List[State], tracker: DialogueStateTracker, domain: Domain, - **kwargs: Any, + rule_only_slots: Optional[List[Text]] = None, + rule_only_loops: Optional[List[Text]] = None, ) -> Optional[Text]: """Finds the action based on the given states. @@ -207,6 +208,8 @@ def recall( states: List of states. tracker: The tracker. domain: The Domain. + rule_only_slots: Slot names, which only occur in rules but not in stories. + rule_only_loops: Loop names, which only occur in rules but not in stories. Returns: The name of the action. @@ -236,11 +239,25 @@ def predict_action_probabilities( interpreter: NaturalLanguageInterpreter, **kwargs: Any, ) -> PolicyPrediction: + rule_only_slots = kwargs.get("rule_only_slots") + rule_only_loops = kwargs.get("rule_only_loops") + result = self._default_predictions(domain) - states = self.prediction_states(tracker, domain, **kwargs) + states = self._prediction_states( + tracker, + domain, + rule_only_slots=rule_only_slots, + rule_only_loops=rule_only_loops, + ) logger.debug(f"Current tracker state:{self.format_tracker_states(states)}") - predicted_action_name = self.recall(states, tracker, domain, **kwargs) + predicted_action_name = self.recall( + states, + tracker, + domain, + rule_only_slots=rule_only_slots, + rule_only_loops=rule_only_loops, + ) if predicted_action_name is not None: logger.debug(f"There is a memorised next action '{predicted_action_name}'") result = self._prediction_result(predicted_action_name, tracker, domain) @@ -319,7 +336,8 @@ def _recall_using_delorean( old_states: List[State], tracker: DialogueStateTracker, domain: Domain, - **kwargs, + rule_only_slots: Optional[List[Text]] = None, + rule_only_loops: Optional[List[Text]] = None, ) -> Optional[Text]: """Applies to the future idea to change the past and get the new future. @@ -330,6 +348,8 @@ def _recall_using_delorean( old_states: List of states. tracker: The tracker. domain: The Domain. + rule_only_slots: Slot names, which only occur in rules but not in stories. + rule_only_loops: Loop names, which only occur in rules but not in stories. Returns: The name of the action. @@ -338,7 +358,12 @@ def _recall_using_delorean( mcfly_tracker = self._back_to_the_future(tracker) while mcfly_tracker is not None: - states = self.prediction_states(mcfly_tracker, domain, **kwargs) + states = self._prediction_states( + mcfly_tracker, + domain, + rule_only_slots=rule_only_slots, + rule_only_loops=rule_only_loops, + ) if old_states != states: # check if we like new futures @@ -360,7 +385,8 @@ def recall( states: List[State], tracker: DialogueStateTracker, domain: Domain, - **kwargs: Any, + rule_only_slots: Optional[List[Text]] = None, + rule_only_loops: Optional[List[Text]] = None, ) -> Optional[Text]: """Finds the action based on the given states. @@ -371,6 +397,8 @@ def recall( states: List of states. tracker: The tracker. domain: The Domain. + rule_only_slots: Slot names, which only occur in rules but not in stories. + rule_only_loops: Loop names, which only occur in rules but not in stories. Returns: The name of the action. @@ -378,6 +406,12 @@ def recall( predicted_action_name = self._recall_states(states) if predicted_action_name is None: # let's try a different method to recall that tracker - return self._recall_using_delorean(states, tracker, domain, **kwargs) + return self._recall_using_delorean( + states, + tracker, + domain, + rule_only_slots=rule_only_slots, + rule_only_loops=rule_only_loops, + ) else: return predicted_action_name diff --git a/rasa/core/policies/policy.py b/rasa/core/policies/policy.py index 270bdd2790b1..c32627783989 100644 --- a/rasa/core/policies/policy.py +++ b/rasa/core/policies/policy.py @@ -151,7 +151,7 @@ def _get_valid_params(func: Callable, **kwargs: Any) -> Dict: logger.debug(f"Parameters ignored by `model.fit(...)`: {ignored_params}") return params - def featurize_for_training( + def _featurize_for_training( self, training_trackers: List[DialogueStateTracker], domain: Domain, @@ -201,12 +201,13 @@ def featurize_for_training( return state_features, label_ids, entity_tags - def prediction_states( + def _prediction_states( self, tracker: DialogueStateTracker, domain: Domain, use_text_for_last_user_input: bool = False, - **kwargs: Any, + rule_only_slots: Optional[List[Text]] = None, + rule_only_loops: Optional[List[Text]] = None, ) -> List[State]: """Transforms tracker to states for prediction. @@ -215,6 +216,8 @@ def prediction_states( domain: The Domain. use_text_for_last_user_input: Indicates whether to use text or intent label for featurizing last user input. + rule_only_slots: Slot names, which only occur in rules but not in stories. + rule_only_loops: Loop names, which only occur in rules but not in stories. Returns: A list of states. @@ -224,17 +227,18 @@ def prediction_states( domain, use_text_for_last_user_input=use_text_for_last_user_input, ignore_rule_only_turns=self.supported_data() == SupportedData.ML_DATA, - rule_only_slots=kwargs.get("rule_only_slots"), - rule_only_loops=kwargs.get("rule_only_loops"), + rule_only_slots=rule_only_slots, + rule_only_loops=rule_only_loops, )[0] - def featurize_for_prediction( + def _featurize_for_prediction( self, tracker: DialogueStateTracker, domain: Domain, interpreter: NaturalLanguageInterpreter, use_text_for_last_user_input: bool = False, - **kwargs: Any, + rule_only_slots: Optional[List[Text]] = None, + rule_only_loops: Optional[List[Text]] = None, ) -> List[List[Dict[Text, List["Features"]]]]: """Transforms training tracker into a vector representation. @@ -247,6 +251,8 @@ def featurize_for_prediction( interpreter: The NLU interpreter. use_text_for_last_user_input: Indicates whether to use text or intent label for featurizing last user input. + rule_only_slots: Slot names, which only occur in rules but not in stories. + rule_only_loops: Loop names, which only occur in rules but not in stories. Returns: A list (corresponds to the list of trackers) @@ -261,8 +267,8 @@ def featurize_for_prediction( interpreter, use_text_for_last_user_input=use_text_for_last_user_input, ignore_rule_only_turns=self.supported_data() == SupportedData.ML_DATA, - rule_only_slots=kwargs.get("rule_only_slots"), - rule_only_loops=kwargs.get("rule_only_loops"), + rule_only_slots=rule_only_slots, + rule_only_loops=rule_only_loops, ) def train( @@ -296,6 +302,9 @@ def predict_action_probabilities( domain: the :class:`rasa.shared.core.domain.Domain` interpreter: Interpreter which may be used by the policies to create additional features. + Keyword Args: + rule_only_slots: Slot names, which only occur in rules but not in stories. + rule_only_loops: Loop names, which only occur in rules but not in stories. Returns: The policy's prediction (e.g. the probabilities for the actions). diff --git a/rasa/core/policies/rule_policy.py b/rasa/core/policies/rule_policy.py index 771f49fca895..bf7f1549dd99 100644 --- a/rasa/core/policies/rule_policy.py +++ b/rasa/core/policies/rule_policy.py @@ -965,7 +965,7 @@ def _find_action_from_rules( # the text or the intent return None, None, False - states = self.prediction_states(tracker, domain, use_text_for_last_user_input) + states = self._prediction_states(tracker, domain, use_text_for_last_user_input) current_states = self.format_tracker_states(states) logger.debug(f"Current tracker state:{current_states}") diff --git a/rasa/core/policies/sklearn_policy.py b/rasa/core/policies/sklearn_policy.py index 8cea67c25bc3..036fb03197cd 100644 --- a/rasa/core/policies/sklearn_policy.py +++ b/rasa/core/policies/sklearn_policy.py @@ -234,7 +234,7 @@ def train( interpreter: NaturalLanguageInterpreter, **kwargs: Any, ) -> None: - tracker_state_features, label_ids, _ = self.featurize_for_training( + tracker_state_features, label_ids, _ = self._featurize_for_training( training_trackers, domain, interpreter, **kwargs ) training_data, zero_state_features = model_data_utils.convert_to_data_format( @@ -284,7 +284,15 @@ def predict_action_probabilities( interpreter: NaturalLanguageInterpreter, **kwargs: Any, ) -> PolicyPrediction: - X = self.featurize_for_prediction(tracker, domain, interpreter, **kwargs) + rule_only_slots = kwargs.get("rule_only_slots") + rule_only_loops = kwargs.get("rule_only_loops") + X = self._featurize_for_prediction( + tracker, + domain, + interpreter, + rule_only_slots=rule_only_slots, + rule_only_loops=rule_only_loops, + ) training_data, _ = model_data_utils.convert_to_data_format( X, self.zero_state_features ) diff --git a/rasa/core/policies/ted_policy.py b/rasa/core/policies/ted_policy.py index f02e2ae08765..e63dc3f92e61 100644 --- a/rasa/core/policies/ted_policy.py +++ b/rasa/core/policies/ted_policy.py @@ -492,7 +492,7 @@ def train( return # dealing with training data - tracker_state_features, label_ids, entity_tags = self.featurize_for_training( + tracker_state_features, label_ids, entity_tags = self._featurize_for_training( training_trackers, domain, interpreter, @@ -547,19 +547,21 @@ def _featurize_tracker_for_e2e( tracker: DialogueStateTracker, domain: Domain, interpreter: NaturalLanguageInterpreter, - **kwargs: Any, + rule_only_slots, + rule_only_loops, ) -> List[List[Dict[Text, List["Features"]]]]: # construct two examples in the batch to be fed to the model - # one by featurizing last user text # and second - an optional one (see conditions below), # the first example in the constructed batch either does not contain user input # or uses intent or text based on whether TED is e2e only. - tracker_state_features = self.featurize_for_prediction( + tracker_state_features = self._featurize_for_prediction( tracker, domain, interpreter, use_text_for_last_user_input=self.only_e2e, - **kwargs, + rule_only_slots=rule_only_slots, + rule_only_loops=rule_only_loops, ) # the second - text, but only after user utterance and if not only e2e if ( @@ -567,12 +569,13 @@ def _featurize_tracker_for_e2e( and TEXT in self.fake_features and not self.only_e2e ): - tracker_state_features += self.featurize_for_prediction( + tracker_state_features += self._featurize_for_prediction( tracker, domain, interpreter, use_text_for_last_user_input=True, - **kwargs, + rule_only_slots=rule_only_slots, + rule_only_loops=rule_only_loops, ) return tracker_state_features @@ -629,12 +632,15 @@ def predict_action_probabilities( See the docstring of the parent class `Policy` for more information. """ + rule_only_slots = kwargs.get("rule_only_slots") + rule_only_loops = kwargs.get("rule_only_loops") + if self.model is None: return self._prediction(self._default_predictions(domain)) # create model data from tracker tracker_state_features = self._featurize_tracker_for_e2e( - tracker, domain, interpreter, **kwargs + tracker, domain, interpreter, rule_only_slots, rule_only_loops ) model_data = self._create_model_data(tracker_state_features) diff --git a/rasa/shared/core/domain.py b/rasa/shared/core/domain.py index d5f7665aaf7f..cc938202a01a 100644 --- a/rasa/shared/core/domain.py +++ b/rasa/shared/core/domain.py @@ -1120,10 +1120,8 @@ def states_for_tracker_history( tracker: Instance of `DialogueStateTracker` to featurize. ignore_rule_only_turns: If True ignore dialogue turns that are present only in rules. - rule_only_slots: The list of slot names, - which only occur in rules but not in stories. - rule_only_loops: The list of loop names, - which only occur in rules but not in stories. + rule_only_slots: Slot names, which only occur in rules but not in stories. + rule_only_loops: Loop names, which only occur in rules but not in stories. Return: A list of states. diff --git a/rasa/shared/core/generator.py b/rasa/shared/core/generator.py index 48508252da00..96130261e251 100644 --- a/rasa/shared/core/generator.py +++ b/rasa/shared/core/generator.py @@ -107,11 +107,21 @@ def _unfreeze_states(frozen_states: Deque[FrozenState]) -> List[State]: for frozen_state in frozen_states ] - def past_states(self, domain: Domain, **kwargs: Any) -> List[State]: - """Generates the past states of this tracker based on the history. + def past_states( + self, + domain: Domain, + ignore_rule_only_turns: bool = False, + rule_only_slots: Optional[List[Text]] = None, + rule_only_loops: Optional[List[Text]] = None, + ) -> List[State]: + """Generate the past states of this tracker based on the history. Args: - domain: a :class:`rasa.shared.core.domain.Domain`. + domain: The Domain. + ignore_rule_only_turns: If True ignore dialogue turns that are present + only in rules. + rule_only_slots: Slot names, which only occur in rules but not in stories. + rule_only_loops: Loop names, which only occur in rules but not in stories. Returns: a list of states diff --git a/rasa/shared/core/trackers.py b/rasa/shared/core/trackers.py index a5244ac4dd5c..5689f36df019 100644 --- a/rasa/shared/core/trackers.py +++ b/rasa/shared/core/trackers.py @@ -284,13 +284,11 @@ def past_states( """Generate the past states of this tracker based on the history. Args: - domain: a :class:`rasa.shared.core.domain.Domain`. + domain: The Domain. ignore_rule_only_turns: If True ignore dialogue turns that are present only in rules. - rule_only_slots: The list of slot names, - which only occur in rules but not in stories. - rule_only_loops: The list of loop names, - which only occur in rules but not in stories. + rule_only_slots: Slot names, which only occur in rules but not in stories. + rule_only_loops: Loop names, which only occur in rules but not in stories. Returns: a list of states @@ -575,7 +573,7 @@ def _undo_till_previous_loop_execution( break if isinstance( - e, (ActionExecuted, UserUttered, DefinePrevUserUtteredFeaturization,), + e, (ActionExecuted, UserUttered, DefinePrevUserUtteredFeaturization), ): del done_events[-1 - offset] else: diff --git a/tests/core/policies/test_ted_policy.py b/tests/core/policies/test_ted_policy.py index 74f219fef08f..ff175c492adc 100644 --- a/tests/core/policies/test_ted_policy.py +++ b/tests/core/policies/test_ted_policy.py @@ -136,7 +136,7 @@ async def test_gen_batch(self, trained_policy: TEDPolicy, default_domain: Domain default_domain, augmentation_factor=0 ) interpreter = RegexInterpreter() - training_data, label_ids, entity_tags = trained_policy.featurize_for_training( + training_data, label_ids, entity_tags = trained_policy._featurize_for_training( training_trackers, default_domain, interpreter ) label_data, all_labels = trained_policy._create_label_data( diff --git a/tests/core/test_policies.py b/tests/core/test_policies.py index 0f4d17cb3f9f..e60c9973fc2f 100644 --- a/tests/core/test_policies.py +++ b/tests/core/test_policies.py @@ -421,7 +421,7 @@ def test_memorise_with_nlu( tracker = DialogueStateTracker(dialogue.name, default_domain.slots) tracker.recreate_from_dialogue(dialogue) - states = trained_policy.prediction_states(tracker, default_domain) + states = trained_policy._prediction_states(tracker, default_domain) recalled = trained_policy.recall(states, tracker, default_domain) assert recalled is not None From 2220a937989f1119e03b40786c62eb07094170b1 Mon Sep 17 00:00:00 2001 From: Vladimir Vlasov Date: Tue, 9 Mar 2021 11:56:08 +0100 Subject: [PATCH 47/68] Update rasa/core/policies/rule_policy.py Co-authored-by: Tobias Wochinger --- rasa/core/policies/rule_policy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rasa/core/policies/rule_policy.py b/rasa/core/policies/rule_policy.py index bf7f1549dd99..10af5999900b 100644 --- a/rasa/core/policies/rule_policy.py +++ b/rasa/core/policies/rule_policy.py @@ -665,7 +665,7 @@ def _analyze_rules( interpreter: Interpreter which can be used by the polices for featurization. Returns: - The list of rules that are not present in the stories. + Rules that are not present in the stories. """ logger.debug("Started checking rules and stories for contradictions.") # during training we run `predict_action_probabilities` to check for From 625dcc5295a3baf6dfce2945dd8963d768b6a9c6 Mon Sep 17 00:00:00 2001 From: Vova Vv Date: Tue, 9 Mar 2021 11:58:13 +0100 Subject: [PATCH 48/68] rename method in ensemble --- rasa/core/policies/ensemble.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/rasa/core/policies/ensemble.py b/rasa/core/policies/ensemble.py index a6b1ab31e007..45e96c914ac7 100644 --- a/rasa/core/policies/ensemble.py +++ b/rasa/core/policies/ensemble.py @@ -70,9 +70,12 @@ def __init__( self._check_priorities() self._check_for_important_policies() - self._rule_only_slots, self._rule_only_loops = self._get_rule_only_slots_loops() + ( + self._rule_only_slots, + self._rule_only_loops, + ) = self._get_rule_only_slots_and_loops() - def _get_rule_only_slots_loops(self) -> Tuple[List[Text], List[Text]]: + def _get_rule_only_slots_and_loops(self) -> Tuple[List[Text], List[Text]]: for policy in self.policies: if isinstance(policy, RulePolicy): return policy.get_rule_only_slots_loops() @@ -217,7 +220,7 @@ def train( ( self._rule_only_slots, self._rule_only_loops, - ) = self._get_rule_only_slots_loops() + ) = self._get_rule_only_slots_and_loops() else: logger.info("Skipped training, because there are no training samples.") From b6784e8310c90d1d02a955f6c489b91980b5424c Mon Sep 17 00:00:00 2001 From: Vova Vv Date: Tue, 9 Mar 2021 12:32:38 +0100 Subject: [PATCH 49/68] use version 2.0 when create domain --- tests/core/actions/test_forms.py | 4 +- tests/core/actions/test_two_stage_fallback.py | 1 + tests/core/policies/test_rule_policy.py | 101 ++++++++++++------ 3 files changed, 74 insertions(+), 32 deletions(-) diff --git a/tests/core/actions/test_forms.py b/tests/core/actions/test_forms.py index c308caa7313e..0d1a45e8595b 100644 --- a/tests/core/actions/test_forms.py +++ b/tests/core/actions/test_forms.py @@ -686,7 +686,9 @@ def test_temporary_tracker(): extra_slot = "some_slot" sender_id = "test" domain = Domain.from_yaml( - f""" slots: + f""" + version: "2.0" + slots: {extra_slot}: type: unfeaturized """ diff --git a/tests/core/actions/test_two_stage_fallback.py b/tests/core/actions/test_two_stage_fallback.py index 7420457d75d3..b9df1b1a4464 100644 --- a/tests/core/actions/test_two_stage_fallback.py +++ b/tests/core/actions/test_two_stage_fallback.py @@ -157,6 +157,7 @@ async def test_ask_rephrase_after_failed_affirmation(): domain = Domain.from_yaml( f""" + version: "2.0" responses: utter_ask_rephrase: - text: {rephrase_text} diff --git a/tests/core/policies/test_rule_policy.py b/tests/core/policies/test_rule_policy.py index af626dc2b785..60ed3f7139c6 100644 --- a/tests/core/policies/test_rule_policy.py +++ b/tests/core/policies/test_rule_policy.py @@ -109,6 +109,7 @@ def test_potential_contradiction_resolved_by_conversation_start(): utter_anti_greet_action = "utter_anti_greet" domain = Domain.from_yaml( f""" + version: "2.0" intents: - {GREET_INTENT_NAME} actions: @@ -161,6 +162,7 @@ def test_potential_contradiction_resolved_by_conversation_start_when_slot_initia some_slot_initial_value = "slot1value" domain = Domain.from_yaml( f""" + version: "2.0" intents: - {GREET_INTENT_NAME} actions: @@ -231,6 +233,7 @@ def test_potential_contradiction_resolved_by_conversation_start_when_slot_initia some_slot_initial_value = "slot1value" domain = Domain.from_yaml( f""" + version: "2.0" intents: - {GREET_INTENT_NAME} actions: @@ -294,6 +297,7 @@ def test_potential_contradiction_resolved_by_conversation_start_when_slot_initia def test_restrict_multiple_user_inputs_in_rules(): domain = Domain.from_yaml( f""" + version: "2.0" intents: - {GREET_INTENT_NAME} actions: @@ -325,6 +329,7 @@ def test_incomplete_rules_due_to_slots(): some_slot = "some_slot" domain = Domain.from_yaml( f""" + version: "2.0" intents: - {GREET_INTENT_NAME} actions: @@ -392,6 +397,7 @@ def test_no_incomplete_rules_due_to_slots_after_listen(): some_slot = "some_slot" domain = Domain.from_yaml( f""" + version: "2.0" intents: - {GREET_INTENT_NAME} actions: @@ -453,6 +459,7 @@ def test_no_incomplete_rules_due_to_additional_slots_set(): some_other_slot_value = "value2" domain = Domain.from_yaml( f""" + version: "2.0" intents: - {GREET_INTENT_NAME} actions: @@ -503,10 +510,11 @@ def test_incomplete_rules_due_to_loops(): some_form = "some_form" domain = Domain.from_yaml( f""" + version: "2.0" intents: - {GREET_INTENT_NAME} forms: -- {some_form} + {some_form}: """ ) policy = RulePolicy() @@ -566,6 +574,7 @@ def test_contradicting_rules(): utter_anti_greet_action = "utter_anti_greet" domain = Domain.from_yaml( f""" + version: "2.0" intents: - {GREET_INTENT_NAME} actions: @@ -605,6 +614,7 @@ def test_contradicting_rules_and_stories(): utter_anti_greet_action = "utter_anti_greet" domain = Domain.from_yaml( f""" + version: "2.0" intents: - {GREET_INTENT_NAME} actions: @@ -746,6 +756,7 @@ async def test_rule_policy_contradicting_rule_finetune( def test_faq_rule(): domain = Domain.from_yaml( f""" + version: "2.0" intents: - {GREET_INTENT_NAME} actions: @@ -791,6 +802,7 @@ async def test_predict_form_action_if_in_form(): domain = Domain.from_yaml( f""" + version: "2.0" intents: - {GREET_INTENT_NAME} actions: @@ -800,7 +812,7 @@ async def test_predict_form_action_if_in_form(): {REQUESTED_SLOT}: type: unfeaturized forms: - - {form_name} + {form_name}: """ ) @@ -833,6 +845,7 @@ async def test_predict_loop_action_if_in_loop_but_there_is_e2e_rule(): domain = Domain.from_yaml( f""" + version: "2.0" intents: - {GREET_INTENT_NAME} actions: @@ -842,7 +855,7 @@ async def test_predict_loop_action_if_in_loop_but_there_is_e2e_rule(): {REQUESTED_SLOT}: type: unfeaturized forms: - - {loop_name} + {loop_name}: """ ) e2e_rule = TrackerWithCachedStates.from_events( @@ -886,6 +899,7 @@ async def test_predict_form_action_if_multiple_turns(): other_intent = "bye" domain = Domain.from_yaml( f""" + version: "2.0" intents: - {GREET_INTENT_NAME} - {other_intent} @@ -896,7 +910,7 @@ async def test_predict_form_action_if_multiple_turns(): {REQUESTED_SLOT}: type: unfeaturized forms: - - {form_name} + {form_name}: """ ) @@ -935,6 +949,7 @@ async def test_predict_action_listen_after_form(): domain = Domain.from_yaml( f""" + version: "2.0" intents: - {GREET_INTENT_NAME} actions: @@ -944,7 +959,7 @@ async def test_predict_action_listen_after_form(): {REQUESTED_SLOT}: type: unfeaturized forms: - - {form_name} + {form_name}: """ ) @@ -981,6 +996,7 @@ async def test_dont_predict_form_if_already_finished(): domain = Domain.from_yaml( f""" + version: "2.0" intents: - {GREET_INTENT_NAME} actions: @@ -990,7 +1006,7 @@ async def test_dont_predict_form_if_already_finished(): {REQUESTED_SLOT}: type: unfeaturized forms: - - {form_name} + {form_name}: """ ) @@ -1031,6 +1047,7 @@ async def test_form_unhappy_path(): domain = Domain.from_yaml( f""" + version: "2.0" intents: - {GREET_INTENT_NAME} actions: @@ -1040,7 +1057,7 @@ async def test_form_unhappy_path(): {REQUESTED_SLOT}: type: unfeaturized forms: - - {form_name} + {form_name}: """ ) @@ -1075,6 +1092,7 @@ async def test_form_unhappy_path_from_general_rule(): domain = Domain.from_yaml( f""" + version: "2.0" intents: - {GREET_INTENT_NAME} actions: @@ -1084,7 +1102,7 @@ async def test_form_unhappy_path_from_general_rule(): {REQUESTED_SLOT}: type: unfeaturized forms: - - {form_name} + {form_name}: """ ) @@ -1131,6 +1149,7 @@ async def test_form_unhappy_path_from_in_form_rule(): domain = Domain.from_yaml( f""" + version: "2.0" intents: - {GREET_INTENT_NAME} actions: @@ -1141,7 +1160,7 @@ async def test_form_unhappy_path_from_in_form_rule(): {REQUESTED_SLOT}: type: unfeaturized forms: - - {form_name} + {form_name}: """ ) @@ -1206,6 +1225,7 @@ async def test_form_unhappy_path_from_story(): domain = Domain.from_yaml( f""" + version: "2.0" intents: - {GREET_INTENT_NAME} actions: @@ -1216,7 +1236,7 @@ async def test_form_unhappy_path_from_story(): {REQUESTED_SLOT}: type: unfeaturized forms: - - {form_name} + {form_name}: """ ) @@ -1280,6 +1300,7 @@ async def test_form_unhappy_path_no_validation_from_rule(): domain = Domain.from_yaml( f""" + version: "2.0" intents: - {GREET_INTENT_NAME} actions: @@ -1290,7 +1311,7 @@ async def test_form_unhappy_path_no_validation_from_rule(): {REQUESTED_SLOT}: type: unfeaturized forms: - - {form_name} + {form_name}: """ ) @@ -1371,6 +1392,7 @@ async def test_form_unhappy_path_no_validation_from_story(): domain = Domain.from_yaml( f""" + version: "2.0" intents: - {GREET_INTENT_NAME} actions: @@ -1381,7 +1403,7 @@ async def test_form_unhappy_path_no_validation_from_story(): {REQUESTED_SLOT}: type: unfeaturized forms: - - {form_name} + {form_name}: """ ) @@ -1440,6 +1462,7 @@ async def test_form_unhappy_path_without_rule(): other_intent = "bye" domain = Domain.from_yaml( f""" + version: "2.0" intents: - {GREET_INTENT_NAME} - {other_intent} @@ -1450,7 +1473,7 @@ async def test_form_unhappy_path_without_rule(): {REQUESTED_SLOT}: type: unfeaturized forms: - - {form_name} + {form_name}: """ ) @@ -1484,6 +1507,7 @@ async def test_form_activation_rule(): other_intent = "bye" domain = Domain.from_yaml( f""" + version: "2.0" intents: - {GREET_INTENT_NAME} - {other_intent} @@ -1494,7 +1518,7 @@ async def test_form_activation_rule(): {REQUESTED_SLOT}: type: unfeaturized forms: - - {form_name} + {form_name}: """ ) @@ -1523,6 +1547,7 @@ async def test_failing_form_activation_due_to_no_rule(): other_intent = "bye" domain = Domain.from_yaml( f""" + version: "2.0" intents: - {GREET_INTENT_NAME} - {other_intent} @@ -1533,7 +1558,7 @@ async def test_failing_form_activation_due_to_no_rule(): {REQUESTED_SLOT}: type: unfeaturized forms: - - {form_name} + {form_name}: """ ) @@ -1562,6 +1587,7 @@ def test_form_submit_rule(): submit_action_name = "utter_submit" domain = Domain.from_yaml( f""" + version: "2.0" intents: - {GREET_INTENT_NAME} actions: @@ -1572,7 +1598,7 @@ def test_form_submit_rule(): {REQUESTED_SLOT}: type: unfeaturized forms: - - {form_name} + {form_name}: """ ) @@ -1615,6 +1641,7 @@ def test_immediate_submit(): slot = "some_slot" domain = Domain.from_yaml( f""" + version: "2.0" intents: - {GREET_INTENT_NAME} actions: @@ -1627,7 +1654,7 @@ def test_immediate_submit(): {slot}: type: unfeaturized forms: - - {form_name} + {form_name}: entities: - {entity} """ @@ -1715,6 +1742,7 @@ async def test_rule_policy_slot_filling_from_text( async def test_one_stage_fallback_rule(): domain = Domain.from_yaml( f""" + version: "2.0" intents: - {GREET_INTENT_NAME} - {DEFAULT_NLU_FALLBACK_INTENT_NAME} @@ -1802,6 +1830,7 @@ async def test_one_stage_fallback_rule(): def test_default_actions(intent_name: Text, expected_action_name: Text): domain = Domain.from_yaml( f""" + version: "2.0" intents: - {GREET_INTENT_NAME} actions: @@ -1832,6 +1861,7 @@ def test_default_actions(intent_name: Text, expected_action_name: Text): def test_e2e_beats_default_actions(intent_name: Text): domain = Domain.from_yaml( f""" + version: "2.0" intents: - {GREET_INTENT_NAME} actions: @@ -1892,6 +1922,7 @@ def test_predict_core_fallback( other_intent = "other" domain = Domain.from_yaml( f""" + version: "2.0" intents: - {GREET_INTENT_NAME} - {other_intent} @@ -1923,6 +1954,7 @@ def test_predict_nothing_if_fallback_disabled(): other_intent = "other" domain = Domain.from_yaml( f""" + version: "2.0" intents: - {GREET_INTENT_NAME} - {other_intent} @@ -1951,6 +1983,8 @@ def test_hide_rule_turn(): action_chitchat = "action_chitchat" domain = Domain.from_yaml( f""" + version: "2.0" + intents: - {GREET_INTENT_NAME} - {chitchat} @@ -2024,12 +2058,13 @@ def test_hide_rule_turn_with_slots(): some_other_action = "some_other_action" some_intent = "some_intent" some_other_intent = "some_other_intent" - some_slot = "some_slot" + slot_which_is_only_in_rule = "slot_which_is_only_in_rule" some_slot_value = "value1" - some_other_slot = "some_other_slot" + slot_which_is_also_in_story = "slot_which_is_also_in_story" some_other_slot_value = "value2" domain = Domain.from_yaml( f""" + version: "2.0" intents: - {some_intent} - {some_other_intent} @@ -2037,9 +2072,9 @@ def test_hide_rule_turn_with_slots(): - {some_action} - {some_other_action} slots: - {some_slot}: + {slot_which_is_only_in_rule}: type: text - {some_other_slot}: + {slot_which_is_also_in_story}: type: text """ ) @@ -2053,7 +2088,7 @@ def test_hide_rule_turn_with_slots(): ActionExecuted(ACTION_LISTEN_NAME), UserUttered(intent={"name": some_intent}), ActionExecuted(some_action), - SlotSet(some_slot, some_slot_value), + SlotSet(slot_which_is_only_in_rule, some_slot_value), ActionExecuted(ACTION_LISTEN_NAME), ], is_rule_tracker=True, @@ -2063,12 +2098,12 @@ def test_hide_rule_turn_with_slots(): domain=domain, slots=domain.slots, evts=[ - SlotSet(some_other_slot, some_other_slot_value), + SlotSet(slot_which_is_also_in_story, some_other_slot_value), ActionExecuted(RULE_SNIPPET_ACTION_NAME), ActionExecuted(ACTION_LISTEN_NAME), UserUttered(intent={"name": some_intent}), ActionExecuted(some_action), - SlotSet(some_slot, some_slot_value), + SlotSet(slot_which_is_only_in_rule, some_slot_value), ActionExecuted(ACTION_LISTEN_NAME), ], is_rule_tracker=True, @@ -2081,7 +2116,7 @@ def test_hide_rule_turn_with_slots(): ActionExecuted(ACTION_LISTEN_NAME), UserUttered(intent={"name": some_other_intent}), ActionExecuted(some_other_action), - SlotSet(some_other_slot, some_other_slot_value), + SlotSet(slot_which_is_also_in_story, some_other_slot_value), ActionExecuted(ACTION_LISTEN_NAME), ], ) @@ -2092,7 +2127,7 @@ def test_hide_rule_turn_with_slots(): domain, RegexInterpreter(), ) - assert policy.lookup[RULE_ONLY_SLOTS] == [some_slot] + assert policy.lookup[RULE_ONLY_SLOTS] == [slot_which_is_only_in_rule] conversation_events = [ ActionExecuted(ACTION_LISTEN_NAME), @@ -2110,7 +2145,7 @@ def test_hide_rule_turn_with_slots(): conversation_events += [ ActionExecuted(some_action, hide_rule_turn=prediction.hide_rule_turn), - SlotSet(some_slot, some_slot_value), + SlotSet(slot_which_is_only_in_rule, some_slot_value), ] prediction = policy.predict_action_probabilities( DialogueStateTracker.from_events( @@ -2150,6 +2185,7 @@ def test_hide_rule_turn_no_last_action_listen(): followup_on_chitchat = "followup_on_chitchat" domain = Domain.from_yaml( f""" + version: "2.0" intents: - {chitchat} actions: @@ -2233,6 +2269,7 @@ def test_hide_rule_turn_with_loops(): action_chitchat = "action_chitchat" domain = Domain.from_yaml( f""" + version: "2.0" intents: - {GREET_INTENT_NAME} - {activate_form} @@ -2245,8 +2282,8 @@ def test_hide_rule_turn_with_loops(): {REQUESTED_SLOT}: type: unfeaturized forms: - - {form_name} - - {another_form_name} + {form_name}: + {another_form_name}: """ ) @@ -2338,13 +2375,14 @@ def test_do_not_hide_rule_turn_with_loops_in_stories(): activate_form = "activate_form" domain = Domain.from_yaml( f""" + version: "2.0" intents: - {activate_form} slots: {REQUESTED_SLOT}: type: unfeaturized forms: - - {form_name} + {form_name}: """ ) @@ -2394,6 +2432,7 @@ def test_hide_rule_turn_with_loops_as_followup_action(): activate_form = "activate_form" domain = Domain.from_yaml( f""" + version: "2.0" intents: - {GREET_INTENT_NAME} - {activate_form} @@ -2403,7 +2442,7 @@ def test_hide_rule_turn_with_loops_as_followup_action(): {REQUESTED_SLOT}: type: unfeaturized forms: - - {form_name} + {form_name}: """ ) From b1bf0115f705588d2596440a383e89061e55fb65 Mon Sep 17 00:00:00 2001 From: Vova Vv Date: Tue, 9 Mar 2021 12:36:01 +0100 Subject: [PATCH 50/68] use f-string --- rasa/core/policies/rule_policy.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/rasa/core/policies/rule_policy.py b/rasa/core/policies/rule_policy.py index 10af5999900b..e613bd23043a 100644 --- a/rasa/core/policies/rule_policy.py +++ b/rasa/core/policies/rule_policy.py @@ -929,10 +929,10 @@ def _find_action_from_loop_happy_path( ) return ( ACTION_LISTEN_NAME, - LOOP_RULES - + active_loop_name - + LOOP_RULES_SEPARATOR - + ACTION_LISTEN_NAME, + ( + f"{LOOP_RULES}{active_loop_name}" + f"{LOOP_RULES_SEPARATOR}{ACTION_LISTEN_NAME}" + ), ) return None, None From 2322f935bc0746235a2da2984fb47dc965abbea2 Mon Sep 17 00:00:00 2001 From: Vova Vv Date: Tue, 9 Mar 2021 15:49:10 +0100 Subject: [PATCH 51/68] add tests to remove DefinePrevUserUtteredFeaturization --- tests/shared/core/test_trackers.py | 63 ++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) diff --git a/tests/shared/core/test_trackers.py b/tests/shared/core/test_trackers.py index 9c98359ed3ef..711998c83f6c 100644 --- a/tests/shared/core/test_trackers.py +++ b/tests/shared/core/test_trackers.py @@ -896,6 +896,36 @@ def test_tracker_does_not_modify_slots( ActiveLoop(None), ], ), + ( + [ + user_uttered("trigger form"), + DefinePrevUserUtteredFeaturization(False), + ActionExecuted("form"), + ActiveLoop("form"), + SlotSet(REQUESTED_SLOT, "some slot"), + BotUttered("ask slot"), + ActionExecuted(ACTION_LISTEN_NAME), + user_uttered("fill requested slots"), + SlotSet("some slot", "value"), + DefinePrevUserUtteredFeaturization(False), + ActionExecuted("form"), + SlotSet("some slot", "value"), + SlotSet(REQUESTED_SLOT, None), + ActiveLoop(None), + ], + [ + user_uttered("trigger form"), + DefinePrevUserUtteredFeaturization(False), + ActionExecuted("form"), + ActiveLoop("form"), + SlotSet(REQUESTED_SLOT, "some slot"), + BotUttered("ask slot"), + SlotSet("some slot", "value"), + SlotSet("some slot", "value"), + SlotSet(REQUESTED_SLOT, None), + ActiveLoop(None), + ], + ), ], ) def test_applied_events_with_loop_happy_path( @@ -1097,6 +1127,39 @@ def test_applied_events_with_loop_happy_path( ActionExecuted("loop"), ], ), + ( + [ + ActionExecuted(ACTION_LISTEN_NAME), + user_uttered("greet"), + DefinePrevUserUtteredFeaturization(False), + ActionExecuted("loop"), + ActiveLoop("loop"), + ActionExecuted(ACTION_LISTEN_NAME), + user_uttered("chitchat"), + DefinePrevUserUtteredFeaturization(False), + ActionExecuted("handle_chitchat"), + ActionExecuted(ACTION_LISTEN_NAME), + user_uttered("affirm"), + DefinePrevUserUtteredFeaturization(False), + ActionExecuted("loop"), + ], + [ + ActionExecuted(ACTION_LISTEN_NAME), + user_uttered("greet"), + DefinePrevUserUtteredFeaturization(False), + ActionExecuted("loop"), + ActiveLoop("loop"), + ActionExecuted(ACTION_LISTEN_NAME), + user_uttered("chitchat"), + DefinePrevUserUtteredFeaturization(False), + # Different action than form action indicates unhappy path + ActionExecuted("handle_chitchat"), + ActionExecuted(ACTION_LISTEN_NAME), + user_uttered("affirm"), + DefinePrevUserUtteredFeaturization(False), + ActionExecuted("loop"), + ], + ), ], ) def test_applied_events_with_loop_unhappy_path( From 869f405c936aadf71986ca9db472d8c4ee1b4252 Mon Sep 17 00:00:00 2001 From: Vova Vv Date: Tue, 9 Mar 2021 18:44:50 +0100 Subject: [PATCH 52/68] test processor to ignore rules --- rasa/core/policies/ensemble.py | 1 + tests/core/conftest.py | 3 +- tests/core/test_processor.py | 110 ++++++++++++++++++++++++++++++++- 3 files changed, 110 insertions(+), 4 deletions(-) diff --git a/rasa/core/policies/ensemble.py b/rasa/core/policies/ensemble.py index 45e96c914ac7..51fcaa4eff4a 100644 --- a/rasa/core/policies/ensemble.py +++ b/rasa/core/policies/ensemble.py @@ -650,6 +650,7 @@ def _pick_best_policy( is_end_to_end_prediction=best_prediction.is_end_to_end_prediction, is_no_user_prediction=best_prediction.is_no_user_prediction, diagnostic_data=best_prediction.diagnostic_data, + hide_rule_turn=best_prediction.hide_rule_turn, ) def _best_policy_prediction( diff --git a/tests/core/conftest.py b/tests/core/conftest.py index b7f032efa08f..2d7b0096ae32 100644 --- a/tests/core/conftest.py +++ b/tests/core/conftest.py @@ -16,12 +16,11 @@ from rasa.shared.core.domain import Domain from rasa.shared.core.events import ReminderScheduled, UserUttered, ActionExecuted from rasa.core.nlg import TemplatedNaturalLanguageGenerator, NaturalLanguageGenerator -from rasa.core.policies.ensemble import PolicyEnsemble from rasa.core.policies.memoization import Policy from rasa.core.processor import MessageProcessor from rasa.shared.core.slots import Slot from rasa.core.tracker_store import InMemoryTrackerStore, MongoTrackerStore -from rasa.core.lock_store import LockStore, InMemoryLockStore +from rasa.core.lock_store import InMemoryLockStore from rasa.shared.core.trackers import DialogueStateTracker DEFAULT_DOMAIN_PATH_WITH_SLOTS = "data/test_domains/default_with_slots.yml" diff --git a/tests/core/test_processor.py b/tests/core/test_processor.py index 3b952d98b31c..8cd6afd9aa2c 100644 --- a/tests/core/test_processor.py +++ b/tests/core/test_processor.py @@ -18,7 +18,7 @@ ActionExecutionRejection, ) import rasa.core.policies.policy -from rasa.core.nlg import NaturalLanguageGenerator +from rasa.core.nlg import NaturalLanguageGenerator, TemplatedNaturalLanguageGenerator from rasa.core.policies.policy import PolicyPrediction import tests.utilities @@ -48,11 +48,13 @@ from rasa.shared.nlu.interpreter import NaturalLanguageInterpreter, RegexInterpreter from rasa.core.policies import SimplePolicyEnsemble, PolicyEnsemble from rasa.core.policies.ted_policy import TEDPolicy +from rasa.core.policies.memoization import MemoizationPolicy from rasa.core.processor import MessageProcessor -from rasa.shared.core.slots import Slot, AnySlot +from rasa.shared.core.slots import Slot from rasa.core.tracker_store import InMemoryTrackerStore from rasa.core.lock_store import InMemoryLockStore from rasa.shared.core.trackers import DialogueStateTracker +from rasa.shared.core.generator import TrackerWithCachedStates from rasa.shared.nlu.constants import INTENT_NAME_KEY from rasa.utils.endpoints import EndpointConfig from rasa.shared.core.constants import ( @@ -63,6 +65,7 @@ EXTERNAL_MESSAGE_PREFIX, IS_EXTERNAL, SESSION_START_METADATA_SLOT, + RULE_SNIPPET_ACTION_NAME, ) import logging @@ -1266,3 +1269,106 @@ def probabilities_using_best_policy( ] for event, expected in zip(tracker.events, expected_events): assert event == expected + + +def test_predict_next_action_with_hidden_rules(): + rule_intent = "rule_intent" + rule_action = "rule_action" + story_intent = "story_intent" + story_action = "story_action" + rule_slot = "rule_slot" + story_slot = "story_slot" + domain = Domain.from_yaml( + f""" + version: "2.0" + intents: + - {rule_intent} + - {story_intent} + actions: + - {rule_action} + - {story_action} + slots: + {rule_slot}: + type: text + {story_slot}: + type: text + """ + ) + + rule = TrackerWithCachedStates.from_events( + "rule", + domain=domain, + slots=domain.slots, + evts=[ + ActionExecuted(RULE_SNIPPET_ACTION_NAME), + ActionExecuted(ACTION_LISTEN_NAME), + UserUttered(intent={"name": rule_intent}), + ActionExecuted(rule_action), + SlotSet(rule_slot, rule_slot), + ActionExecuted(ACTION_LISTEN_NAME), + ], + is_rule_tracker=True, + ) + story = TrackerWithCachedStates.from_events( + "story", + domain=domain, + slots=domain.slots, + evts=[ + ActionExecuted(ACTION_LISTEN_NAME), + UserUttered(intent={"name": story_intent}), + ActionExecuted(story_action), + SlotSet(story_slot, story_slot), + ActionExecuted(ACTION_LISTEN_NAME), + ], + ) + interpreter = RegexInterpreter() + ensemble = SimplePolicyEnsemble(policies=[RulePolicy(), MemoizationPolicy()]) + ensemble.train([rule, story], domain, interpreter) + + tracker_store = InMemoryTrackerStore(domain) + lock_store = InMemoryLockStore() + processor = MessageProcessor( + interpreter, + ensemble, + domain, + tracker_store, + lock_store, + TemplatedNaturalLanguageGenerator(domain.templates), + ) + + tracker = DialogueStateTracker.from_events( + "casd", + evts=[ + ActionExecuted(ACTION_LISTEN_NAME), + UserUttered(intent={"name": rule_intent}), + ], + slots=domain.slots, + ) + action, prediction = processor.predict_next_action(tracker) + assert action._name == rule_action + assert prediction.hide_rule_turn + + processor._log_action_on_tracker( + tracker, action, [SlotSet(rule_slot, rule_slot)], prediction + ) + + action, prediction = processor.predict_next_action(tracker) + assert isinstance(action, ActionListen) + assert prediction.hide_rule_turn + + processor._log_action_on_tracker(tracker, action, None, prediction) + + tracker.events.append(UserUttered(intent={"name": story_intent})) + + # rules are hidden correctly if memo policy predicts next actions correctly + action, prediction = processor.predict_next_action(tracker) + assert action._name == story_action + assert not prediction.hide_rule_turn + + processor._log_action_on_tracker( + tracker, action, [SlotSet(story_slot, story_slot)], prediction + ) + + action, prediction = processor.predict_next_action(tracker) + assert isinstance(action, ActionListen) + assert not prediction.hide_rule_turn From 9c4f3fcdb5536e842b5a7d4414b511bda8af63e7 Mon Sep 17 00:00:00 2001 From: Vova Vv Date: Wed, 10 Mar 2021 15:09:02 +0100 Subject: [PATCH 53/68] fix domain yaml strings in test --- tests/core/policies/test_rule_policy.py | 353 ++++++++++++------------ tests/core/test_actions.py | 70 ++--- tests/core/training/test_interactive.py | 17 +- tests/shared/core/test_domain.py | 95 ++++--- tests/shared/core/test_trackers.py | 20 +- 5 files changed, 284 insertions(+), 271 deletions(-) diff --git a/tests/core/policies/test_rule_policy.py b/tests/core/policies/test_rule_policy.py index 60ed3f7139c6..5c6a986bc703 100644 --- a/tests/core/policies/test_rule_policy.py +++ b/tests/core/policies/test_rule_policy.py @@ -110,12 +110,12 @@ def test_potential_contradiction_resolved_by_conversation_start(): domain = Domain.from_yaml( f""" version: "2.0" -intents: -- {GREET_INTENT_NAME} -actions: -- {UTTER_GREET_ACTION} -- {utter_anti_greet_action} - """ + intents: + - {GREET_INTENT_NAME} + actions: + - {UTTER_GREET_ACTION} + - {utter_anti_greet_action} + """ ) policy = RulePolicy() greet_rule_at_conversation_start = TrackerWithCachedStates.from_events( @@ -163,16 +163,16 @@ def test_potential_contradiction_resolved_by_conversation_start_when_slot_initia domain = Domain.from_yaml( f""" version: "2.0" -intents: -- {GREET_INTENT_NAME} -actions: -- {UTTER_GREET_ACTION} -- {utter_anti_greet_action} -slots: - {some_slot}: - type: text - initial_value: {some_slot_initial_value} - """ + intents: + - {GREET_INTENT_NAME} + actions: + - {UTTER_GREET_ACTION} + - {utter_anti_greet_action} + slots: + {some_slot}: + type: text + initial_value: {some_slot_initial_value} + """ ) policy = RulePolicy() greet_rule_at_conversation_start = TrackerWithCachedStates.from_events( @@ -234,16 +234,16 @@ def test_potential_contradiction_resolved_by_conversation_start_when_slot_initia domain = Domain.from_yaml( f""" version: "2.0" -intents: -- {GREET_INTENT_NAME} -actions: -- {UTTER_GREET_ACTION} -- {utter_anti_greet_action} -slots: - {some_slot}: - type: text - initial_value: {some_slot_initial_value} - """ + intents: + - {GREET_INTENT_NAME} + actions: + - {UTTER_GREET_ACTION} + - {utter_anti_greet_action} + slots: + {some_slot}: + type: text + initial_value: {some_slot_initial_value} + """ ) policy = RulePolicy() greet_rule_at_conversation_start = TrackerWithCachedStates.from_events( @@ -298,11 +298,11 @@ def test_restrict_multiple_user_inputs_in_rules(): domain = Domain.from_yaml( f""" version: "2.0" -intents: -- {GREET_INTENT_NAME} -actions: -- {UTTER_GREET_ACTION} - """ + intents: + - {GREET_INTENT_NAME} + actions: + - {UTTER_GREET_ACTION} + """ ) policy = RulePolicy() greet_events = [ @@ -330,14 +330,14 @@ def test_incomplete_rules_due_to_slots(): domain = Domain.from_yaml( f""" version: "2.0" -intents: -- {GREET_INTENT_NAME} -actions: -- {some_action} -slots: - {some_slot}: - type: text - """ + intents: + - {GREET_INTENT_NAME} + actions: + - {some_action} + slots: + {some_slot}: + type: text + """ ) policy = RulePolicy() complete_rule = TrackerWithCachedStates.from_events( @@ -398,16 +398,16 @@ def test_no_incomplete_rules_due_to_slots_after_listen(): domain = Domain.from_yaml( f""" version: "2.0" -intents: -- {GREET_INTENT_NAME} -actions: -- {some_action} -entities: -- {some_slot} -slots: - {some_slot}: - type: text - """ + intents: + - {GREET_INTENT_NAME} + actions: + - {some_action} + entities: + - {some_slot} + slots: + {some_slot}: + type: text + """ ) policy = RulePolicy() complete_rule = TrackerWithCachedStates.from_events( @@ -460,16 +460,16 @@ def test_no_incomplete_rules_due_to_additional_slots_set(): domain = Domain.from_yaml( f""" version: "2.0" -intents: -- {GREET_INTENT_NAME} -actions: -- {some_action} -slots: - {some_slot}: - type: text - {some_other_slot}: - type: text - """ + intents: + - {GREET_INTENT_NAME} + actions: + - {some_action} + slots: + {some_slot}: + type: text + {some_other_slot}: + type: text + """ ) policy = RulePolicy() simple_rule = TrackerWithCachedStates.from_events( @@ -511,11 +511,11 @@ def test_incomplete_rules_due_to_loops(): domain = Domain.from_yaml( f""" version: "2.0" -intents: -- {GREET_INTENT_NAME} -forms: - {some_form}: - """ + intents: + - {GREET_INTENT_NAME} + forms: + {some_form}: + """ ) policy = RulePolicy() complete_rule = TrackerWithCachedStates.from_events( @@ -575,12 +575,12 @@ def test_contradicting_rules(): domain = Domain.from_yaml( f""" version: "2.0" -intents: -- {GREET_INTENT_NAME} -actions: -- {UTTER_GREET_ACTION} -- {utter_anti_greet_action} - """ + intents: + - {GREET_INTENT_NAME} + actions: + - {UTTER_GREET_ACTION} + - {utter_anti_greet_action} + """ ) policy = RulePolicy() anti_greet_rule = TrackerWithCachedStates.from_events( @@ -615,12 +615,12 @@ def test_contradicting_rules_and_stories(): domain = Domain.from_yaml( f""" version: "2.0" -intents: -- {GREET_INTENT_NAME} -actions: -- {UTTER_GREET_ACTION} -- {utter_anti_greet_action} - """ + intents: + - {GREET_INTENT_NAME} + actions: + - {UTTER_GREET_ACTION} + - {utter_anti_greet_action} + """ ) policy = RulePolicy() anti_greet_story = TrackerWithCachedStates.from_events( @@ -757,11 +757,11 @@ def test_faq_rule(): domain = Domain.from_yaml( f""" version: "2.0" -intents: -- {GREET_INTENT_NAME} -actions: -- {UTTER_GREET_ACTION} - """ + intents: + - {GREET_INTENT_NAME} + actions: + - {UTTER_GREET_ACTION} + """ ) policy = RulePolicy() @@ -803,17 +803,17 @@ async def test_predict_form_action_if_in_form(): domain = Domain.from_yaml( f""" version: "2.0" - intents: - - {GREET_INTENT_NAME} - actions: - - {UTTER_GREET_ACTION} - - some-action - slots: - {REQUESTED_SLOT}: - type: unfeaturized - forms: - {form_name}: -""" + intents: + - {GREET_INTENT_NAME} + actions: + - {UTTER_GREET_ACTION} + - some-action + slots: + {REQUESTED_SLOT}: + type: unfeaturized + forms: + {form_name}: + """ ) policy = RulePolicy() @@ -846,17 +846,17 @@ async def test_predict_loop_action_if_in_loop_but_there_is_e2e_rule(): domain = Domain.from_yaml( f""" version: "2.0" - intents: - - {GREET_INTENT_NAME} - actions: - - {UTTER_GREET_ACTION} - - some-action - slots: - {REQUESTED_SLOT}: - type: unfeaturized - forms: - {loop_name}: -""" + intents: + - {GREET_INTENT_NAME} + actions: + - {UTTER_GREET_ACTION} + - some-action + slots: + {REQUESTED_SLOT}: + type: unfeaturized + forms: + {loop_name}: + """ ) e2e_rule = TrackerWithCachedStates.from_events( "bla", @@ -900,18 +900,18 @@ async def test_predict_form_action_if_multiple_turns(): domain = Domain.from_yaml( f""" version: "2.0" - intents: - - {GREET_INTENT_NAME} - - {other_intent} - actions: - - {UTTER_GREET_ACTION} - - some-action - slots: - {REQUESTED_SLOT}: - type: unfeaturized - forms: - {form_name}: -""" + intents: + - {GREET_INTENT_NAME} + - {other_intent} + actions: + - {UTTER_GREET_ACTION} + - some-action + slots: + {REQUESTED_SLOT}: + type: unfeaturized + forms: + {form_name}: + """ ) policy = RulePolicy() @@ -960,7 +960,7 @@ async def test_predict_action_listen_after_form(): type: unfeaturized forms: {form_name}: - """ + """ ) policy = RulePolicy() @@ -997,17 +997,17 @@ async def test_dont_predict_form_if_already_finished(): domain = Domain.from_yaml( f""" version: "2.0" - intents: - - {GREET_INTENT_NAME} - actions: - - {UTTER_GREET_ACTION} - - some-action - slots: - {REQUESTED_SLOT}: - type: unfeaturized - forms: - {form_name}: -""" + intents: + - {GREET_INTENT_NAME} + actions: + - {UTTER_GREET_ACTION} + - some-action + slots: + {REQUESTED_SLOT}: + type: unfeaturized + forms: + {form_name}: + """ ) policy = RulePolicy() @@ -1058,7 +1058,7 @@ async def test_form_unhappy_path(): type: unfeaturized forms: {form_name}: - """ + """ ) policy = RulePolicy() @@ -1103,7 +1103,7 @@ async def test_form_unhappy_path_from_general_rule(): type: unfeaturized forms: {form_name}: - """ + """ ) policy = RulePolicy() @@ -1161,7 +1161,7 @@ async def test_form_unhappy_path_from_in_form_rule(): type: unfeaturized forms: {form_name}: - """ + """ ) unhappy_rule = TrackerWithCachedStates.from_events( @@ -1237,7 +1237,7 @@ async def test_form_unhappy_path_from_story(): type: unfeaturized forms: {form_name}: - """ + """ ) unhappy_story = TrackerWithCachedStates.from_events( @@ -1312,7 +1312,7 @@ async def test_form_unhappy_path_no_validation_from_rule(): type: unfeaturized forms: {form_name}: - """ + """ ) unhappy_rule = TrackerWithCachedStates.from_events( @@ -1404,7 +1404,7 @@ async def test_form_unhappy_path_no_validation_from_story(): type: unfeaturized forms: {form_name}: - """ + """ ) unhappy_story = TrackerWithCachedStates.from_events( @@ -1474,7 +1474,7 @@ async def test_form_unhappy_path_without_rule(): type: unfeaturized forms: {form_name}: - """ + """ ) policy = RulePolicy() @@ -1519,7 +1519,7 @@ async def test_form_activation_rule(): type: unfeaturized forms: {form_name}: - """ + """ ) form_activation_rule = _form_activation_rule(domain, form_name, other_intent) @@ -1559,7 +1559,7 @@ async def test_failing_form_activation_due_to_no_rule(): type: unfeaturized forms: {form_name}: - """ + """ ) policy = RulePolicy() @@ -1599,7 +1599,7 @@ def test_form_submit_rule(): type: unfeaturized forms: {form_name}: - """ + """ ) form_submit_rule = _form_submit_rule(domain, submit_action_name, form_name) @@ -1657,7 +1657,7 @@ def test_immediate_submit(): {form_name}: entities: - {entity} - """ + """ ) form_activation_rule = _form_activation_rule(domain, form_name, GREET_INTENT_NAME) @@ -1748,7 +1748,7 @@ async def test_one_stage_fallback_rule(): - {DEFAULT_NLU_FALLBACK_INTENT_NAME} actions: - {UTTER_GREET_ACTION} - """ + """ ) fallback_recover_rule = TrackerWithCachedStates.from_events( @@ -1831,11 +1831,11 @@ def test_default_actions(intent_name: Text, expected_action_name: Text): domain = Domain.from_yaml( f""" version: "2.0" -intents: -- {GREET_INTENT_NAME} -actions: -- {UTTER_GREET_ACTION} - """ + intents: + - {GREET_INTENT_NAME} + actions: + - {UTTER_GREET_ACTION} + """ ) policy = RulePolicy() policy.train([GREET_RULE], domain, RegexInterpreter()) @@ -1862,11 +1862,11 @@ def test_e2e_beats_default_actions(intent_name: Text): domain = Domain.from_yaml( f""" version: "2.0" -intents: -- {GREET_INTENT_NAME} -actions: -- {UTTER_GREET_ACTION} - """ + intents: + - {GREET_INTENT_NAME} + actions: + - {UTTER_GREET_ACTION} + """ ) e2e_rule = TrackerWithCachedStates.from_events( @@ -1923,12 +1923,12 @@ def test_predict_core_fallback( domain = Domain.from_yaml( f""" version: "2.0" - intents: - - {GREET_INTENT_NAME} - - {other_intent} - actions: - - {UTTER_GREET_ACTION} - - my_core_fallback + intents: + - {GREET_INTENT_NAME} + - {other_intent} + actions: + - {UTTER_GREET_ACTION} + - my_core_fallback """ ) rule_policy.train([GREET_RULE], domain, RegexInterpreter()) @@ -1955,11 +1955,11 @@ def test_predict_nothing_if_fallback_disabled(): domain = Domain.from_yaml( f""" version: "2.0" - intents: - - {GREET_INTENT_NAME} - - {other_intent} - actions: - - {UTTER_GREET_ACTION} + intents: + - {GREET_INTENT_NAME} + - {other_intent} + actions: + - {UTTER_GREET_ACTION} """ ) policy = RulePolicy(enable_fallback_prediction=False) @@ -1984,14 +1984,13 @@ def test_hide_rule_turn(): domain = Domain.from_yaml( f""" version: "2.0" - intents: - {GREET_INTENT_NAME} - {chitchat} actions: - {UTTER_GREET_ACTION} - {action_chitchat} - """ + """ ) chitchat_story = TrackerWithCachedStates.from_events( "chitchat story", @@ -2065,17 +2064,17 @@ def test_hide_rule_turn_with_slots(): domain = Domain.from_yaml( f""" version: "2.0" - intents: - - {some_intent} - - {some_other_intent} - actions: - - {some_action} - - {some_other_action} - slots: - {slot_which_is_only_in_rule}: - type: text - {slot_which_is_also_in_story}: - type: text + intents: + - {some_intent} + - {some_other_intent} + actions: + - {some_action} + - {some_other_action} + slots: + {slot_which_is_only_in_rule}: + type: text + {slot_which_is_also_in_story}: + type: text """ ) @@ -2194,7 +2193,7 @@ def test_hide_rule_turn_no_last_action_listen(): slots: {followup_on_chitchat}: type: bool - """ + """ ) simple_rule_no_last_action_listen = TrackerWithCachedStates.from_events( "simple rule without action listen in the end", @@ -2284,7 +2283,7 @@ def test_hide_rule_turn_with_loops(): forms: {form_name}: {another_form_name}: - """ + """ ) form_activation_rule = _form_activation_rule(domain, form_name, activate_form) @@ -2383,7 +2382,7 @@ def test_do_not_hide_rule_turn_with_loops_in_stories(): type: unfeaturized forms: {form_name}: - """ + """ ) form_activation_rule = _form_activation_rule(domain, form_name, activate_form) @@ -2443,7 +2442,7 @@ def test_hide_rule_turn_with_loops_as_followup_action(): type: unfeaturized forms: {form_name}: - """ + """ ) form_activation_rule = _form_activation_rule(domain, form_name, activate_form) diff --git a/tests/core/test_actions.py b/tests/core/test_actions.py index 5ab56eb67a4f..d9eb973670ee 100644 --- a/tests/core/test_actions.py +++ b/tests/core/test_actions.py @@ -752,12 +752,13 @@ def test_get_form_action(slot_mapping: Text): domain = Domain.from_yaml( textwrap.dedent( f""" - actions: - - my_action - forms: - {form_action_name}: - {slot_mapping} - """ + version: "2.0" + actions: + - my_action + forms: + {form_action_name}: + {slot_mapping} + """ ) ) @@ -771,11 +772,12 @@ def test_get_form_action_with_rasa_open_source_1_forms(): domain = Domain.from_yaml( textwrap.dedent( f""" - actions: - - my_action - forms: - - {form_action_name} - """ + version: "2.0" + actions: + - my_action + forms: + - {form_action_name} + """ ) ) @@ -788,12 +790,13 @@ def test_overridden_form_action(): domain = Domain.from_yaml( textwrap.dedent( f""" - actions: - - my_action - - {form_action_name} - forms: - {form_action_name}: - """ + version: "2.0" + actions: + - my_action + - {form_action_name} + forms: + {form_action_name}: + """ ) ) @@ -806,9 +809,10 @@ def test_get_form_action_if_not_in_forms(): domain = Domain.from_yaml( textwrap.dedent( """ - actions: - - my_action - """ + version: "2.0" + actions: + - my_action + """ ) ) @@ -823,12 +827,13 @@ def test_get_end_to_end_utterance_action(end_to_end_utterance: Text): domain = Domain.from_yaml( textwrap.dedent( f""" - actions: - - my_action - {KEY_E2E_ACTIONS}: - - {end_to_end_utterance} - - Bye Bye -""" + version: "2.0" + actions: + - my_action + {KEY_E2E_ACTIONS}: + - {end_to_end_utterance} + - Bye Bye + """ ) ) @@ -844,12 +849,13 @@ async def test_run_end_to_end_utterance_action(): domain = Domain.from_yaml( textwrap.dedent( f""" - actions: - - my_action - {KEY_E2E_ACTIONS}: - - {end_to_end_utterance} - - Bye Bye -""" + version: "2.0" + actions: + - my_action + {KEY_E2E_ACTIONS}: + - {end_to_end_utterance} + - Bye Bye + """ ) ) diff --git a/tests/core/training/test_interactive.py b/tests/core/training/test_interactive.py index 90b3203ee3df..f6ed6af30ecf 100644 --- a/tests/core/training/test_interactive.py +++ b/tests/core/training/test_interactive.py @@ -582,14 +582,15 @@ async def test_write_domain_to_file_with_form(tmp_path: Path): form_name = "my_form" old_domain = Domain.from_yaml( f""" - actions: - - utter_greet - - utter_goodbye - forms: - - {form_name} - intents: - - greet - """ + version: "2.0" + actions: + - utter_greet + - utter_goodbye + forms: + {form_name}: + intents: + - greet + """ ) events = [ActionExecuted(form_name), ActionExecuted(ACTION_LISTEN_NAME)] diff --git a/tests/shared/core/test_domain.py b/tests/shared/core/test_domain.py index d486c7c54e5a..92edfad3c81a 100644 --- a/tests/shared/core/test_domain.py +++ b/tests/shared/core/test_domain.py @@ -188,13 +188,13 @@ def test_domain_from_template(): def test_avoid_action_repetition(default_domain: Domain): domain = Domain.from_yaml( """ -actions: -- utter_greet - -responses: - utter_greet: - - text: "hi" - """ + version: "2.0" + actions: + - utter_greet + responses: + utter_greet: + - text: "hi" + """ ) assert len(domain.action_names_or_texts) == len(DEFAULT_ACTION_NAMES) + 1 @@ -448,23 +448,26 @@ def test_merge_session_config_if_first_is_not_default(): def test_merge_with_empty_domain(): domain = Domain.from_yaml( - """config: - store_entities_as_slots: false -session_config: - session_expiration_time: 20 - carry_over_slots: true -entities: -- cuisine -intents: -- greet -slots: - cuisine: - type: text -responses: - utter_goodbye: - - text: bye! - utter_greet: - - text: hey you!""" + """ + version: "2.0" + config: + store_entities_as_slots: false + session_config: + session_expiration_time: 20 + carry_over_slots: true + entities: + - cuisine + intents: + - greet + slots: + cuisine: + type: text + responses: + utter_goodbye: + - text: bye! + utter_greet: + - text: hey you! + """ ) merged = Domain.empty().merge(domain) @@ -475,23 +478,26 @@ def test_merge_with_empty_domain(): @pytest.mark.parametrize("other", [Domain.empty(), None]) def test_merge_with_empty_other_domain(other: Optional[Domain]): domain = Domain.from_yaml( - """config: - store_entities_as_slots: false -session_config: - session_expiration_time: 20 - carry_over_slots: true -entities: -- cuisine -intents: -- greet -slots: - cuisine: - type: text -responses: - utter_goodbye: - - text: bye! - utter_greet: - - text: hey you!""" + """ + version: "2.0" + config: + store_entities_as_slots: false + session_config: + session_expiration_time: 20 + carry_over_slots: true + entities: + - cuisine + intents: + - greet + slots: + cuisine: + type: text + responses: + utter_goodbye: + - text: bye! + utter_greet: + - text: hey you! + """ ) merged = domain.merge(other, override=True) @@ -925,9 +931,10 @@ def test_not_add_knowledge_base_slots(): def test_add_knowledge_base_slots(): test_domain = Domain.from_yaml( f""" -actions: -- {DEFAULT_KNOWLEDGE_BASE_ACTION} - """ + version: "2.0" + actions: + - {DEFAULT_KNOWLEDGE_BASE_ACTION} + """ ) slot_names = [s.name for s in test_domain.slots] diff --git a/tests/shared/core/test_trackers.py b/tests/shared/core/test_trackers.py index 711998c83f6c..ddd276b8a417 100644 --- a/tests/shared/core/test_trackers.py +++ b/tests/shared/core/test_trackers.py @@ -1384,16 +1384,16 @@ def test_autofill_slots_for_policy_entities(): domain = Domain.from_yaml( textwrap.dedent( f""" - entities: - - {nlu_entity} - - {policy_entity} - - slots: - {nlu_entity}: - type: text - {policy_entity}: - type: text - """ + version: "2.0" + entities: + - {nlu_entity} + - {policy_entity} + slots: + {nlu_entity}: + type: text + {policy_entity}: + type: text + """ ) ) From 72a03aa7675632c0eb3b9be7e627660398314e69 Mon Sep 17 00:00:00 2001 From: Vova Vv Date: Wed, 10 Mar 2021 15:13:48 +0100 Subject: [PATCH 54/68] mention rule only staff directly in arg --- rasa/core/policies/memoization.py | 16 ++++++++++++++-- rasa/core/policies/policy.py | 3 --- rasa/core/policies/sklearn_policy.py | 17 +++++++++++++++-- rasa/core/policies/ted_policy.py | 18 +++++++++++++----- 4 files changed, 42 insertions(+), 12 deletions(-) diff --git a/rasa/core/policies/memoization.py b/rasa/core/policies/memoization.py index 595495e82644..540140ba58e1 100644 --- a/rasa/core/policies/memoization.py +++ b/rasa/core/policies/memoization.py @@ -237,11 +237,23 @@ def predict_action_probabilities( tracker: DialogueStateTracker, domain: Domain, interpreter: NaturalLanguageInterpreter, + rule_only_slots: Optional[List[Text]] = None, + rule_only_loops: Optional[List[Text]] = None, **kwargs: Any, ) -> PolicyPrediction: - rule_only_slots = kwargs.get("rule_only_slots") - rule_only_loops = kwargs.get("rule_only_loops") + """Predicts the next action the bot should take after seeing the tracker. + + Args: + tracker: the :class:`rasa.core.trackers.DialogueStateTracker` + domain: the :class:`rasa.shared.core.domain.Domain` + interpreter: Interpreter which may be used by the policies to create + additional features. + rule_only_slots: Slot names, which only occur in rules but not in stories. + rule_only_loops: Loop names, which only occur in rules but not in stories. + Returns: + The policy's prediction (e.g. the probabilities for the actions). + """ result = self._default_predictions(domain) states = self._prediction_states( diff --git a/rasa/core/policies/policy.py b/rasa/core/policies/policy.py index c32627783989..6c98d9fd1724 100644 --- a/rasa/core/policies/policy.py +++ b/rasa/core/policies/policy.py @@ -302,9 +302,6 @@ def predict_action_probabilities( domain: the :class:`rasa.shared.core.domain.Domain` interpreter: Interpreter which may be used by the policies to create additional features. - Keyword Args: - rule_only_slots: Slot names, which only occur in rules but not in stories. - rule_only_loops: Loop names, which only occur in rules but not in stories. Returns: The policy's prediction (e.g. the probabilities for the actions). diff --git a/rasa/core/policies/sklearn_policy.py b/rasa/core/policies/sklearn_policy.py index 036fb03197cd..33a6ca41878e 100644 --- a/rasa/core/policies/sklearn_policy.py +++ b/rasa/core/policies/sklearn_policy.py @@ -282,10 +282,23 @@ def predict_action_probabilities( tracker: DialogueStateTracker, domain: Domain, interpreter: NaturalLanguageInterpreter, + rule_only_slots: Optional[List[Text]] = None, + rule_only_loops: Optional[List[Text]] = None, **kwargs: Any, ) -> PolicyPrediction: - rule_only_slots = kwargs.get("rule_only_slots") - rule_only_loops = kwargs.get("rule_only_loops") + """Predicts the next action the bot should take after seeing the tracker. + + Args: + tracker: the :class:`rasa.core.trackers.DialogueStateTracker` + domain: the :class:`rasa.shared.core.domain.Domain` + interpreter: Interpreter which may be used by the policies to create + additional features. + rule_only_slots: Slot names, which only occur in rules but not in stories. + rule_only_loops: Loop names, which only occur in rules but not in stories. + + Returns: + The policy's prediction (e.g. the probabilities for the actions). + """ X = self._featurize_for_prediction( tracker, domain, diff --git a/rasa/core/policies/ted_policy.py b/rasa/core/policies/ted_policy.py index e63dc3f92e61..08358d0fdf7d 100644 --- a/rasa/core/policies/ted_policy.py +++ b/rasa/core/policies/ted_policy.py @@ -626,15 +626,23 @@ def predict_action_probabilities( tracker: DialogueStateTracker, domain: Domain, interpreter: NaturalLanguageInterpreter, + rule_only_slots: Optional[List[Text]] = None, + rule_only_loops: Optional[List[Text]] = None, **kwargs: Any, ) -> PolicyPrediction: - """Predicts the next action the bot should take. + """Predicts the next action the bot should take after seeing the tracker. - See the docstring of the parent class `Policy` for more information. - """ - rule_only_slots = kwargs.get("rule_only_slots") - rule_only_loops = kwargs.get("rule_only_loops") + Args: + tracker: the :class:`rasa.core.trackers.DialogueStateTracker` + domain: the :class:`rasa.shared.core.domain.Domain` + interpreter: Interpreter which may be used by the policies to create + additional features. + rule_only_slots: Slot names, which only occur in rules but not in stories. + rule_only_loops: Loop names, which only occur in rules but not in stories. + Returns: + The policy's prediction (e.g. the probabilities for the actions). + """ if self.model is None: return self._prediction(self._default_predictions(domain)) From 02edec07ec9cd833d9e16a036ed5d6ea77795364 Mon Sep 17 00:00:00 2001 From: Vladimir Vlasov Date: Wed, 10 Mar 2021 15:15:19 +0100 Subject: [PATCH 55/68] Update rasa/shared/core/domain.py Co-authored-by: Tobias Wochinger --- rasa/shared/core/domain.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/rasa/shared/core/domain.py b/rasa/shared/core/domain.py index cc938202a01a..fba86d5f9de3 100644 --- a/rasa/shared/core/domain.py +++ b/rasa/shared/core/domain.py @@ -1083,8 +1083,7 @@ def _remove_rule_only_features( # remove slots which only occur in rules but not in stories if rule_only_slots: for slot in rule_only_slots: - if state.get(rasa.shared.core.constants.SLOTS, {}).get(slot): - del state[rasa.shared.core.constants.SLOTS][slot] + state.get(rasa.shared.core.constants.SLOTS, {}).pop(slot, None) # remove active loop which only occur in rules but not in stories if ( rule_only_loops From 249dcb850c995207804c1ceffa92a89879401873 Mon Sep 17 00:00:00 2001 From: Vova Vv Date: Wed, 10 Mar 2021 15:27:52 +0100 Subject: [PATCH 56/68] create _predict helper --- rasa/core/policies/rule_policy.py | 170 ++++++++++-------------------- 1 file changed, 57 insertions(+), 113 deletions(-) diff --git a/rasa/core/policies/rule_policy.py b/rasa/core/policies/rule_policy.py index e613bd23043a..e6f81d6ea01c 100644 --- a/rasa/core/policies/rule_policy.py +++ b/rasa/core/policies/rule_policy.py @@ -445,12 +445,9 @@ def _find_rule_only_slots_loops( ) def _predict_next_action( - self, - tracker: TrackerWithCachedStates, - domain: Domain, - interpreter: NaturalLanguageInterpreter, + self, tracker: TrackerWithCachedStates, domain: Domain ) -> Tuple[Optional[Text], Optional[Text]]: - prediction = self.predict_action_probabilities(tracker, domain, interpreter) + prediction, prediction_source = self._predict(tracker, domain) probabilities = prediction.probabilities # do not raise an error if RulePolicy didn't predict anything for stories; # however for rules RulePolicy should always predict an action @@ -463,17 +460,13 @@ def _predict_next_action( np.argmax(probabilities) ] - return predicted_action_name, prediction.prediction_source + return predicted_action_name, prediction_source def _predicted_action_name( - self, - tracker: TrackerWithCachedStates, - domain: Domain, - interpreter: NaturalLanguageInterpreter, - gold_action_name: Text, + self, tracker: TrackerWithCachedStates, domain: Domain, gold_action_name: Text ) -> Tuple[Optional[Text], Optional[Text]]: predicted_action_name, prediction_source = self._predict_next_action( - tracker, domain, interpreter + tracker, domain ) # if there is an active_loop, # RulePolicy will always predict active_loop first, @@ -485,7 +478,7 @@ def _predicted_action_name( ): rasa.core.test.emulate_loop_rejection(tracker) predicted_action_name, prediction_source = self._predict_next_action( - tracker, domain, interpreter + tracker, domain ) return predicted_action_name, prediction_source @@ -558,7 +551,6 @@ def _run_prediction_on_trackers( self, trackers: List[TrackerWithCachedStates], domain: Domain, - interpreter: NaturalLanguageInterpreter, collect_sources: bool, ) -> Tuple[List[Text], Set[Text]]: if collect_sources: @@ -596,7 +588,7 @@ def _run_prediction_on_trackers( gold_action_name = event.action_name or event.action_text predicted_action_name, prediction_source = self._predicted_action_name( - running_tracker, domain, interpreter, gold_action_name + running_tracker, domain, gold_action_name ) if collect_sources: self._collect_sources( @@ -627,31 +619,20 @@ def _run_prediction_on_trackers( return error_messages, rules_used_in_stories def _collect_rule_sources( - self, - rule_trackers: List[TrackerWithCachedStates], - domain: Domain, - interpreter: NaturalLanguageInterpreter, + self, rule_trackers: List[TrackerWithCachedStates], domain: Domain ) -> None: - self._run_prediction_on_trackers( - rule_trackers, domain, interpreter, collect_sources=True - ) + self._run_prediction_on_trackers(rule_trackers, domain, collect_sources=True) def _find_contradicting_and_used_in_stories_rules( - self, - trackers: List[TrackerWithCachedStates], - domain: Domain, - interpreter: NaturalLanguageInterpreter, + self, trackers: List[TrackerWithCachedStates], domain: Domain ) -> Tuple[List[Text], Set[Text]]: - return self._run_prediction_on_trackers( - trackers, domain, interpreter, collect_sources=False - ) + return self._run_prediction_on_trackers(trackers, domain, collect_sources=False) def _analyze_rules( self, rule_trackers: List[TrackerWithCachedStates], all_trackers: List[TrackerWithCachedStates], domain: Domain, - interpreter: NaturalLanguageInterpreter, ) -> List[Text]: """Analyze learned rules by running prediction on training trackers. @@ -676,13 +657,11 @@ def _analyze_rules( # we need to run prediction on rule trackers twice, because we need to collect # the information about which rule snippets contributed to the learned rules - self._collect_rule_sources(rule_trackers, domain, interpreter) + self._collect_rule_sources(rule_trackers, domain) ( error_messages, rules_used_in_stories, - ) = self._find_contradicting_and_used_in_stories_rules( - all_trackers, domain, interpreter - ) + ) = self._find_contradicting_and_used_in_stories_rules(all_trackers, domain) logger.setLevel(logger_level) # reset logger level if error_messages: @@ -779,7 +758,7 @@ def train( # using trackers here might not be the most efficient way, however # it allows us to directly test `predict_action_probabilities` method self.lookup[RULES_NOT_IN_STORIES] = self._analyze_rules( - rule_trackers, training_trackers, domain, interpreter + rule_trackers, training_trackers, domain ) logger.debug(f"Memorized '{len(self.lookup[RULES])}' unique rules.") @@ -1051,8 +1030,14 @@ def predict_action_probabilities( domain: Domain, interpreter: NaturalLanguageInterpreter, **kwargs: Any, - ) -> "RulePolicyPrediction": + ) -> "PolicyPrediction": """Predicts the next action (see parent class for more information).""" + prediction, _ = self._predict(tracker, domain) + return prediction + + def _predict( + self, tracker: DialogueStateTracker, domain: Domain + ) -> Tuple[PolicyPrediction, Text]: ( rules_action_name_from_text, prediction_source_from_text, @@ -1072,8 +1057,11 @@ def predict_action_probabilities( # text has priority over intents including default, # however loop happy path has priority over rules prediction if default_action_name and not rules_action_name_from_text: - return self._rule_prediction( - self._prediction_result(default_action_name, tracker, domain), + return ( + self._rule_prediction( + self._prediction_result(default_action_name, tracker, domain), + default_prediction_source, + ), default_prediction_source, ) @@ -1088,19 +1076,29 @@ def predict_action_probabilities( if loop_happy_path_action_name: # this prediction doesn't use user input # and happy user input anyhow should be ignored during featurization - return self._rule_prediction( - self._prediction_result(loop_happy_path_action_name, tracker, domain), + return ( + self._rule_prediction( + self._prediction_result( + loop_happy_path_action_name, tracker, domain + ), + loop_happy_path_prediction_source, + is_no_user_prediction=True, + ), loop_happy_path_prediction_source, - is_no_user_prediction=True, ) # predict rules from text first if rules_action_name_from_text: - return self._rule_prediction( - self._prediction_result(rules_action_name_from_text, tracker, domain), + return ( + self._rule_prediction( + self._prediction_result( + rules_action_name_from_text, tracker, domain + ), + prediction_source_from_text, + returning_from_unhappy_path=returning_from_unhappy_path_from_text, + is_end_to_end_prediction=True, + ), prediction_source_from_text, - returning_from_unhappy_path=returning_from_unhappy_path_from_text, - is_end_to_end_prediction=True, ) ( @@ -1118,16 +1116,19 @@ def predict_action_probabilities( else: probabilities = self._default_predictions(domain) - return self._rule_prediction( - probabilities, - prediction_source_from_intent, - returning_from_unhappy_path=( - # returning_from_unhappy_path is a negative condition, - # so `or` should be applied - returning_from_unhappy_path_from_text - or returning_from_unhappy_path_from_intent + return ( + self._rule_prediction( + probabilities, + prediction_source_from_intent, + returning_from_unhappy_path=( + # returning_from_unhappy_path is a negative condition, + # so `or` should be applied + returning_from_unhappy_path_from_text + or returning_from_unhappy_path_from_intent + ), + is_end_to_end_prediction=False, ), - is_end_to_end_prediction=False, + prediction_source_from_intent, ) def _rule_prediction( @@ -1137,8 +1138,8 @@ def _rule_prediction( returning_from_unhappy_path: bool = False, is_end_to_end_prediction: bool = False, is_no_user_prediction: bool = False, - ) -> "RulePolicyPrediction": - return RulePolicyPrediction( + ) -> PolicyPrediction: + return PolicyPrediction( probabilities, self.__class__.__name__, self.priority, @@ -1150,7 +1151,6 @@ def _rule_prediction( if prediction_source in self.lookup.get(RULES_NOT_IN_STORIES, []) else False ), - prediction_source=prediction_source, ) def _default_predictions(self, domain: Domain) -> List[float]: @@ -1185,59 +1185,3 @@ def get_rule_only_slots_loops(self) -> Tuple[List[Text], List[Text]]: self.lookup.get(RULE_ONLY_SLOTS, []), self.lookup.get(RULE_ONLY_LOOPS, []), ) - - -class RulePolicyPrediction(PolicyPrediction): - """Stores information about the prediction of a `RulePolicy`.""" - - def __init__( - self, - probabilities: List[float], - policy_name: Optional[Text], - policy_priority: int = 1, - events: Optional[List[Event]] = None, - optional_events: Optional[List[Event]] = None, - is_end_to_end_prediction: bool = False, - is_no_user_prediction: bool = False, - diagnostic_data: Optional[Dict[Text, Any]] = None, - hide_rule_turn: bool = False, - prediction_source: Optional[Text] = None, - ) -> None: - """Creates a `RulePolicyPrediction`. - - Args: - probabilities: The probabilities for each action. - policy_name: Name of the policy which made the prediction. - policy_priority: The priority of the policy which made the prediction. - events: Events which the `Policy` needs to have applied to the tracker - after the prediction. These events are applied independent of whether - the policy wins against other policies or not. Be careful which events - you return as they can potentially influence the conversation flow. - optional_events: Events which the `Policy` needs to have applied to the - tracker after the prediction in case it wins. These events are only - applied in case the policy's prediction wins. Be careful which events - you return as they can potentially influence the conversation flow. - is_end_to_end_prediction: `True` if the prediction used the text of the - user message instead of the intent. - is_no_user_prediction: `True` if the prediction uses neither the text - of the user message nor the intent. This is for the example the case - for happy loop paths. - diagnostic_data: Intermediate results or other information that is not - necessary for Rasa to function, but intended for debugging and - fine-tuning purposes. - hide_rule_turn: `True` if the prediction was made by the rules which - do not appear in the stories - prediction_source: A description of the matching rule. - """ - self.prediction_source = prediction_source - super().__init__( - probabilities, - policy_name, - policy_priority, - events, - optional_events, - is_end_to_end_prediction, - is_no_user_prediction, - diagnostic_data, - hide_rule_turn, - ) From 16984c8421d6b19589fc57aa614c514570dff37c Mon Sep 17 00:00:00 2001 From: Vova Vv Date: Wed, 10 Mar 2021 15:32:31 +0100 Subject: [PATCH 57/68] add new arguments to changelog --- changelog/7701.improvement.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/changelog/7701.improvement.md b/changelog/7701.improvement.md index 13e4d85285fd..701dac6788a9 100644 --- a/changelog/7701.improvement.md +++ b/changelog/7701.improvement.md @@ -1,3 +1,8 @@ Hide dialogue turns predicted by `RulePolicy` in the tracker states for ML-only policies like `TEDPolicy` if those dialogue turns only appear as rules in the training data and do not appear in stories. + +The method `predict_action_probabilities` of ML policies like `TEDPolicy`, +`MemoizationPolicy` and `SklearnPolicy` get new arguments: +- `rule_only_slots`: Slot names, which only occur in rules but not in stories. +- `rule_only_loops`: Loop names, which only occur in rules but not in stories. From 35978ed88b00dfb2f719343ac625df01cad40ef3 Mon Sep 17 00:00:00 2001 From: Vova Vv Date: Wed, 10 Mar 2021 15:37:43 +0100 Subject: [PATCH 58/68] add story without a rule as example to docs --- docs/docs/rules.mdx | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/docs/docs/rules.mdx b/docs/docs/rules.mdx index a7bcf719c87a..eb72eb114a21 100644 --- a/docs/docs/rules.mdx +++ b/docs/docs/rules.mdx @@ -49,6 +49,22 @@ to a send a message with an intent `greet` in the middle of an ongoing conversat Dialogue turns that only appear as rules in the training data and do not appear in stories will be ignored by ML-only policies like `TEDPolicy` at prediction time. + +```yaml-rasa +rules: +- rule: Say `hello` whenever the user sends a message with intent `greet` + steps: + - intent: greet + - action: utter_greet + +stories: +- story: story to find a restaurant + steps: + - intent: find_restaurant + - action: restaurant_form + - action: utter_restaurant_found +``` + For example if you define the greeting rule as above and don't add it to any of your stories, after `RulePolicy` predicts `utter_greet`, `TEDPolicy` will make predictions as if the `greet, utter_greet` turn did not occur. From 9ce5e91de48a92ba73ef8362556e0cdf19d6d1dc Mon Sep 17 00:00:00 2001 From: Vova Vv Date: Wed, 10 Mar 2021 17:55:40 +0100 Subject: [PATCH 59/68] don't try to fix domain yaml in test_actions.py --- tests/core/test_actions.py | 70 +++++++++++++++++--------------------- 1 file changed, 32 insertions(+), 38 deletions(-) diff --git a/tests/core/test_actions.py b/tests/core/test_actions.py index d9eb973670ee..5ab56eb67a4f 100644 --- a/tests/core/test_actions.py +++ b/tests/core/test_actions.py @@ -752,13 +752,12 @@ def test_get_form_action(slot_mapping: Text): domain = Domain.from_yaml( textwrap.dedent( f""" - version: "2.0" - actions: - - my_action - forms: - {form_action_name}: - {slot_mapping} - """ + actions: + - my_action + forms: + {form_action_name}: + {slot_mapping} + """ ) ) @@ -772,12 +771,11 @@ def test_get_form_action_with_rasa_open_source_1_forms(): domain = Domain.from_yaml( textwrap.dedent( f""" - version: "2.0" - actions: - - my_action - forms: - - {form_action_name} - """ + actions: + - my_action + forms: + - {form_action_name} + """ ) ) @@ -790,13 +788,12 @@ def test_overridden_form_action(): domain = Domain.from_yaml( textwrap.dedent( f""" - version: "2.0" - actions: - - my_action - - {form_action_name} - forms: - {form_action_name}: - """ + actions: + - my_action + - {form_action_name} + forms: + {form_action_name}: + """ ) ) @@ -809,10 +806,9 @@ def test_get_form_action_if_not_in_forms(): domain = Domain.from_yaml( textwrap.dedent( """ - version: "2.0" - actions: - - my_action - """ + actions: + - my_action + """ ) ) @@ -827,13 +823,12 @@ def test_get_end_to_end_utterance_action(end_to_end_utterance: Text): domain = Domain.from_yaml( textwrap.dedent( f""" - version: "2.0" - actions: - - my_action - {KEY_E2E_ACTIONS}: - - {end_to_end_utterance} - - Bye Bye - """ + actions: + - my_action + {KEY_E2E_ACTIONS}: + - {end_to_end_utterance} + - Bye Bye +""" ) ) @@ -849,13 +844,12 @@ async def test_run_end_to_end_utterance_action(): domain = Domain.from_yaml( textwrap.dedent( f""" - version: "2.0" - actions: - - my_action - {KEY_E2E_ACTIONS}: - - {end_to_end_utterance} - - Bye Bye - """ + actions: + - my_action + {KEY_E2E_ACTIONS}: + - {end_to_end_utterance} + - Bye Bye +""" ) ) From d65feaafc1223b89760b2a6a5671b8c2c3ea581a Mon Sep 17 00:00:00 2001 From: Vova Vv Date: Thu, 11 Mar 2021 11:15:32 +0100 Subject: [PATCH 60/68] simplify rule_only_data for policies --- rasa/core/featurizers/tracker_featurizers.py | 53 ++++++----------- rasa/core/policies/ensemble.py | 31 ++++------ rasa/core/policies/form_policy.py | 9 +-- rasa/core/policies/memoization.py | 61 +++----------------- rasa/core/policies/policy.py | 18 ++---- rasa/core/policies/rule_policy.py | 13 ++--- rasa/core/policies/sklearn_policy.py | 12 +--- rasa/core/policies/ted_policy.py | 22 +------ rasa/shared/core/domain.py | 23 +++++--- rasa/shared/core/generator.py | 7 +-- rasa/shared/core/trackers.py | 9 ++- tests/core/policies/test_rule_policy.py | 16 ++--- 12 files changed, 82 insertions(+), 192 deletions(-) diff --git a/rasa/core/featurizers/tracker_featurizers.py b/rasa/core/featurizers/tracker_featurizers.py index a3bc8996ec4e..abe41a4d401f 100644 --- a/rasa/core/featurizers/tracker_featurizers.py +++ b/rasa/core/featurizers/tracker_featurizers.py @@ -55,8 +55,7 @@ def _create_states( tracker: DialogueStateTracker, domain: Domain, ignore_rule_only_turns: bool = False, - rule_only_slots: Optional[List[Text]] = None, - rule_only_loops: Optional[List[Text]] = None, + rule_only_data: Optional[Dict[Text, Any]] = None, ) -> List[State]: """Create states for the given tracker. @@ -65,8 +64,8 @@ def _create_states( domain: a :class:`rasa.shared.core.domain.Domain` ignore_rule_only_turns: If `True` ignore dialogue turns that are present only in rules. - rule_only_slots: Slot names, which only occur in rules but not in stories. - rule_only_loops: Loop names, which only occur in rules but not in stories. + rule_only_data: Slots and loops, + which only occur in rules but not in stories. Returns: a list of states @@ -74,8 +73,7 @@ def _create_states( return tracker.past_states( domain, ignore_rule_only_turns=ignore_rule_only_turns, - rule_only_slots=rule_only_slots, - rule_only_loops=rule_only_loops, + rule_only_data=rule_only_data, ) def _featurize_states( @@ -259,8 +257,7 @@ def prediction_states( domain: Domain, use_text_for_last_user_input: bool = False, ignore_rule_only_turns: bool = False, - rule_only_slots: Optional[List[Text]] = None, - rule_only_loops: Optional[List[Text]] = None, + rule_only_data: Optional[Dict[Text, Any]] = None, ) -> List[List[State]]: """Transforms list of trackers to lists of states for prediction. @@ -271,8 +268,8 @@ def prediction_states( for featurizing last user input. ignore_rule_only_turns: If True ignore dialogue turns that are present only in rules. - rule_only_slots: Slot names, which only occur in rules but not in stories. - rule_only_loops: Loop names, which only occur in rules but not in stories. + rule_only_data: Slots and loops, + which only occur in rules but not in stories. Returns: A list of states. @@ -288,8 +285,7 @@ def create_state_features( interpreter: NaturalLanguageInterpreter, use_text_for_last_user_input: bool = False, ignore_rule_only_turns: bool = False, - rule_only_slots: Optional[List[Text]] = None, - rule_only_loops: Optional[List[Text]] = None, + rule_only_data: Optional[Dict[Text, Any]] = None, ) -> List[List[Dict[Text, List["Features"]]]]: """Create state features for prediction. @@ -301,8 +297,8 @@ def create_state_features( for featurizing last user input. ignore_rule_only_turns: If True ignore dialogue turns that are present only in rules. - rule_only_slots: Slot names, which only occur in rules but not in stories. - rule_only_loops: Loop names, which only occur in rules but not in stories. + rule_only_data: Slots and loops, + which only occur in rules but not in stories. Returns: A list (corresponds to the list of trackers) @@ -316,8 +312,7 @@ def create_state_features( domain, use_text_for_last_user_input, ignore_rule_only_turns, - rule_only_slots, - rule_only_loops, + rule_only_data, ) return self._featurize_states(trackers_as_states, interpreter) @@ -442,8 +437,7 @@ def prediction_states( domain: Domain, use_text_for_last_user_input: bool = False, ignore_rule_only_turns: bool = False, - rule_only_slots: Optional[List[Text]] = None, - rule_only_loops: Optional[List[Text]] = None, + rule_only_data: Optional[Dict[Text, Any]] = None, ) -> List[List[State]]: """Transforms list of trackers to lists of states for prediction. @@ -454,19 +448,15 @@ def prediction_states( for featurizing last user input. ignore_rule_only_turns: If True ignore dialogue turns that are present only in rules. - rule_only_slots: Slot names, which only occur in rules but not in stories. - rule_only_loops: Loop names, which only occur in rules but not in stories. + rule_only_data: Slots and loops, + which only occur in rules but not in stories. Returns: A list of states. """ trackers_as_states = [ self._create_states( - tracker, - domain, - ignore_rule_only_turns, - rule_only_slots, - rule_only_loops, + tracker, domain, ignore_rule_only_turns, rule_only_data, ) for tracker in trackers ] @@ -611,8 +601,7 @@ def prediction_states( domain: Domain, use_text_for_last_user_input: bool = False, ignore_rule_only_turns: bool = False, - rule_only_slots: Optional[List[Text]] = None, - rule_only_loops: Optional[List[Text]] = None, + rule_only_data: Optional[Dict[Text, Any]] = None, ) -> List[List[State]]: """Transforms list of trackers to lists of states for prediction. @@ -623,19 +612,15 @@ def prediction_states( for featurizing last user input. ignore_rule_only_turns: If True ignore dialogue turns that are present only in rules. - rule_only_slots: Slot names, which only occur in rules but not in stories. - rule_only_loops: Loop names, which only occur in rules but not in stories. + rule_only_data: Slots and loops, + which only occur in rules but not in stories. Returns: A list of states. """ trackers_as_states = [ self._create_states( - tracker, - domain, - ignore_rule_only_turns, - rule_only_slots, - rule_only_loops, + tracker, domain, ignore_rule_only_turns, rule_only_data, ) for tracker in trackers ] diff --git a/rasa/core/policies/ensemble.py b/rasa/core/policies/ensemble.py index 51fcaa4eff4a..10502802040f 100644 --- a/rasa/core/policies/ensemble.py +++ b/rasa/core/policies/ensemble.py @@ -30,8 +30,6 @@ ACTION_LISTEN_NAME, ACTION_RESTART_NAME, ACTION_BACK_NAME, - RULE_ONLY_SLOTS, - RULE_ONLY_LOOPS, ) from rasa.shared.core.domain import InvalidDomain, Domain from rasa.shared.core.events import ( @@ -70,17 +68,17 @@ def __init__( self._check_priorities() self._check_for_important_policies() - ( - self._rule_only_slots, - self._rule_only_loops, - ) = self._get_rule_only_slots_and_loops() + self._set_rule_only_data() - def _get_rule_only_slots_and_loops(self) -> Tuple[List[Text], List[Text]]: + def _set_rule_only_data(self) -> None: + rule_only_data = {} for policy in self.policies: if isinstance(policy, RulePolicy): - return policy.get_rule_only_slots_loops() + rule_only_data = policy.get_rule_only_data() + break - return [], [] + for policy in self.policies: + policy.set_rule_only_data(rule_only_data) def _check_for_important_policies(self) -> None: from rasa.core.policies.mapping_policy import MappingPolicy @@ -217,10 +215,9 @@ def train( self.action_fingerprints = rasa.core.training.training.create_action_fingerprints( training_trackers, domain ) - ( - self._rule_only_slots, - self._rule_only_loops, - ) = self._get_rule_only_slots_and_loops() + # set rule only data after training in order to make ensemble usable + # without loading + self._set_rule_only_data() else: logger.info("Skipped training, because there are no training samples.") @@ -706,8 +703,8 @@ def _best_policy_prediction( return self._pick_best_policy(predictions) + @staticmethod def _get_prediction( - self, policy: Policy, tracker: DialogueStateTracker, domain: Domain, @@ -723,11 +720,7 @@ def _get_prediction( and "interpreter" in arguments ): prediction = policy.predict_action_probabilities( - tracker, - domain, - interpreter, - rule_only_slots=self._rule_only_slots, - rule_only_loops=self._rule_only_loops, + tracker, domain, interpreter ) else: rasa.shared.utils.io.raise_warning( diff --git a/rasa/core/policies/form_policy.py b/rasa/core/policies/form_policy.py index 5c049e87e30d..57c9812657e5 100644 --- a/rasa/core/policies/form_policy.py +++ b/rasa/core/policies/form_policy.py @@ -103,12 +103,7 @@ def _create_lookup_from_states( return lookup def recall( - self, - states: List[State], - tracker: DialogueStateTracker, - domain: Domain, - rule_only_slots: Optional[List[Text]] = None, - rule_only_loops: Optional[List[Text]] = None, + self, states: List[State], tracker: DialogueStateTracker, domain: Domain, ) -> Optional[Text]: """Finds the action based on the given states. @@ -116,8 +111,6 @@ def recall( states: List of states. tracker: The tracker. domain: The Domain. - rule_only_slots: Slot names, which only occur in rules but not in stories. - rule_only_loops: Loop names, which only occur in rules but not in stories. Returns: The name of the action. diff --git a/rasa/core/policies/memoization.py b/rasa/core/policies/memoization.py index 540140ba58e1..401bfac371e6 100644 --- a/rasa/core/policies/memoization.py +++ b/rasa/core/policies/memoization.py @@ -195,12 +195,7 @@ def _recall_states(self, states: List[State]) -> Optional[Text]: return self.lookup.get(self._create_feature_key(states)) def recall( - self, - states: List[State], - tracker: DialogueStateTracker, - domain: Domain, - rule_only_slots: Optional[List[Text]] = None, - rule_only_loops: Optional[List[Text]] = None, + self, states: List[State], tracker: DialogueStateTracker, domain: Domain, ) -> Optional[Text]: """Finds the action based on the given states. @@ -208,8 +203,6 @@ def recall( states: List of states. tracker: The tracker. domain: The Domain. - rule_only_slots: Slot names, which only occur in rules but not in stories. - rule_only_loops: Loop names, which only occur in rules but not in stories. Returns: The name of the action. @@ -237,8 +230,6 @@ def predict_action_probabilities( tracker: DialogueStateTracker, domain: Domain, interpreter: NaturalLanguageInterpreter, - rule_only_slots: Optional[List[Text]] = None, - rule_only_loops: Optional[List[Text]] = None, **kwargs: Any, ) -> PolicyPrediction: """Predicts the next action the bot should take after seeing the tracker. @@ -248,28 +239,15 @@ def predict_action_probabilities( domain: the :class:`rasa.shared.core.domain.Domain` interpreter: Interpreter which may be used by the policies to create additional features. - rule_only_slots: Slot names, which only occur in rules but not in stories. - rule_only_loops: Loop names, which only occur in rules but not in stories. Returns: The policy's prediction (e.g. the probabilities for the actions). """ result = self._default_predictions(domain) - states = self._prediction_states( - tracker, - domain, - rule_only_slots=rule_only_slots, - rule_only_loops=rule_only_loops, - ) + states = self._prediction_states(tracker, domain) logger.debug(f"Current tracker state:{self.format_tracker_states(states)}") - predicted_action_name = self.recall( - states, - tracker, - domain, - rule_only_slots=rule_only_slots, - rule_only_loops=rule_only_loops, - ) + predicted_action_name = self.recall(states, tracker, domain) if predicted_action_name is not None: logger.debug(f"There is a memorised next action '{predicted_action_name}'") result = self._prediction_result(predicted_action_name, tracker, domain) @@ -344,12 +322,7 @@ def _back_to_the_future( return mcfly_tracker def _recall_using_delorean( - self, - old_states: List[State], - tracker: DialogueStateTracker, - domain: Domain, - rule_only_slots: Optional[List[Text]] = None, - rule_only_loops: Optional[List[Text]] = None, + self, old_states: List[State], tracker: DialogueStateTracker, domain: Domain, ) -> Optional[Text]: """Applies to the future idea to change the past and get the new future. @@ -360,8 +333,6 @@ def _recall_using_delorean( old_states: List of states. tracker: The tracker. domain: The Domain. - rule_only_slots: Slot names, which only occur in rules but not in stories. - rule_only_loops: Loop names, which only occur in rules but not in stories. Returns: The name of the action. @@ -370,12 +341,7 @@ def _recall_using_delorean( mcfly_tracker = self._back_to_the_future(tracker) while mcfly_tracker is not None: - states = self._prediction_states( - mcfly_tracker, - domain, - rule_only_slots=rule_only_slots, - rule_only_loops=rule_only_loops, - ) + states = self._prediction_states(mcfly_tracker, domain,) if old_states != states: # check if we like new futures @@ -393,12 +359,7 @@ def _recall_using_delorean( return None def recall( - self, - states: List[State], - tracker: DialogueStateTracker, - domain: Domain, - rule_only_slots: Optional[List[Text]] = None, - rule_only_loops: Optional[List[Text]] = None, + self, states: List[State], tracker: DialogueStateTracker, domain: Domain, ) -> Optional[Text]: """Finds the action based on the given states. @@ -409,8 +370,6 @@ def recall( states: List of states. tracker: The tracker. domain: The Domain. - rule_only_slots: Slot names, which only occur in rules but not in stories. - rule_only_loops: Loop names, which only occur in rules but not in stories. Returns: The name of the action. @@ -418,12 +377,6 @@ def recall( predicted_action_name = self._recall_states(states) if predicted_action_name is None: # let's try a different method to recall that tracker - return self._recall_using_delorean( - states, - tracker, - domain, - rule_only_slots=rule_only_slots, - rule_only_loops=rule_only_loops, - ) + return self._recall_using_delorean(states, tracker, domain,) else: return predicted_action_name diff --git a/rasa/core/policies/policy.py b/rasa/core/policies/policy.py index 2f25c458ee6c..92c0666e0dde 100644 --- a/rasa/core/policies/policy.py +++ b/rasa/core/policies/policy.py @@ -126,12 +126,16 @@ def __init__( self.__featurizer = self._create_featurizer(featurizer) self.priority = priority self.finetune_mode = should_finetune + self._rule_only_data = {} @property def featurizer(self): """Returns the policy's featurizer.""" return self.__featurizer + def set_rule_only_data(self, rule_only_data: Dict[Text, Any]) -> None: + self._rule_only_data = rule_only_data + @staticmethod def _get_valid_params(func: Callable, **kwargs: Any) -> Dict: """Filters out kwargs that cannot be passed to func. @@ -206,8 +210,6 @@ def _prediction_states( tracker: DialogueStateTracker, domain: Domain, use_text_for_last_user_input: bool = False, - rule_only_slots: Optional[List[Text]] = None, - rule_only_loops: Optional[List[Text]] = None, ) -> List[State]: """Transforms tracker to states for prediction. @@ -216,8 +218,6 @@ def _prediction_states( domain: The Domain. use_text_for_last_user_input: Indicates whether to use text or intent label for featurizing last user input. - rule_only_slots: Slot names, which only occur in rules but not in stories. - rule_only_loops: Loop names, which only occur in rules but not in stories. Returns: A list of states. @@ -227,8 +227,7 @@ def _prediction_states( domain, use_text_for_last_user_input=use_text_for_last_user_input, ignore_rule_only_turns=self.supported_data() == SupportedData.ML_DATA, - rule_only_slots=rule_only_slots, - rule_only_loops=rule_only_loops, + rule_only_data=self._rule_only_data, )[0] def _featurize_for_prediction( @@ -237,8 +236,6 @@ def _featurize_for_prediction( domain: Domain, interpreter: NaturalLanguageInterpreter, use_text_for_last_user_input: bool = False, - rule_only_slots: Optional[List[Text]] = None, - rule_only_loops: Optional[List[Text]] = None, ) -> List[List[Dict[Text, List["Features"]]]]: """Transforms training tracker into a vector representation. @@ -251,8 +248,6 @@ def _featurize_for_prediction( interpreter: The NLU interpreter. use_text_for_last_user_input: Indicates whether to use text or intent label for featurizing last user input. - rule_only_slots: Slot names, which only occur in rules but not in stories. - rule_only_loops: Loop names, which only occur in rules but not in stories. Returns: A list (corresponds to the list of trackers) @@ -267,8 +262,7 @@ def _featurize_for_prediction( interpreter, use_text_for_last_user_input=use_text_for_last_user_input, ignore_rule_only_turns=self.supported_data() == SupportedData.ML_DATA, - rule_only_slots=rule_only_slots, - rule_only_loops=rule_only_loops, + rule_only_data=self._rule_only_data, ) def train( diff --git a/rasa/core/policies/rule_policy.py b/rasa/core/policies/rule_policy.py index e6f81d6ea01c..3abcec966421 100644 --- a/rasa/core/policies/rule_policy.py +++ b/rasa/core/policies/rule_policy.py @@ -1175,13 +1175,12 @@ def _metadata(self) -> Dict[Text, Any]: def _metadata_filename(cls) -> Text: return "rule_policy.json" - def get_rule_only_slots_loops(self) -> Tuple[List[Text], List[Text]]: - """Gets the lists of slots and loops that are used only in rule data. + def get_rule_only_data(self) -> Dict[Text, Any]: + """Gets the slots and loops that are used only in rule data. Returns: - A tuple of lists of slots and loops that are used only in rule data. + Slots and loops that are used only in rule data. """ - return ( - self.lookup.get(RULE_ONLY_SLOTS, []), - self.lookup.get(RULE_ONLY_LOOPS, []), - ) + return { + key: self.lookup.get(key, []) for key in [RULE_ONLY_SLOTS, RULE_ONLY_LOOPS] + } diff --git a/rasa/core/policies/sklearn_policy.py b/rasa/core/policies/sklearn_policy.py index 33a6ca41878e..bd380cc8b347 100644 --- a/rasa/core/policies/sklearn_policy.py +++ b/rasa/core/policies/sklearn_policy.py @@ -282,8 +282,6 @@ def predict_action_probabilities( tracker: DialogueStateTracker, domain: Domain, interpreter: NaturalLanguageInterpreter, - rule_only_slots: Optional[List[Text]] = None, - rule_only_loops: Optional[List[Text]] = None, **kwargs: Any, ) -> PolicyPrediction: """Predicts the next action the bot should take after seeing the tracker. @@ -293,19 +291,11 @@ def predict_action_probabilities( domain: the :class:`rasa.shared.core.domain.Domain` interpreter: Interpreter which may be used by the policies to create additional features. - rule_only_slots: Slot names, which only occur in rules but not in stories. - rule_only_loops: Loop names, which only occur in rules but not in stories. Returns: The policy's prediction (e.g. the probabilities for the actions). """ - X = self._featurize_for_prediction( - tracker, - domain, - interpreter, - rule_only_slots=rule_only_slots, - rule_only_loops=rule_only_loops, - ) + X = self._featurize_for_prediction(tracker, domain, interpreter) training_data, _ = model_data_utils.convert_to_data_format( X, self.zero_state_features ) diff --git a/rasa/core/policies/ted_policy.py b/rasa/core/policies/ted_policy.py index 54166953b4bd..a9f3024bb6b5 100644 --- a/rasa/core/policies/ted_policy.py +++ b/rasa/core/policies/ted_policy.py @@ -577,8 +577,6 @@ def _featurize_tracker_for_e2e( tracker: DialogueStateTracker, domain: Domain, interpreter: NaturalLanguageInterpreter, - rule_only_slots, - rule_only_loops, ) -> List[List[Dict[Text, List["Features"]]]]: # construct two examples in the batch to be fed to the model - # one by featurizing last user text @@ -586,12 +584,7 @@ def _featurize_tracker_for_e2e( # the first example in the constructed batch either does not contain user input # or uses intent or text based on whether TED is e2e only. tracker_state_features = self._featurize_for_prediction( - tracker, - domain, - interpreter, - use_text_for_last_user_input=self.only_e2e, - rule_only_slots=rule_only_slots, - rule_only_loops=rule_only_loops, + tracker, domain, interpreter, use_text_for_last_user_input=self.only_e2e, ) # the second - text, but only after user utterance and if not only e2e if ( @@ -600,12 +593,7 @@ def _featurize_tracker_for_e2e( and not self.only_e2e ): tracker_state_features += self._featurize_for_prediction( - tracker, - domain, - interpreter, - use_text_for_last_user_input=True, - rule_only_slots=rule_only_slots, - rule_only_loops=rule_only_loops, + tracker, domain, interpreter, use_text_for_last_user_input=True, ) return tracker_state_features @@ -656,8 +644,6 @@ def predict_action_probabilities( tracker: DialogueStateTracker, domain: Domain, interpreter: NaturalLanguageInterpreter, - rule_only_slots: Optional[List[Text]] = None, - rule_only_loops: Optional[List[Text]] = None, **kwargs: Any, ) -> PolicyPrediction: """Predicts the next action the bot should take after seeing the tracker. @@ -667,8 +653,6 @@ def predict_action_probabilities( domain: the :class:`rasa.shared.core.domain.Domain` interpreter: Interpreter which may be used by the policies to create additional features. - rule_only_slots: Slot names, which only occur in rules but not in stories. - rule_only_loops: Loop names, which only occur in rules but not in stories. Returns: The policy's prediction (e.g. the probabilities for the actions). @@ -678,7 +662,7 @@ def predict_action_probabilities( # create model data from tracker tracker_state_features = self._featurize_tracker_for_e2e( - tracker, domain, interpreter, rule_only_slots, rule_only_loops + tracker, domain, interpreter ) model_data = self._create_model_data(tracker_state_features) output = self.model.rasa_predict(model_data) diff --git a/rasa/shared/core/domain.py b/rasa/shared/core/domain.py index 09f747e3fd02..fc059bbc76f0 100644 --- a/rasa/shared/core/domain.py +++ b/rasa/shared/core/domain.py @@ -1117,10 +1117,18 @@ def get_active_states(self, tracker: "DialogueStateTracker") -> State: @staticmethod def _remove_rule_only_features( - state: State, - rule_only_slots: Optional[List[Text]], - rule_only_loops: Optional[List[Text]], + state: State, rule_only_data: Optional[Dict[Text, Any]], ) -> None: + if not rule_only_data: + return + + rule_only_slots = rule_only_data.get( + rasa.shared.core.constants.RULE_ONLY_SLOTS, [] + ) + rule_only_loops = rule_only_data.get( + rasa.shared.core.constants.RULE_ONLY_LOOPS, [] + ) + # remove slots which only occur in rules but not in stories if rule_only_slots: for slot in rule_only_slots: @@ -1151,8 +1159,7 @@ def states_for_tracker_history( self, tracker: "DialogueStateTracker", ignore_rule_only_turns: bool = False, - rule_only_slots: Optional[List[Text]] = None, - rule_only_loops: Optional[List[Text]] = None, + rule_only_data: Optional[Dict[Text, Any]] = None, ) -> List[State]: """Array of states for each state of the trackers history. @@ -1160,8 +1167,8 @@ def states_for_tracker_history( tracker: Instance of `DialogueStateTracker` to featurize. ignore_rule_only_turns: If True ignore dialogue turns that are present only in rules. - rule_only_slots: Slot names, which only occur in rules but not in stories. - rule_only_loops: Loop names, which only occur in rules but not in stories. + rule_only_data: Slots and loops, + which only occur in rules but not in stories. Return: A list of states. @@ -1191,7 +1198,7 @@ def states_for_tracker_history( if ignore_rule_only_turns: # clean state from only rule features - self._remove_rule_only_features(state, rule_only_slots, rule_only_loops) + self._remove_rule_only_features(state, rule_only_data) # make sure user input is the same as for previous state # for non action_listen turns if states: diff --git a/rasa/shared/core/generator.py b/rasa/shared/core/generator.py index 96130261e251..fca246dff268 100644 --- a/rasa/shared/core/generator.py +++ b/rasa/shared/core/generator.py @@ -111,8 +111,7 @@ def past_states( self, domain: Domain, ignore_rule_only_turns: bool = False, - rule_only_slots: Optional[List[Text]] = None, - rule_only_loops: Optional[List[Text]] = None, + rule_only_data: Optional[Dict[Text, Any]] = None, ) -> List[State]: """Generate the past states of this tracker based on the history. @@ -120,8 +119,8 @@ def past_states( domain: The Domain. ignore_rule_only_turns: If True ignore dialogue turns that are present only in rules. - rule_only_slots: Slot names, which only occur in rules but not in stories. - rule_only_loops: Loop names, which only occur in rules but not in stories. + rule_only_data: Slots and loops, + which only occur in rules but not in stories. Returns: a list of states diff --git a/rasa/shared/core/trackers.py b/rasa/shared/core/trackers.py index 5689f36df019..94a1ddfa8b39 100644 --- a/rasa/shared/core/trackers.py +++ b/rasa/shared/core/trackers.py @@ -278,8 +278,7 @@ def past_states( self, domain: Domain, ignore_rule_only_turns: bool = False, - rule_only_slots: Optional[List[Text]] = None, - rule_only_loops: Optional[List[Text]] = None, + rule_only_data: Optional[Dict[Text, Any]] = None, ) -> List[State]: """Generate the past states of this tracker based on the history. @@ -287,14 +286,14 @@ def past_states( domain: The Domain. ignore_rule_only_turns: If True ignore dialogue turns that are present only in rules. - rule_only_slots: Slot names, which only occur in rules but not in stories. - rule_only_loops: Loop names, which only occur in rules but not in stories. + rule_only_data: Slots and loops, + which only occur in rules but not in stories. Returns: a list of states """ return domain.states_for_tracker_history( - self, ignore_rule_only_turns, rule_only_slots, rule_only_loops + self, ignore_rule_only_turns, rule_only_data ) def change_loop_to(self, loop_name: Optional[Text]) -> None: diff --git a/tests/core/policies/test_rule_policy.py b/tests/core/policies/test_rule_policy.py index 48dbc79a313d..ed4c31aad5a3 100644 --- a/tests/core/policies/test_rule_policy.py +++ b/tests/core/policies/test_rule_policy.py @@ -2164,9 +2164,7 @@ def test_hide_rule_turn_with_slots(): "casd", evts=conversation_events, slots=domain.slots ) states = tracker.past_states( - domain, - ignore_rule_only_turns=True, - rule_only_slots=policy.lookup[RULE_ONLY_SLOTS], + domain, ignore_rule_only_turns=True, rule_only_data=policy.get_rule_only_data() ) assert states == [ {}, @@ -2221,7 +2219,7 @@ def test_hide_rule_turn_no_last_action_listen(): ) policy = RulePolicy() policy.train( - [simple_rule_no_last_action_listen, chitchat_story], domain, RegexInterpreter(), + [simple_rule_no_last_action_listen, chitchat_story], domain, RegexInterpreter() ) assert policy.lookup[RULE_ONLY_SLOTS] == [followup_on_chitchat] @@ -2242,15 +2240,13 @@ def test_hide_rule_turn_no_last_action_listen(): assert prediction.hide_rule_turn conversation_events += [ - ActionExecuted(action_after_chitchat, hide_rule_turn=prediction.hide_rule_turn), + ActionExecuted(action_after_chitchat, hide_rule_turn=prediction.hide_rule_turn) ] tracker = DialogueStateTracker.from_events( "casd", evts=conversation_events, slots=domain.slots ) states = tracker.past_states( - domain, - ignore_rule_only_turns=True, - rule_only_slots=policy.lookup[RULE_ONLY_SLOTS], + domain, ignore_rule_only_turns=True, rule_only_data=policy.get_rule_only_data() ) assert states == [ {}, @@ -2356,9 +2352,7 @@ def test_hide_rule_turn_with_loops(): "casd", evts=conversation_events, slots=domain.slots ) states = tracker.past_states( - domain, - ignore_rule_only_turns=True, - rule_only_loops=policy.lookup[RULE_ONLY_LOOPS], + domain, ignore_rule_only_turns=True, rule_only_data=policy.get_rule_only_data() ) assert states == [ {}, From d981e270fc67fb2e8c0423858f905a2077c0df04 Mon Sep 17 00:00:00 2001 From: Vova Vv Date: Thu, 11 Mar 2021 12:45:07 +0100 Subject: [PATCH 61/68] add docstring --- rasa/core/policies/policy.py | 1 + 1 file changed, 1 insertion(+) diff --git a/rasa/core/policies/policy.py b/rasa/core/policies/policy.py index 92c0666e0dde..8c8607c85ddf 100644 --- a/rasa/core/policies/policy.py +++ b/rasa/core/policies/policy.py @@ -134,6 +134,7 @@ def featurizer(self): return self.__featurizer def set_rule_only_data(self, rule_only_data: Dict[Text, Any]) -> None: + """Sets policy's shared rule only data needed for correct featurization.""" self._rule_only_data = rule_only_data @staticmethod From 34d48dd41b80934034128ef463c67730aa3aeccc Mon Sep 17 00:00:00 2001 From: Vova Vv Date: Mon, 15 Mar 2021 13:18:19 +0100 Subject: [PATCH 62/68] fix omit_unset_slots merge --- rasa/core/featurizers/tracker_featurizers.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/rasa/core/featurizers/tracker_featurizers.py b/rasa/core/featurizers/tracker_featurizers.py index a01c985b83ae..4f9ff345c1e9 100644 --- a/rasa/core/featurizers/tracker_featurizers.py +++ b/rasa/core/featurizers/tracker_featurizers.py @@ -475,7 +475,10 @@ def prediction_states( """ trackers_as_states = [ self._create_states( - tracker, domain, ignore_rule_only_turns, rule_only_data, + tracker, + domain, + ignore_rule_only_turns=ignore_rule_only_turns, + rule_only_data=rule_only_data, ) for tracker in trackers ] @@ -645,7 +648,10 @@ def prediction_states( """ trackers_as_states = [ self._create_states( - tracker, domain, ignore_rule_only_turns, rule_only_data, + tracker, + domain, + ignore_rule_only_turns=ignore_rule_only_turns, + rule_only_data=rule_only_data, ) for tracker in trackers ] From 8a298ec1078f71ae3848a2d63c24d9e74eebd579 Mon Sep 17 00:00:00 2001 From: Vova Vv Date: Thu, 18 Mar 2021 17:52:18 +0100 Subject: [PATCH 63/68] rename to shared states --- rasa/core/policies/ensemble.py | 2 +- rasa/core/policies/policy.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/rasa/core/policies/ensemble.py b/rasa/core/policies/ensemble.py index c7fd1008d606..fd6ca76dca10 100644 --- a/rasa/core/policies/ensemble.py +++ b/rasa/core/policies/ensemble.py @@ -78,7 +78,7 @@ def _set_rule_only_data(self) -> None: break for policy in self.policies: - policy.set_rule_only_data(rule_only_data) + policy.set_shared_policy_states(rule_only_data=rule_only_data) def _check_for_important_policies(self) -> None: from rasa.core.policies.mapping_policy import MappingPolicy diff --git a/rasa/core/policies/policy.py b/rasa/core/policies/policy.py index 06a3b29d9744..5bd870d13117 100644 --- a/rasa/core/policies/policy.py +++ b/rasa/core/policies/policy.py @@ -133,9 +133,9 @@ def featurizer(self): """Returns the policy's featurizer.""" return self.__featurizer - def set_rule_only_data(self, rule_only_data: Dict[Text, Any]) -> None: - """Sets policy's shared rule only data needed for correct featurization.""" - self._rule_only_data = rule_only_data + def set_shared_policy_states(self, **kwargs: Any) -> None: + """Sets policy's shared states for correct featurization.""" + self._rule_only_data = kwargs.get("rule_only_data", {}) @staticmethod def _get_valid_params(func: Callable, **kwargs: Any) -> Dict: From 5d5f83dc30f7d3fa16f45f7fd1ef406cedd5d866 Mon Sep 17 00:00:00 2001 From: Vladimir Vlasov Date: Fri, 19 Mar 2021 10:23:14 +0100 Subject: [PATCH 64/68] Update rasa/shared/core/domain.py Co-authored-by: Tobias Wochinger --- rasa/shared/core/domain.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rasa/shared/core/domain.py b/rasa/shared/core/domain.py index 1c3c18b9cad8..c2b1a46e87a9 100644 --- a/rasa/shared/core/domain.py +++ b/rasa/shared/core/domain.py @@ -1232,7 +1232,7 @@ def states_for_tracker_history( return states def slots_for_entities(self, entities: List[Dict[Text, Any]]) -> List[SlotSet]: - """Find which slots should be auto fill with entities. + """Creates slot events for entities if auto-filling is enabled. Args: entities: The list of entities. From e2b7342290d0d38dab0e5885867fb90ffa5e0bff Mon Sep 17 00:00:00 2001 From: Vladimir Vlasov Date: Fri, 19 Mar 2021 10:23:37 +0100 Subject: [PATCH 65/68] Update rasa/shared/core/trackers.py Co-authored-by: Tobias Wochinger --- rasa/shared/core/trackers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rasa/shared/core/trackers.py b/rasa/shared/core/trackers.py index 8da620066551..9c71ff4bca0a 100644 --- a/rasa/shared/core/trackers.py +++ b/rasa/shared/core/trackers.py @@ -295,7 +295,7 @@ def past_states( ignore_rule_only_turns: bool = False, rule_only_data: Optional[Dict[Text, Any]] = None, ) -> List[State]: - """Generate the past states of this tracker based on the history. + """Generates the past states of this tracker based on the history. Args: domain: The Domain. From fb039cdba8e3844f82a3ed08c1d99d587205d252 Mon Sep 17 00:00:00 2001 From: Vladimir Vlasov Date: Fri, 19 Mar 2021 10:23:49 +0100 Subject: [PATCH 66/68] Update rasa/core/policies/rule_policy.py Co-authored-by: Tobias Wochinger --- rasa/core/policies/rule_policy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rasa/core/policies/rule_policy.py b/rasa/core/policies/rule_policy.py index de8dab5a4f53..13a7f8089f14 100644 --- a/rasa/core/policies/rule_policy.py +++ b/rasa/core/policies/rule_policy.py @@ -634,7 +634,7 @@ def _analyze_rules( all_trackers: List[TrackerWithCachedStates], domain: Domain, ) -> List[Text]: - """Analyze learned rules by running prediction on training trackers. + """Analyzes learned rules by running prediction on training trackers. This method collects error messages for contradicting rules and creates the lookup for rules that are not present in the stories. From c1ed6ba87a977802897650384cbb02ec42ecb76a Mon Sep 17 00:00:00 2001 From: Vladimir Vlasov Date: Fri, 19 Mar 2021 10:24:47 +0100 Subject: [PATCH 67/68] Update rasa/shared/core/generator.py Co-authored-by: Tobias Wochinger --- rasa/shared/core/generator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rasa/shared/core/generator.py b/rasa/shared/core/generator.py index d4c059b8ef5f..f73e99f11377 100644 --- a/rasa/shared/core/generator.py +++ b/rasa/shared/core/generator.py @@ -125,7 +125,7 @@ def past_states( ignore_rule_only_turns: bool = False, rule_only_data: Optional[Dict[Text, Any]] = None, ) -> List[State]: - """Generate the past states of this tracker based on the history. + """Generates the past states of this tracker based on the history. Args: domain: The Domain. From fe2aab6950372b61906c0fc3529cea05aa96ddc0 Mon Sep 17 00:00:00 2001 From: Vova Vv Date: Fri, 19 Mar 2021 10:30:16 +0100 Subject: [PATCH 68/68] update changelog --- changelog/7701.improvement.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/changelog/7701.improvement.md b/changelog/7701.improvement.md index 701dac6788a9..357e3e54ffde 100644 --- a/changelog/7701.improvement.md +++ b/changelog/7701.improvement.md @@ -2,7 +2,9 @@ Hide dialogue turns predicted by `RulePolicy` in the tracker states for ML-only policies like `TEDPolicy` if those dialogue turns only appear as rules in the training data and do not appear in stories. -The method `predict_action_probabilities` of ML policies like `TEDPolicy`, -`MemoizationPolicy` and `SklearnPolicy` get new arguments: +Add `set_shared_policy_states(...)` method to all policies. +This method sets `_rule_only_data` dict with keys: - `rule_only_slots`: Slot names, which only occur in rules but not in stories. - `rule_only_loops`: Loop names, which only occur in rules but not in stories. + +This information is needed for correct featurization to hide dialogue turns that appear only in rules.