Drop 10 work

chatbot/consumers/async_consumer.py

@@ -2,6 +2,7 @@
 import json
 import os
 from django.conf import settings
+from django.db import transaction
 from chatbot.celery_tasks.common_chat_tasks import save_in_company_db
 from chatbot.consumers.async_base_consumer import AsyncBaseConsumer
 from chatbot.models import ChatStatus, ChatSession, Profile, CompanyBot, Voice, VoiceType, ChatType, CompanyChat, \
@@ -43,6 +44,15 @@ async def disconnect(self, code):
             # Don't call self.close() here - let the parent handle that
             await super().disconnect(code)
 
+    async def profile_update(self, event):
+        """Merge freshly-submitted profile values (e.g. from submit_user_context) into the live session's
+        ums_profile, prioritizing them over the stale snapshot fetched at authenticate time."""
+        updates = event.get('ums_profile_updates') or {}
+        if not updates:
+            return
+        self.ums_profile = {**(self.ums_profile or {}), **updates}
+        logger.info('[profile_update] merged into live ums_profile=%s', self.ums_profile)
+
     async def receive(self, text_data):
         self.last_activity = asyncio.get_running_loop().time()
         try:
@@ -266,15 +276,21 @@ def create_chat_session(self, session_id, profile, company_bot, ip_address, user
         logger.info(f"Chatsession: %s %s", cs, cs_created)
 
         if not cs_created:
-            if cs.language != self.route:
-                cs.language = self.route
+            # Locked so this read-modify-write of other_params (on a reconnect) can't
+            # race with a still-in-flight celery task's other_params writes for a
+            # previous turn on this same session (usage, finalized_sources, the
+            # pending text-conversion override).
+            with transaction.atomic():
+                cs = ChatSession.objects.select_for_update().get(pk=cs.pk)
+                if cs.language != self.route:
+                    cs.language = self.route
 
-            other_params = cs.other_params or {}
-            other_params["ip_address"] = ip_address
+                other_params = cs.other_params or {}
+                other_params["ip_address"] = ip_address
 
-            cs.other_params = other_params
+                cs.other_params = other_params
 
-            cs.save(update_fields=["language", "other_params"])
+                cs.save(update_fields=["language", "other_params"])
         else:
             cs.other_params = {"ip_address": ip_address}
             cs.save(update_fields=["other_params"])
@@ -285,6 +301,25 @@ def create_chat_session(self, session_id, profile, company_bot, ip_address, user
     @database_sync_to_async
     def translate_message(self, message):
         try:
+            # One-shot override: respond_to_user.next_reply_conversion from the previous bot
+            # turn (persisted in ChatSession.other_params by
+            # BaseResponseHandler._save_pending_text_conversion) takes priority over the
+            # state's static text_conversion_type. Consumed and cleared here, before any
+            # early return below, so it's removed exactly once regardless of whether a
+            # voice provider is actually configured for this message. Locked so this
+            # read-modify-write can't race with a concurrent _save_pending_text_conversion
+            # write for the next turn (e.g. the user sends a new message before the
+            # previous turn's celery task has finished writing other_params).
+            pending_conversion = None
+            with transaction.atomic():
+                chat_session = ChatSession.objects.select_for_update().filter(session=self.session_id).first()
+                if chat_session:
+                    other_params = chat_session.other_params or {}
+                    pending_conversion = other_params.pop('pending_text_conversion_type', None)
+                    if pending_conversion is not None:
+                        chat_session.other_params = other_params
+                        chat_session.save(update_fields=['other_params'])
+
             if not self.company_bot:
                 return message
 
@@ -297,15 +332,20 @@ def translate_message(self, message):
             if not voice_provider:
                 return message
 
-            chat_session = ChatSession.objects.filter(session=self.session_id).first()
             if not chat_session:
                 return message
 
-            state_machine = CompanyStateMachine.objects.filter(
-                company_bot=self.company_bot, step=chat_session.current_step
-            ).first()
+            if pending_conversion is not None:
+                use_transliterate = str(pending_conversion).strip().upper() == TextConversionType.TRANSLITERATE
+            else:
+                state_machine = CompanyStateMachine.objects.filter(
+                    company_bot=self.company_bot, step=chat_session.current_step
+                ).first()
+                use_transliterate = bool(
+                    state_machine and state_machine.text_conversion_type == TextConversionType.TRANSLITERATE
+                )
 
-            if state_machine and state_machine.text_conversion_type == TextConversionType.TRANSLITERATE:
+            if use_transliterate:
                 transliterate_voice_provider = Voice.objects.filter(
                     company_bot=self.company_bot,
                     type=VoiceType.Transliterate,

chatbot/scripts/check_kb_web_search_scores.py

@@ -0,0 +1,91 @@
+# Standalone shell_plus script.
+#
+# Usage: open `python manage.py shell_plus`, paste this ENTIRE file, then run:
+#
+#     check_kb_web_search_scores(output_path='/path/to/kb_web_search_scores.txt')
+#
+# It's wrapped in exec("""...""") on purpose — some shell_plus setups (plain
+# Python REPL inside tmux/screen, no bracketed paste) mis-parse a pasted
+# multi-line script because blank lines inside a function body look like
+# "end of block" to the incremental parser. Wrapping the whole body in a
+# single string literal sidesteps that: the REPL just buffers lines until
+# the closing triple-quote, then exec() runs it as one unit.
+#
+# Scans CompanyChat.chunks and splits chats into two groups, written as two
+# clearly separate sections in the output file:
+#   - KB + WEB SEARCH: both a kb_search and a web_search chunk are present
+#     (KB had a result, but web search still got pulled in)
+#   - KB ONLY: kb_search chunks present, no web_search chunk at all
+#     (KB result was accepted on its own)
+# For each chat, writes every score found in its chunks, keyed by chat id.
+# No DB writes — read-only.
+
+exec("""
+import json
+from chatbot.models import CompanyChat
+
+
+def _parse_chunks(raw):
+    if not raw:
+        return None
+    try:
+        data = json.loads(raw) if isinstance(raw, str) else raw
+    except (json.JSONDecodeError, TypeError):
+        return None
+    return data if isinstance(data, list) else None
+
+
+def _split_kb_and_web_search_scores():
+    \"\"\"Return (kb_and_web, kb_only) — each a list of (chat_id, scores).\"\"\"
+    kb_and_web = []
+    kb_only = []
+
+    qs = CompanyChat.objects.exclude(chunks__isnull=True).exclude(chunks='').order_by('id')
+    for chat in qs.iterator():
+        chunks = _parse_chunks(chat.chunks)
+        if not chunks:
+            continue
+
+        sources = {c.get('source') for c in chunks if isinstance(c, dict)}
+        if 'kb_search' not in sources:
+            continue
+
+        scores = [c.get('score') for c in chunks if isinstance(c, dict) and 'score' in c]
+
+        if 'web_search' in sources:
+            kb_and_web.append((chat.id, scores))
+        else:
+            kb_only.append((chat.id, scores))
+
+    return kb_and_web, kb_only
+
+
+def _write_section(f, title, results):
+    f.write(f'{title}\\n')
+    f.write('=' * len(title) + '\\n\\n')
+    for chat_id, scores in results:
+        f.write(f'Chat ID: {chat_id}\\n')
+        f.write(f'Scores: {scores}\\n\\n')
+    f.write('\\n')
+
+
+def check_kb_web_search_scores(output_path='kb_web_search_scores.txt'):
+    kb_and_web, kb_only = _split_kb_and_web_search_scores()
+
+    with open(output_path, 'w') as f:
+        _write_section(f, 'KB + WEB SEARCH (KB had a result, web search still triggered)', kb_and_web)
+        _write_section(f, 'KB ONLY (no web search — KB result accepted alone)', kb_only)
+
+    print(f'[check_kb_web_search_scores] KB + web search: {len(kb_and_web)} chats')
+    print(f'[check_kb_web_search_scores] KB only        : {len(kb_only)} chats')
+    print(f'[check_kb_web_search_scores] written to {output_path}')
+
+    return kb_and_web, kb_only
+""")
+
+# ============================================================================
+# Run — edit the path below before pasting into shell_plus
+# ============================================================================
+# check_kb_web_search_scores(
+#     output_path='chatbot/kb_web_search_scores.txt',
+# )

chatbot/scripts/export_chats_by_userid.py

@@ -164,7 +164,7 @@ def export_chats_by_userid(input_csv, output_xlsx):
 # ============================================================================
 # Run — edit the paths below before pasting into shell_plus
 # ============================================================================
-export_chats_by_userid(
-    input_csv='/home/ubuntu/user_sample_data.csv',
-    output_xlsx='/home/ubuntu/sample_output.xlsx',
-)
+# export_chats_by_userid(
+#     input_csv='/home/kunal/user_sample_data.csv',
+#     output_xlsx='/home/kunal/sample_output.xlsx',
+# )