Fix web search fallback and citation parsing for KB search

chatbot/services/response_handlers/base_response_handler.py

@@ -498,6 +498,16 @@ def _handle_gateway_response(
                 },
             ]
 
+            if tool_name == 'search_knowledge_base' and use_web_search and not new_chunks:
+                # Not shown to the user — just nudges the gateway/LLM to actually invoke web_search
+                # instead of answering from internal knowledge, whenever our fallback logic enables it.
+                current_messages = current_messages + [
+                    {
+                        'role': 'assistant',
+                        'content': "I couldn't find this in our knowledge base. Let me search the web for this.",
+                    },
+                ]
+
             # Remove executed tool so the LLM cannot call it again
             current_tools = [
                 t for t in (current_tools or [])
@@ -526,11 +536,18 @@ def _execute_tool(self, tool_name, arguments, company_bot):
                     parts.append(f'Source: {header}\n{text}')
                 chunks_text = self._wrap_retrieved_content('\n\n---\n\n'.join(parts), source='repository')
             else:
-                chunks_text = self._wrap_retrieved_content(
-                    '(no repository result found — use web search if available, otherwise respond '
-                    'from general knowledge if appropriate, per no-hallucination rules)',
-                    source='none',
-                )
+                if getattr(company_bot, 'enable_web_search', False):
+                    no_result_message = (
+                        'No repository result found for this query. Web search is enabled for this bot — '
+                        'call the web_search tool now to answer this query before responding. '
+                        'Do not answer from general/internal knowledge first without doing web search.'
+                    )
+                else:
+                    no_result_message = (
+                        '(no repository result found — respond from general knowledge if appropriate, '
+                        'per no-hallucination rules)'
+                    )
+                chunks_text = self._wrap_retrieved_content(no_result_message, source='none')
             logger.info(f'[tool_loop] search_knowledge_base: {len(retrieved_chunks)} chunks for query: {query}')
             return chunks_text, retrieved_chunks
 
@@ -802,19 +819,44 @@ def _prepare_sources(self, chunks):
 
     def _extract_citation_chunks(self, message):
         """Extract web search citations from a non-stream gateway message and return as chunk dicts."""
-        citations_raw = message.get('citations') or []
         chunks = []
-        for group in citations_raw:
-            if not isinstance(group, list):
-                continue
-            for citation in group:
-                if not isinstance(citation, dict):
+
+        def _collect_from_tool_results(tool_results):
+            for result in (tool_results or []):
+                if not isinstance(result, dict):
                     continue
-                url = citation.get('url', '')
-                title = citation.get('title', '')
-                text = citation.get('cited_text', '')
-                if url or title:
-                    chunks.append({'text': text, 'title': title, 'url': url})
+                for item in result.get('content') or []:
+                    if not isinstance(item, dict):
+                        continue
+                    url = item.get('url', '')
+                    title = item.get('title', '')
+                    if url or title:
+                        chunks.append({'text': item.get('cited_text', ''), 'title': title, 'url': url})
+
+        citations_raw = message.get('citations') or []
+        if citations_raw and isinstance(citations_raw[0], dict) and 'content' in citations_raw[0]:
+            # Anthropic (via litellm): list of web_search_tool_result objects, each with a
+            # nested content[] of {title, url, ...} — not a flat {url, title, cited_text} dict.
+            _collect_from_tool_results(citations_raw)
+        else:
+            for group in citations_raw:
+                if not isinstance(group, list):
+                    continue
+                for citation in group:
+                    if not isinstance(citation, dict):
+                        continue
+                    url = citation.get('url', '')
+                    title = citation.get('title', '')
+                    text = citation.get('cited_text', '')
+                    if url or title:
+                        chunks.append({'text': text, 'title': title, 'url': url})
+
+        if not chunks:
+            # 'citations' can be null even when a web search happened — the raw provider
+            # payload nests results here instead.
+            web_search_results = (message.get('provider_specific_fields') or {}).get('web_search_results') or []
+            _collect_from_tool_results(web_search_results)
+
         return chunks
 
     def _extract_citation_chunks_from_stream(self, citation_events):

chatbot/urls.py

@@ -40,7 +40,7 @@
 
 
 urlpatterns = [
-    path('api/profile/', api_views.post_profile),
+    # path('api/profile/', api_views.post_profile),  # disabled: unvalidated profile creation
     path('api/get-profile/', api_views.get_profile_view, name='get-profile'),
     path('api/accept-tnc/', api_views.accept_tnc_view, name='accept-tnc'),
     path('api/logout/', api_views.logout_profile, name='logout-profile'),
@@ -103,7 +103,7 @@
     path('api/generate-recommendation/', generate_recommendation, name='generate-recommendation'),
     path('api/sync-user-project/', sync_user_project_view, name='sync-user-project'),
     path('api/get-location/', get_location_view, name='get-location'),
-    path('api/get-ip-location/', get_ip_location_view, name='get-ip-location'),
+    # path('api/get-ip-location/', get_ip_location_view, name='get-ip-location'),  # disabled
     path("api/get-presigned-url/", get_presigned_url,  name='get-presigned-url'),
     path("api/image-converter/", convert_image, name='image-converter'),
     path('api/questions/save/', save_ptm_chats, name="save_ptm_chats"),

chatbot/utils/elevate/profile_utils.py

@@ -38,7 +38,7 @@ def fetch_elevate_user(access_token):
 
         json_data = response.json()
 
-        if json_data.get('responseCode', '').lower() != 'ok':
+        if str(json_data.get('responseCode') or '').lower() != 'ok':
             logger.error('[fetch_elevate_user] unexpected responseCode=%s', json_data.get('responseCode'))
             return {}
 
@@ -150,7 +150,7 @@ def logout_elevate_user(access_token, refresh_token):
 
         json_data = response.json()
 
-        if json_data.get('responseCode', '').lower() != 'ok':
+        if str(json_data.get('responseCode') or '').lower() != 'ok':
             logger.error('[logout_elevate_user] unexpected responseCode=%s', json_data.get('responseCode'))
             return {'error': 'elevate_server_error', 'status_code': response.status_code}
 
@@ -172,6 +172,10 @@ def logout_elevate_user(access_token, refresh_token):
 def update_elevate_profile(access_token, name=None, role=None, school_name=None, district=None, state=None,
                             has_accepted_terms_and_conditions=None):
     try:
+        if not elevate_base_url:
+            logger.error('[update_elevate_profile] ELEVATE_BASE_URL is not configured')
+            return {'error': 'elevate_server_error', 'status_code': 502}
+
         url = f"{elevate_base_url}/user/v1/user/update"
         headers = {'X-auth-token': access_token}
         body = {'about': 'please get hardcode the about'}  # hardcoded for now
@@ -191,10 +195,30 @@ def update_elevate_profile(access_token, name=None, role=None, school_name=None,
         logger.info(f'[update_elevate_profile] sending body={body}')
         response = requests.patch(url, headers=headers, json=body, timeout=30)
         logger.info(f'[update_elevate_profile] status={response.status_code} body={response.text}')
+
+        if response.status_code == 401:
+            logger.error('[update_elevate_profile] unauthorized — token invalid or expired body=%s', _safe_body(response))
+            return {'error': 'unauthorized', 'status_code': 401}
+
+        if response.status_code >= 500:
+            logger.error('[update_elevate_profile] Elevate server error status=%s body=%s', response.status_code, _safe_body(response))
+            return {'error': 'elevate_server_error', 'status_code': response.status_code}
+
         response.raise_for_status()
-        return response.json()
+
+        json_data = response.json()
+        if str(json_data.get('responseCode') or '').lower() != 'ok':
+            logger.error('[update_elevate_profile] unexpected responseCode=%s', json_data.get('responseCode'))
+            return {'error': 'elevate_server_error', 'status_code': response.status_code}
+
+        return json_data
+    except requests.exceptions.HTTPError as e:
+        upstream_status = e.response.status_code if e.response is not None else None
+        logger.error('[update_elevate_profile] HTTP error status=%s body=%s', upstream_status, _safe_body(e.response) if e.response is not None else '')
+        return {'error': 'elevate_server_error', 'status_code': upstream_status}
     except requests.exceptions.RequestException as e:
         logger.error(f'[update_elevate_profile] request failed: {e}', exc_info=True)
+        return {'error': 'elevate_server_error'}
     except Exception as e:
         logger.error(f'[update_elevate_profile] unexpected error: {e}', exc_info=True)
-    return {}
+    return {'error': 'elevate_server_error'}

chatbot/views/api_views.py

@@ -224,18 +224,22 @@ def get_profile_view(request):
             profile = Profile.objects.get(email=email, company=company)
 
         access_token = _get_access_token(request)
-        is_tnc_accepted = False
-        if access_token:
-            elevate_user_data = fetch_elevate_user(access_token)
-            if elevate_user_data.get('error'):
-                logger.error(
-                    '[get_profile_view] failed to fetch tnc status from Elevate: %s',
-                    elevate_user_data.get('error')
-                )
-            else:
-                is_tnc_accepted = elevate_user_data.get('has_accepted_tnc', False)
-        else:
-            logger.error('[get_profile_view] no access_token available — defaulting is_tnc_accepted to False')
+        elevate_user_data = fetch_elevate_user(access_token)
+        if elevate_user_data.get('error') == 'unauthorized':
+            logger.error('[get_profile_view] Elevate auth failure')
+            return Response({
+                'status': 'error',
+                'message': 'Unauthorized.'
+            }, status=elevate_user_data.get('status_code'))
+
+        if elevate_user_data.get('error') == 'elevate_server_error':
+            logger.error('[get_profile_view] Elevate server error')
+            return Response({
+                'status': 'error',
+                'message': 'Elevate service unavailable.'
+            }, status=elevate_user_data.get('status_code') or 502)
+
+        is_tnc_accepted = elevate_user_data.get('has_accepted_tnc', False)
 
         is_onboarding_completed = bool(
             profile.other_params and profile.other_params.get('is_onboarding_completed', False)
@@ -295,18 +299,21 @@ def accept_tnc_view(request):
             profile = Profile.objects.get(email=email, company=company)
 
         access_token = _get_access_token(request)
-        if not access_token:
+        result = update_elevate_profile(access_token, has_accepted_terms_and_conditions=True)
+
+        if result.get('error') == 'unauthorized':
+            logger.error('[accept_tnc_view] Elevate auth failure')
             return Response({
                 'status': 'error',
-                'message': 'access_token is required to accept terms and conditions'
-            }, status=400)
+                'message': 'Unauthorized.'
+            }, status=result.get('status_code'))
 
-        result = update_elevate_profile(access_token, has_accepted_terms_and_conditions=True)
-        if not result:
+        if result.get('error') == 'elevate_server_error':
+            logger.error('[accept_tnc_view] Elevate server error')
             return Response({
                 'status': 'error',
-                'message': 'Failed to update terms and conditions acceptance'
-            }, status=502)
+                'message': 'Elevate service unavailable.'
+            }, status=result.get('status_code') or 502)
 
         return Response({
             'status': 'ok',