diff --git a/chatbot/admin/company_admin.py b/chatbot/admin/company_admin.py index 40d4c88..5664cbc 100644 --- a/chatbot/admin/company_admin.py +++ b/chatbot/admin/company_admin.py @@ -3,11 +3,11 @@ from pydantic import ValidationError from simple_history.admin import SimpleHistoryAdmin from .generic_upload_admin import BatchUploadMixin -from chatbot.filter.admin_filter import (CompanyChatCompanyFilter, ChatSessionFilter, ProfileCityFilter, - ProfileStateFilter, ProfileCompanyChatFilter, ProfileEmailFilter) +from chatbot.filter.admin_filter import (CompanyChatCompanyFilter, ChatSessionFilter, + ProfileCompanyChatFilter, ProfileEmailFilter) from chatbot.filter.custom_date_from_filter import CustomAdvanceDateFilter -from chatbot.models import Company, Profile, ProfileType, CompanyBot, CompanyChat, ChatSession, \ - CompanyBotTypeChoices, Voice, VoiceProvider, VoiceType, ImageConfiguration, Flow +from chatbot.models import Company, Profile, ProfileType, CompanyBot, CompanyChat, CompanyChatFeedback, \ + ChatSession, CompanyBotTypeChoices, Voice, VoiceProvider, VoiceType, ImageConfiguration, Flow from chatbot.models.company_models import CompanyStateMachine from chatbot.resources.resource import CompanyChatResource from chatbot.resources.company_resource import ChatSessionResource @@ -292,9 +292,30 @@ def changelist_view(self, request, extra_context=None): duplicate_bot.short_description = "Duplicate selected bot" +class CompanyChatFeedbackInline(admin.TabularInline): + """Read-only: feedback rows are created via the feedback API only and are never edited, + so admins can view the full history here but can't add/change/delete from this screen.""" + model = CompanyChatFeedback + fk_name = 'company_chat' + extra = 0 + fields = ('thumbs_up', 'thumbs_down', 'comment', 'created_at') + readonly_fields = fields + ordering = ('-created_at',) + + def has_add_permission(self, request, obj=None): + return False + + def has_change_permission(self, request, obj=None): + return False + + def has_delete_permission(self, request, obj=None): + return False + + @admin.register(CompanyChat) class CompanyChatAdmin(ExportAllFieldsMixin, admin.ModelAdmin): list_display = ('session', 'sender', 'receiver', 'message', 'translated_message', 'created_at', 'stage', 'status') + inlines = [CompanyChatFeedbackInline] list_filter = ( CustomAdvanceDateFilter, ProfileCompanyChatFilter, @@ -313,44 +334,16 @@ class CompanyChatAdmin(ExportAllFieldsMixin, admin.ModelAdmin): resource_class = CompanyChatResource def get_queryset(self, request): - qs = super().get_queryset(request) + qs = super().get_queryset(request).prefetch_related('sender__company', 'receiver__company') user_email = request.user.email profile = Profile.objects.filter(email=user_email).select_related('company').first() if request.user.is_superuser: - return qs.prefetch_related('sender__company', 'receiver__company') + return qs elif profile and profile.profile_type == ProfileType.MODERATOR: - return qs.filter( - Q(sender__company=profile.company) | Q(receiver__company=profile.company) - ).prefetch_related('sender__company', 'receiver__company') + return qs.filter(Q(sender__company=profile.company) | Q(receiver__company=profile.company)) else: return qs.none() - def get_search_results(self, request, queryset, search_term): - queryset, use_distinct = super().get_search_results(request, queryset, search_term) - - user_email = request.user.email - profile = Profile.objects.filter(email=user_email).select_related('company').first() - if not request.user.is_superuser and profile and profile.profile_type == ProfileType.MODERATOR: - if profile.company: - queryset = queryset.filter( - Q(sender__company=profile.company) | Q(receiver__company=profile.company) - ).prefetch_related('sender__company', 'receiver__company') - return queryset, use_distinct - - def get_list_filter(self, request): - user = request.user - user_email = request.user.email - profile = Profile.objects.filter(email=user_email).select_related('company').first() - if not user.is_superuser and profile and profile.profile_type == ProfileType.MODERATOR: - company = profile.company - if company.slug == 'fmch': - return (CustomAdvanceDateFilter, ProfileCompanyChatFilter, - ProfileEmailFilter, 'session', ProfileCityFilter, ProfileStateFilter, 'message_type') - if company.slug == 'tfistaging': - return (CustomAdvanceDateFilter, ProfileCompanyChatFilter, - ProfileEmailFilter, 'session', CompanyChatCompanyFilter, 'stage') - return super().get_list_filter(request) - @admin.register(ChatSession) class ChatSessionAdmin(ExportAllFieldsMixin, admin.ModelAdmin): diff --git a/chatbot/migrations/0089_alter_chatsession_language_alter_story_language_and_more.py b/chatbot/migrations/0089_alter_chatsession_language_alter_story_language_and_more.py new file mode 100644 index 0000000..b6a62ab --- /dev/null +++ b/chatbot/migrations/0089_alter_chatsession_language_alter_story_language_and_more.py @@ -0,0 +1,44 @@ +# Generated by Django 5.2 on 2026-07-30 09:49 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('chatbot', '0088_remove_botvernacular_bot_vernacu_company_483975_idx_and_more'), + ] + + operations = [ + migrations.AlterField( + model_name='chatsession', + name='language', + field=models.CharField(choices=[('en', 'English'), ('hi', 'Hindi'), ('kn', 'Kannada'), ('te', 'Telugu'), ('or', 'Odia'), ('ta', 'Tamil')], default='en', max_length=1000), + ), + migrations.AlterField( + model_name='story', + name='language', + field=models.CharField(choices=[('en', 'English'), ('hi', 'Hindi'), ('kn', 'Kannada'), ('te', 'Telugu'), ('or', 'Odia'), ('ta', 'Tamil')], default='en', max_length=1000), + ), + migrations.AlterField( + model_name='storytranslation', + name='language', + field=models.CharField(choices=[('en', 'English'), ('hi', 'Hindi'), ('kn', 'Kannada'), ('te', 'Telugu'), ('or', 'Odia'), ('ta', 'Tamil')], max_length=10), + ), + migrations.CreateModel( + name='CompanyChatFeedback', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('thumbs_up', models.BooleanField(default=False, help_text='True if the user gave a positive rating in this submission.')), + ('thumbs_down', models.BooleanField(default=False, help_text='True if the user gave a negative rating in this submission. Cannot be True at the same time as thumbs_up (enforced in the serializer).')), + ('comment', models.TextField(blank=True, help_text='Optional free-text feedback typed by the user.', null=True)), + ('created_at', models.DateTimeField(auto_now_add=True, help_text='When this feedback was submitted. Immutable — also used to determine the current state (latest row wins) and submission order.')), + ('company_chat', models.ForeignKey(help_text='The bot response (CompanyChat row) this feedback is about.', on_delete=django.db.models.deletion.CASCADE, related_name='feedbacks', to='chatbot.companychat')), + ], + options={ + 'ordering': ['-created_at'], + 'indexes': [models.Index(fields=['company_chat', '-created_at'], name='chatbot_com_company_f623f2_idx')], + }, + ), + ] diff --git a/chatbot/models/company_models.py b/chatbot/models/company_models.py index f989826..ef58c69 100644 --- a/chatbot/models/company_models.py +++ b/chatbot/models/company_models.py @@ -233,6 +233,43 @@ def save(self, *args, **kwargs): super(CompanyChat, self).save(*args, **kwargs) +class CompanyChatFeedback(models.Model): + """ + A single feedback submission (thumbs up/down + optional comment) for a bot response. + Rows are append-only — never updated — so the full history is preserved and the + most recent row (by created_at) represents the current state. + """ + company_chat = models.ForeignKey( + CompanyChat, related_name='feedbacks', on_delete=models.CASCADE, + help_text='The bot response (CompanyChat row) this feedback is about.' + ) + thumbs_up = models.BooleanField( + default=False, help_text='True if the user gave a positive rating in this submission.' + ) + thumbs_down = models.BooleanField( + default=False, + help_text='True if the user gave a negative rating in this submission. ' + 'Cannot be True at the same time as thumbs_up (enforced in the serializer).' + ) + comment = models.TextField( + null=True, blank=True, help_text='Optional free-text feedback typed by the user.' + ) + created_at = models.DateTimeField( + auto_now_add=True, + help_text='When this feedback was submitted. Immutable — also used to determine ' + 'the current state (latest row wins) and submission order.' + ) + + class Meta: + ordering = ['-created_at'] + indexes = [ + models.Index(fields=['company_chat', '-created_at']), + ] + + def __str__(self): + return f'Feedback #{self.id} for CompanyChat #{self.company_chat_id}' + + class Voice(models.Model): """ Defines a text-to-speech voice configuration for a company bot. diff --git a/chatbot/parsers.py b/chatbot/parsers.py new file mode 100644 index 0000000..1446c5a --- /dev/null +++ b/chatbot/parsers.py @@ -0,0 +1,36 @@ +import codecs + +from django.conf import settings +from rest_framework.exceptions import ParseError +from rest_framework.parsers import JSONParser +from rest_framework.utils import json + + +def _reject_duplicate_keys(pairs): + seen = set() + for key, _ in pairs: + if key in seen: + raise ParseError(f'Duplicate key "{key}" in request body.') + seen.add(key) + return dict(pairs) + + +class StrictJSONParser(JSONParser): + """Like DRF's JSONParser, but rejects a JSON object with duplicate keys instead of + silently keeping only the last occurrence.""" + + def parse(self, stream, media_type=None, parser_context=None): + parser_context = parser_context or {} + encoding = parser_context.get('encoding', settings.DEFAULT_CHARSET) + try: + decoded_stream = codecs.getreader(encoding)(stream) + parse_constant = json.strict_constant if self.strict else None + return json.load( + decoded_stream, + object_pairs_hook=_reject_duplicate_keys, + parse_constant=parse_constant, + ) + except ParseError: + raise + except ValueError as exc: + raise ParseError('JSON parse error - %s' % str(exc)) \ No newline at end of file diff --git a/chatbot/scripts/export_chats_by_userid.py b/chatbot/scripts/export_chats_by_userid.py new file mode 100644 index 0000000..8cda430 --- /dev/null +++ b/chatbot/scripts/export_chats_by_userid.py @@ -0,0 +1,170 @@ +# Standalone shell_plus script. +# +# Usage: open `python manage.py shell_plus`, paste this ENTIRE file, then run: +# +# export_chats_by_userid( +# input_csv='/path/to/phnNumberData.csv', +# output_xlsx='/path/to/output.xlsx', +# ) +# +# It's wrapped in exec("""...""") on purpose — some shell_plus setups (plain +# Python REPL inside tmux/screen, no bracketed paste) mis-parse a pasted +# multi-line script because blank lines inside a function body look like +# "end of block" to the incremental parser. Wrapping the whole body in a +# single string literal sidesteps that: the REPL just buffers lines until +# the closing triple-quote, then exec() runs it as one unit. +# +# Input CSV must have columns: Name, Phone Number, User Id +# (header names are matched case-insensitively, spaces/underscores ignored). +# +# For every input row, this fetches chatbot_profile by userid, then every +# CompanyChat where that profile is sender or receiver, and writes one xlsx +# row per chat message (all CompanyChat fields except other_params), with +# the original Name/Phone Number/User Id repeated on every row so it can be +# filtered/pivoted in Google Sheets. Rows whose userid doesn't map to a +# profile, or has no chats, still get exactly one output row with the chat +# columns left blank — a mapping_status column says why. + +exec(""" +import csv +import pandas as pd +from django.db.models import Q +from chatbot.models import Profile +from chatbot.models.company_models import CompanyChat + +COMPANY_CHAT_FIELDS = [ + 'id', 'session', 'message', 'translated_message', 'chunks', + 'sender_id', 'receiver_id', 'created_at', 'updated_at', 'status', + 'feedback', 'source', 'source_msg_id', 'whatsapp_message_id', + 'message_type', 'stage', 'file_url', 'audio_file', +] # deliberately excludes other_params + +EMPTY_CHAT_ROW = {f'chat_{f}': '' for f in COMPANY_CHAT_FIELDS} +EMPTY_CHAT_ROW['chat_file_url_https'] = '' + + +def _s3_to_https(value): + if value and value.startswith('s3://'): + return 'https://' + value[len('s3://'):] + return '' + + +def _read_input_rows(input_csv): + with open(input_csv, newline='', encoding='utf-8-sig') as f: + reader = csv.DictReader(f) + header_map = {} + for raw_name in reader.fieldnames or []: + key = raw_name.strip().lower().replace(' ', '').replace('_', '') + header_map[key] = raw_name + + name_col = header_map.get('name') + phone_col = header_map.get('phonenumber') or header_map.get('phone') + userid_col = header_map.get('userid') + + if not (name_col and phone_col and userid_col): + raise ValueError( + f"Could not find Name/Phone Number/User Id columns. " + f"Found headers: {reader.fieldnames}" + ) + + rows = [] + for row in reader: + rows.append({ + 'csv_name': (row.get(name_col) or '').strip(), + 'csv_phone_number': (row.get(phone_col) or '').strip(), + 'csv_user_id': (row.get(userid_col) or '').strip(), + }) + return rows + + +def _chat_row(chat): + out = {} + for field in COMPANY_CHAT_FIELDS: + value = getattr(chat, field) + out[f'chat_{field}'] = str(value) if value not in (None, '') else '' + out['chat_file_url_https'] = _s3_to_https(out['chat_file_url']) + return out + + +def export_chats_by_userid(input_csv, output_xlsx): + input_rows = _read_input_rows(input_csv) + print(f"[export_chats_by_userid] read {len(input_rows)} input rows from {input_csv}") + + output_rows = [] + stats = {'no_userid_in_csv': 0, 'profile_not_found': 0, 'profile_found_no_chats': 0, 'profile_found': 0} + + for i, row in enumerate(input_rows, start=1): + userid = row['csv_user_id'] + base = { + 'name': row['csv_name'], + 'phone_number': row['csv_phone_number'], + 'user_id': userid, + } + + if not userid: + stats['no_userid_in_csv'] += 1 + output_rows.append({**base, 'mapping_status': 'no_userid_in_csv', + 'profile_id': '', 'profile_first_name': '', 'profile_last_name': '', + 'profile_phone': '', 'profile_email': '', 'profile_status': '', + **EMPTY_CHAT_ROW}) + continue + + profile = Profile.objects.filter(userid=userid).first() + + if not profile: + stats['profile_not_found'] += 1 + output_rows.append({**base, 'mapping_status': 'profile_not_found', + 'profile_id': '', 'profile_first_name': '', 'profile_last_name': '', + 'profile_phone': '', 'profile_email': '', 'profile_status': '', + **EMPTY_CHAT_ROW}) + continue + + profile_cols = { + 'profile_id': profile.id, + 'profile_first_name': profile.first_name or '', + 'profile_last_name': profile.last_name or '', + 'profile_phone': profile.phone or '', + 'profile_email': profile.email or '', + 'profile_status': profile.status or '', + } + + chats = CompanyChat.objects.filter( + Q(sender_id=profile.id) | Q(receiver_id=profile.id) + ).order_by('session', 'created_at') + + if not chats.exists(): + stats['profile_found_no_chats'] += 1 + output_rows.append({**base, 'mapping_status': 'profile_found_no_chats', + **profile_cols, **EMPTY_CHAT_ROW}) + continue + + stats['profile_found'] += 1 + for chat in chats: + output_rows.append({**base, 'mapping_status': 'profile_found', + **profile_cols, **_chat_row(chat)}) + + if i % 50 == 0: + print(f"[export_chats_by_userid] processed {i}/{len(input_rows)} input rows") + + df = pd.DataFrame(output_rows) + df.to_excel(output_xlsx, index=False, engine='openpyxl') + + print("[export_chats_by_userid] done") + print(f" input rows : {len(input_rows)}") + print(f" no_userid_in_csv : {stats['no_userid_in_csv']}") + print(f" profile_not_found : {stats['profile_not_found']}") + print(f" profile_found_no_chats: {stats['profile_found_no_chats']}") + print(f" profile_found : {stats['profile_found']}") + print(f" output rows (xlsx) : {len(output_rows)}") + print(f" written to : {output_xlsx}") + + return df +""") + +# ============================================================================ +# Run — edit the paths below before pasting into shell_plus +# ============================================================================ +export_chats_by_userid( + input_csv='/home/ubuntu/user_sample_data.csv', + output_xlsx='/home/ubuntu/sample_output.xlsx', +) diff --git a/chatbot/serializer/profile_serializer.py b/chatbot/serializer/profile_serializer.py index 82aca04..a2ea229 100644 --- a/chatbot/serializer/profile_serializer.py +++ b/chatbot/serializer/profile_serializer.py @@ -1,7 +1,8 @@ +from django.db import transaction from rest_framework import serializers from chatbot.models.media_models import ProfileMedia from chatbot.models.profile_models import Profile -from chatbot.models.company_models import CompanyChat +from chatbot.models.company_models import CompanyChat, CompanyChatFeedback from chatbot.models.geo_models import ProfileAddress from chatbot.serializer.company_serializer import CompanySerializer @@ -82,9 +83,93 @@ def update(self, instance, validated_data): return instance class CompanyChatSerializer(serializers.ModelSerializer): + """Note: thumbs_up/thumbs_down reflect only the latest CompanyChatFeedback row for this + message (comment text and older feedback history are intentionally not exposed here).""" sender = ProfileSerializer(read_only=True) receiver = ProfileSerializer(read_only=True) + thumbs_up = serializers.SerializerMethodField() + thumbs_down = serializers.SerializerMethodField() class Meta: model = CompanyChat fields = '__all__' + + def _latest_feedback(self, obj): + # The view's queryset annotates latest_thumbs_up/latest_thumbs_down via a Subquery + # so the full feedback history is never loaded. Fall back to a direct query for + # instances not fetched through that queryset (e.g. a freshly created row on POST). + if hasattr(obj, 'latest_thumbs_up'): + return obj.latest_thumbs_up, obj.latest_thumbs_down + latest = obj.feedbacks.order_by('-created_at').first() + return (latest.thumbs_up, latest.thumbs_down) if latest else (None, None) + + def get_thumbs_up(self, obj): + thumbs_up, _ = self._latest_feedback(obj) + return bool(thumbs_up) + + def get_thumbs_down(self, obj): + _, thumbs_down = self._latest_feedback(obj) + return bool(thumbs_down) + + +class CompanyChatFeedbackSerializer(serializers.ModelSerializer): + """Creates a new feedback row. Never updates an existing one — every submission + (including switching thumbs up <-> down) is stored as its own history entry.""" + + class Meta: + model = CompanyChatFeedback + fields = ['id', 'company_chat', 'thumbs_up', 'thumbs_down', 'comment', 'created_at'] + read_only_fields = ['id', 'created_at'] + + def to_internal_value(self, data): + # ModelSerializer silently drops unknown keys by default; reject them instead so + # the request body is required to strictly match the schema. + if hasattr(data, 'keys'): + unknown_fields = set(data.keys()) - set(self.fields.keys()) + if unknown_fields: + raise serializers.ValidationError( + {field: 'This field is not allowed.' for field in unknown_fields} + ) + return super().to_internal_value(data) + + def validate(self, attrs): + thumbs_up = attrs.get('thumbs_up', False) + thumbs_down = attrs.get('thumbs_down', False) + has_comment = bool((attrs.get('comment') or '').strip()) + + # Explicit thumbs decisions are validated here; the comment-only carry-forward + # case is resolved atomically in create() to avoid a read-then-insert race with + # a concurrent feedback submission for the same company_chat. + if thumbs_up and thumbs_down: + raise serializers.ValidationError('thumbs_up and thumbs_down cannot both be true.') + + # thumbs_up/thumbs_down default to False, so "both present but false, no comment" + # is a no-op submission, not a carry-forward — reject it rather than silently + # writing an empty feedback row. + if not thumbs_up and not thumbs_down and not has_comment: + raise serializers.ValidationError( + 'At least one of thumbs_up, thumbs_down must be true, or a comment must be provided.' + ) + return attrs + + def create(self, validated_data): + has_thumbs_key = 'thumbs_up' in self.initial_data or 'thumbs_down' in self.initial_data + company_chat = validated_data['company_chat'] + + with transaction.atomic(): + # Lock the parent row so ALL feedback submissions for this company_chat — + # explicit thumbs decisions and comment-only carry-forwards alike — serialize + # here. Without locking on the explicit-thumbs path too, a concurrent + # comment-only request could still read a stale "latest" and, since it's + # inserted later, overwrite a newer explicit decision. + CompanyChat.objects.select_for_update().get(pk=company_chat.pk) + + if not has_thumbs_key: + latest = CompanyChatFeedback.objects.filter( + company_chat=company_chat + ).order_by('-created_at').first() + if latest: + validated_data['thumbs_up'] = latest.thumbs_up + validated_data['thumbs_down'] = latest.thumbs_down + + return super().create(validated_data) diff --git a/chatbot/services/response_handlers/base_response_handler.py b/chatbot/services/response_handlers/base_response_handler.py index 91af860..ce6a2eb 100644 --- a/chatbot/services/response_handlers/base_response_handler.py +++ b/chatbot/services/response_handlers/base_response_handler.py @@ -812,11 +812,39 @@ def _prepare_sources(self, chunks): continue seen.add(key) if url: - sources.append({'title': title, 'url': url}) + source_entry = {'title': title, 'url': url} else: - sources.append({'title': f'Referred: {title}'}) + source_entry = {'title': f'Referred: {title}'} + + chunk_source = chunk.get('source', '') + if chunk_source == 'web_search': + source_entry['source'] = 'web_search' + domain = self._extract_domain(url) + if domain: + source_entry['domain'] = domain + elif chunk_source == 'kb_search': + source_entry['source'] = 'kb_search' + company = chunk.get('company', '') + if company: + source_entry['company'] = company + logo = chunk.get('logo', '') + if logo: + source_entry['logo'] = logo + + sources.append(source_entry) return sources + @staticmethod + def _extract_domain(url): + """Return just the site name from a URL's domain — no 'www.' prefix, no TLD (e.g. 'impriindia').""" + if not url: + return '' + from urllib.parse import urlparse + netloc = urlparse(url).netloc + if netloc.startswith('www.'): + netloc = netloc[len('www.'):] + return netloc.split('.')[0] + def _extract_citation_chunks(self, message): """Extract web search citations from a non-stream gateway message and return as chunk dicts.""" chunks = [] @@ -831,7 +859,8 @@ def _collect_from_tool_results(tool_results): url = item.get('url', '') title = item.get('title', '') if url or title: - chunks.append({'text': item.get('cited_text', ''), 'title': title, 'url': url}) + chunks.append({'text': item.get('cited_text', ''), 'title': title, 'url': url, + 'source': 'web_search'}) citations_raw = message.get('citations') or [] if citations_raw and isinstance(citations_raw[0], dict) and 'content' in citations_raw[0]: @@ -849,7 +878,7 @@ def _collect_from_tool_results(tool_results): title = citation.get('title', '') text = citation.get('cited_text', '') if url or title: - chunks.append({'text': text, 'title': title, 'url': url}) + chunks.append({'text': text, 'title': title, 'url': url, 'source': 'web_search'}) if not chunks: # 'citations' can be null even when a web search happened — the raw provider @@ -875,7 +904,7 @@ def _extract_citation_chunks_from_stream(self, citation_events): title = item.get('title', '') text = item.get('cited_text', '') or item.get('text', '') if url or title: - chunks.append({'text': text, 'title': title, 'url': url}) + chunks.append({'text': text, 'title': title, 'url': url, 'source': 'web_search'}) return chunks def _parse_if_string(self, value, fallback): diff --git a/chatbot/services/vector/vector_service.py b/chatbot/services/vector/vector_service.py index ed24093..e154e5d 100644 --- a/chatbot/services/vector/vector_service.py +++ b/chatbot/services/vector/vector_service.py @@ -16,11 +16,15 @@ def _fetch_chunks(query, top_k, filter_score, priority): score = item.get('score', 0) text = item.get('text', '') if text and len(text) > 20 and score >= filter_score: + metadata = item.get('metadata', {}) or {} chunks.append({ 'text': text, 'title': item.get('title', ''), - 'url': item.get('metadata', {}).get('url', ''), + 'url': metadata.get('url', ''), 'score': score, + 'source': 'kb_search', + 'company': metadata.get('company_name') or metadata.get('company', ''), + 'logo': metadata.get('logo', ''), }) return chunks diff --git a/chatbot/urls.py b/chatbot/urls.py index 180ce2f..85e9dd5 100644 --- a/chatbot/urls.py +++ b/chatbot/urls.py @@ -21,6 +21,7 @@ from chatbot.views.bhashini_views import text_speech_view, speech_text, text_translation_view, text_transliterate_view from chatbot.views.chat_view import save_chats_view, create_chatsession, save_ptm_chats from chatbot.views.drf_views import CompanyChatListCreateView, CompanyChatRetrieveUpdateDestroyView, \ + CompanyChatFeedbackCreateView, \ CompanyBotListCreateView, CompanyBotRetrieveUpdateDestroyView, ProfileListCreateView, \ ProfileRetrieveUpdateDestroyView, ChatSessionListCreateView, ChatSessionRetrieveUpdateDestroyView, \ ChatSessionRetrieveUpdateDestroyViewSession, BotVernacularListCreateView, BotVernacularRetrieveUpdateDestroyView, \ @@ -43,6 +44,7 @@ # path('api/profile/', api_views.post_profile), # disabled: unvalidated profile creation path('api/get-profile/', api_views.get_profile_view, name='get-profile'), path('api/accept-tnc/', api_views.accept_tnc_view, name='accept-tnc'), + path('api/update-profile/', api_views.update_profile_view, name='update-profile'), path('api/logout/', api_views.logout_profile, name='logout-profile'), path('api/user_profile/', ProfileListCreateView.as_view(), name='profile-list-create'), @@ -60,6 +62,7 @@ path('api/companychat/', CompanyChatListCreateView.as_view(), name='companychat-list-create'), path('api/companychat//', CompanyChatRetrieveUpdateDestroyView.as_view(), name='companychat-retrieve-update-destroy'), + path('api/companychat-feedback/', CompanyChatFeedbackCreateView.as_view(), name='companychat-feedback-create'), path('api/companybot/', CompanyBotListCreateView.as_view(), name='companybot-list-create'), path('api/companybot//', CompanyBotRetrieveUpdateDestroyView.as_view(), diff --git a/chatbot/utils/elevate/profile_utils.py b/chatbot/utils/elevate/profile_utils.py index fc28090..15d4ed7 100644 --- a/chatbot/utils/elevate/profile_utils.py +++ b/chatbot/utils/elevate/profile_utils.py @@ -49,6 +49,8 @@ def fetch_elevate_user(access_token): logger.error('[fetch_elevate_user] no userid in Elevate response') return {} + name = user_data.get('name') + language = user_data.get('preferred_language') if isinstance(language, dict): language = language.get('value', 'en') @@ -71,6 +73,7 @@ def fetch_elevate_user(access_token): return { 'userid': userid, + 'name': name, 'language': language, 'designation': designation_value, 'school_name': school_name, @@ -111,6 +114,7 @@ def upsert_elevate_profile(user_data): "has_accepted_tnc": user_data.get('has_accepted_tnc', False), "route": language, "reroute_url": os.getenv('SSO_REROUTE_URL'), + "name": user_data.get('name'), "ums_profile": { "designation": user_data['designation'], "org_associated": user_data['school_name'], @@ -170,7 +174,7 @@ def logout_elevate_user(access_token, refresh_token): def update_elevate_profile(access_token, name=None, role=None, school_name=None, district=None, state=None, - has_accepted_terms_and_conditions=None): + about=None, has_accepted_terms_and_conditions=None): try: if not elevate_base_url: logger.error('[update_elevate_profile] ELEVATE_BASE_URL is not configured') @@ -178,7 +182,7 @@ def update_elevate_profile(access_token, name=None, role=None, school_name=None, url = f"{elevate_base_url}/user/v1/user/update" headers = {'X-auth-token': access_token} - body = {'about': 'please get hardcode the about'} # hardcoded for now + body = {} if name: body['name'] = name if role: @@ -189,12 +193,14 @@ def update_elevate_profile(access_token, name=None, role=None, school_name=None, body['userDistrict'] = district if state: body['profileState'] = state + if about: + body['about'] = about if has_accepted_terms_and_conditions is not None: body['has_accepted_terms_and_conditions'] = has_accepted_terms_and_conditions - logger.info(f'[update_elevate_profile] sending body={body}') + logger.info('[update_elevate_profile] sending fields=%s', list(body.keys())) response = requests.patch(url, headers=headers, json=body, timeout=30) - logger.info(f'[update_elevate_profile] status={response.status_code} body={response.text}') + logger.info('[update_elevate_profile] status=%s', response.status_code) if response.status_code == 401: logger.error('[update_elevate_profile] unauthorized — token invalid or expired body=%s', _safe_body(response)) diff --git a/chatbot/views/api_views.py b/chatbot/views/api_views.py index b2e6c7f..9b2e4c3 100644 --- a/chatbot/views/api_views.py +++ b/chatbot/views/api_views.py @@ -340,6 +340,52 @@ def accept_tnc_view(request): }, status=500) +@api_view(['PATCH']) +def update_profile_view(request): + try: + update_fields = {} + for field in ('name', 'role', 'school_name', 'district', 'state'): + value = request.data.get(field) + if isinstance(value, str) and value.strip(): + update_fields[field] = value.strip() + + if not update_fields: + return Response({ + 'status': 'error', + 'message': 'at least one of name, role, school_name, district, state is required' + }, status=400) + + access_token = _get_access_token(request) + result = update_elevate_profile(access_token, **update_fields) + + if result.get('error') == 'unauthorized': + logger.error('[update_profile_view] Elevate auth failure') + return Response({ + 'status': 'error', + 'message': 'Unauthorized.' + }, status=result.get('status_code')) + + if result.get('error') == 'elevate_server_error': + logger.error('[update_profile_view] Elevate server error') + return Response({ + 'status': 'error', + 'message': 'Elevate service unavailable.' + }, status=result.get('status_code') or 502) + + logger.info('[update_profile_view] updated fields=%s', list(update_fields)) + return Response({ + 'status': 'ok', + 'updated_fields': list(update_fields), + }, status=200) + + except Exception: + logger.error('[update_profile_view] unexpected error', exc_info=True) + return Response({ + 'status': 'error', + 'message': 'Internal server error.' + }, status=500) + + @api_view(['POST']) def logout_profile(request): access_token = request.COOKIES.get(ACCESS_TOKEN_COOKIE_KEY) if ACCESS_TOKEN_COOKIE_KEY else None diff --git a/chatbot/views/drf_views.py b/chatbot/views/drf_views.py index 83817fa..89e0245 100644 --- a/chatbot/views/drf_views.py +++ b/chatbot/views/drf_views.py @@ -1,32 +1,55 @@ import django_filters +from django.db.models import OuterRef, Subquery from rest_framework import generics from rest_framework.filters import OrderingFilter from rest_framework.response import Response from rest_framework import status from chatbot.filter.drf_filter import ChatSessionProfileFilter +from chatbot.parsers import StrictJSONParser from chatbot.models import ChatSession, BotVernacular, SessionFlowName, ChatType -from chatbot.models.company_models import CompanyChat, CompanyBot, Flow +from chatbot.models.company_models import CompanyChat, CompanyChatFeedback, CompanyBot, Flow from chatbot.models.profile_models import Profile from chatbot.serializer.base_serializer import ChatSessionSerializer from chatbot.serializer.company_serializer import ( CompanyBotSerializer, BotVernacularSerializer, ImageConfigurationSerializer, FlowLanguagesSerializer, FlowConnectionInfoSerializer ) -from chatbot.serializer.profile_serializer import ProfileSerializer, CompanyChatSerializer +from chatbot.serializer.profile_serializer import ( + ProfileSerializer, CompanyChatSerializer, CompanyChatFeedbackSerializer +) + + +def _with_latest_feedback(queryset): + """Annotate each row with its latest feedback's thumbs_up/thumbs_down instead of + prefetching the full (potentially unbounded, append-only) feedback history.""" + latest_feedback = CompanyChatFeedback.objects.filter( + company_chat=OuterRef('pk') + ).order_by('-created_at') + return queryset.annotate( + latest_thumbs_up=Subquery(latest_feedback.values('thumbs_up')[:1]), + latest_thumbs_down=Subquery(latest_feedback.values('thumbs_down')[:1]), + ) class CompanyChatListCreateView(generics.ListCreateAPIView): - queryset = CompanyChat.objects.all().order_by('created_at') + queryset = _with_latest_feedback(CompanyChat.objects.all().order_by('created_at')) serializer_class = CompanyChatSerializer filter_backends = [django_filters.rest_framework.DjangoFilterBackend] filterset_fields = ['message', 'sender', 'receiver', 'session', 'status'] class CompanyChatRetrieveUpdateDestroyView(generics.RetrieveUpdateAPIView): - queryset = CompanyChat.objects.all() + queryset = _with_latest_feedback(CompanyChat.objects.all()) serializer_class = CompanyChatSerializer +class CompanyChatFeedbackCreateView(generics.CreateAPIView): + """Create-only: FE always POSTs a new feedback row, never PATCH/PUT an existing one.""" + queryset = CompanyChatFeedback.objects.all() + serializer_class = CompanyChatFeedbackSerializer + parser_classes = [StrictJSONParser] + + class CompanyBotListCreateView(generics.ListCreateAPIView): queryset = CompanyBot.objects.all() serializer_class = CompanyBotSerializer diff --git a/docs/setup/gotenberg_server_setup.md b/docs/setup/gotenberg_server_setup.md index d8276cd..e811d81 100644 --- a/docs/setup/gotenberg_server_setup.md +++ b/docs/setup/gotenberg_server_setup.md @@ -27,6 +27,38 @@ through to the Noto font named later in the stack. --- +## 0. Find the running Gotenberg container name + +The container name varies per server — it may be a fixed name from `docker-compose.yml` +(e.g. `saathi_gotenberg`), or an auto-generated Docker name (e.g. `jolly_brattain`) if +Gotenberg was started with a plain `docker run` without `--name`. `-a` includes +stopped containers, so you can also spot stale leftovers from earlier testing — +ignore anything `Created` (never started) or `Exited` long ago; only look at rows +marked `Up`: + +```bash +sudo docker ps -a --format "{{.Names}}\t{{.Image}}\t{{.Status}}\t{{.Ports}}" | grep -i gotenberg +``` + +Auto-capture the running one into a variable — every command below uses `$GC`, so +there's nothing to manually type or mis-paste. Filter by **image**, not container +name: an auto-generated name (e.g. `bold_taussig`) never contains "gotenberg", but +the image always does: + +```bash +export GC=$(sudo docker ps --filter status=running --format "{{.Names}}\t{{.Image}}" | grep -i gotenberg | cut -f1 | head -1) +echo "Gotenberg container: $GC" +``` + +(If that `grep` matches more than one running container, inspect the list above and +set `GC` to the correct name yourself.) + +Confirm it's running the font-baked image: + +```bash +sudo docker inspect "$GC" --format '{{.Config.Image}}' # expect gotenberg-noto:8 +``` + ## 1. Install the fonts in the Gotenberg container `fonts-noto-core` covers Tamil, Devanagari (hi), Kannada (kn), Oriya (or), Telugu, etc. @@ -40,7 +72,8 @@ mkdir -p ~/gotenberg-custom cat > ~/gotenberg-custom/Dockerfile <<'EOF' FROM gotenberg/gotenberg:8 USER root -RUN apt-get update \ +RUN rm -f /etc/apt/sources.list.d/*chrome* \ + && apt-get update \ && apt-get install -y --no-install-recommends fonts-noto-core \ && fc-cache -f \ && rm -rf /var/lib/apt/lists/* @@ -50,7 +83,37 @@ EOF sudo docker build -t gotenberg-noto:8 ~/gotenberg-custom ``` -Point the deployment at it. In `docker-compose.yml`, the gotenberg service: +> **Note:** the `rm -f /etc/apt/sources.list.d/*chrome*` drops the base image's +> Google Chrome apt repo before updating. That repo's signing key can be +> expired/rotated, which makes `apt-get update` fail outright (`NO_PUBKEY`, +> `not signed`) before the font packages are ever reached. It's safe to remove — +> Chromium is already installed as a binary in the base image; nothing re-installs +> it via apt. + +Point the deployment at it. **Check first whether Gotenberg is managed by +docker-compose or a plain `docker run`** — the earlier `docker ps -a` output tells +you: an auto-generated name (e.g. `jolly_brattain`) means plain `docker run`, no +`--name` given; a fixed/meaningful name (e.g. `saathi_gotenberg`) usually means +compose or a scripted `docker run --name ...`. + +**Case A — docker-compose manages it:** + +Find the `docker-compose.yml` actually driving the container (path varies by server): + +```bash +find / -maxdepth 4 -iname "docker-compose*.y*ml" 2>/dev/null +ls ~/saathi-backend/docker-compose*.y*ml # usually here +``` + +Confirm it's the right file — its `gotenberg:` block should match the running +container's image and port mapping (edit the `COMPOSE_FILE` value first): + +```bash +COMPOSE_FILE=~/saathi-backend/docker-compose.yml +grep -n -A10 "gotenberg:" "$COMPOSE_FILE" +``` + +Edit the gotenberg service in that file: ```yaml gotenberg: @@ -71,6 +134,31 @@ cd ~/saathi-backend && sudo docker compose up -d gotenberg > next reboot — no manual Compose call needed once the image is built and the > `image:` line is edited. +**Case B — plain `docker run` (no compose file), e.g. started inside a tmux session:** + +There's no config file to edit — you recreate the container directly. Capture its +current host port automatically (don't hand-type it), then recreate with an +explicit `--name` so future lookups don't depend on Docker's random name generator: + +```bash +export PORT=$(sudo docker inspect "$GC" --format '{{(index (index .HostConfig.PortBindings "3000/tcp") 0).HostPort}}') +echo "Gotenberg host port: $PORT" + +sudo docker rm -f "$GC" +sudo docker run -d --name gotenberg --restart unless-stopped -p "$PORT:3000" gotenberg-noto:8 +export GC=gotenberg +``` + +Then find whatever started the *old* container (a tmux session, `~/.bash_history` +entry, cron `@reboot`, or systemd unit) and update it to launch `gotenberg-noto:8` +with `--name gotenberg` too — otherwise the next reboot may bring the stock, +font-less image back up alongside/instead of this one: + +```bash +grep -rn "gotenberg/gotenberg\|docker run" ~/.bash_history /etc/systemd/system /etc/rc.local 2>/dev/null +crontab -l 2>/dev/null | grep -i gotenberg +``` + ### Quick / temporary (for immediate testing only) Installs into the running container. **Lost when the container is recreated** @@ -78,9 +166,9 @@ Installs into the running container. **Lost when the container is recreated** in production: ```bash -sudo docker exec -u root saathi_gotenberg sh -c \ +sudo docker exec -u root "$GC" sh -c \ "apt-get update && apt-get install -y --no-install-recommends fonts-noto-core && fc-cache -f" -sudo docker restart saathi_gotenberg +sudo docker restart "$GC" ``` --- @@ -129,15 +217,17 @@ print('patched:', 'Noto Sans Tamil' in t.template) ```bash # fonts present in the container? -sudo docker exec saathi_gotenberg fc-list | grep -iE "tamil|devanagari|kannada|oriya" +sudo docker exec "$GC" fc-list | grep -iE "tamil|devanagari|kannada|oriya" ``` End-to-end: trigger a non-English PDF download and confirm the labels render. To confirm at the font level which font a rendered PDF actually used (tofu = only -Liberation embedded; working = Noto embedded): +Liberation embedded; working = Noto embedded). `GOTENBERG_URL` lives in the app's +`.env`, not the shell, so pull it from there: ```bash +export GOTENBERG_URL=$(grep -m1 '^GOTENBERG_URL=' ~/saathi-backend/.env | cut -d= -f2- | tr -d '"') curl -s -o out.pdf -F 'files=@test.html;filename=index.html' "$GOTENBERG_URL" strings out.pdf | grep -oiE "(Liberation|Noto)[A-Za-z]*" | sort -u ``` diff --git a/shikshalokam/views/profile_views.py b/shikshalokam/views/profile_views.py index addce42..89098a6 100644 --- a/shikshalokam/views/profile_views.py +++ b/shikshalokam/views/profile_views.py @@ -42,6 +42,8 @@ def read_elevate_profile(request): 'message': 'Failed to fetch or create profile from Elevate.' }, status=500) + ums_profile = profile_details.get('ums_profile') or {} + logger.info('[read_elevate_profile] profile=%s', profile_details.get('profileid')) return Response({ 'status': 'ok', @@ -51,5 +53,10 @@ def read_elevate_profile(request): 'has_accepted_tnc': profile_details.get('has_accepted_tnc', False), 'route': profile_details.get('route'), 'reroute_url': profile_details.get('reroute_url'), + 'name': profile_details.get('name'), + 'role': ums_profile.get('designation'), + 'school_name': ums_profile.get('org_associated'), + 'district': ums_profile.get('district'), + 'state': ums_profile.get('state'), } }, status=200)