feat(nestbot): Add image extraction support for NestBot AI assistant

backend/apps/common/open_ai.py

@@ -2,6 +2,7 @@
 
 from __future__ import annotations
 
+import base64
 import logging
 
 import openai
@@ -32,6 +33,8 @@ def __init__(
         self.max_tokens = max_tokens
         self.model = model
         self.temperature = temperature
+        self.image_data: bytes | None = None
+        self.image_mime_type: str | None = None
 
     def set_input(self, content: str) -> OpenAi:
         """Set system role content.
@@ -75,6 +78,22 @@ def set_prompt(self, content: str) -> OpenAi:
 
         return self
 
+    def set_image(self, image_data: bytes, mime_type: str) -> OpenAi:
+        """Set image data for vision API.
+
+        Args:
+            image_data (bytes): Raw image bytes.
+            mime_type (str): MIME type of the image (e.g., "image/png").
+
+        Returns:
+            OpenAi: The current instance.
+
+        """
+        self.image_data = image_data
+        self.image_mime_type = mime_type
+
+        return self
+
     def complete(self) -> str | None:
         """Get API response.
 
@@ -87,11 +106,27 @@ def complete(self) -> str | None:
 
         """
         try:
+            # Build user message content
+            user_content: str | list[dict[str, object]]
+            if self.image_data and self.image_mime_type:
+                # Vision API with image
+                base64_image = base64.b64encode(self.image_data).decode("utf-8")
+                user_content = [
+                    {"type": "text", "text": self.input},
+                    {
+                        "type": "image_url",
+                        "image_url": {"url": f"data:{self.image_mime_type};base64,{base64_image}"},
+                    },
+                ]
+            else:
+                # Text-only
+                user_content = self.input
+
             response = self.client.chat.completions.create(
                 max_tokens=self.max_tokens,
                 messages=[
                     {"role": "system", "content": self.prompt},
-                    {"role": "user", "content": self.input},
+                    {"role": "user", "content": user_content},
                 ],
                 model=self.model,
                 temperature=self.temperature,

backend/apps/slack/events/message_posted.py

@@ -25,18 +25,27 @@ def __init__(self):
 
     def handle_event(self, event, client):
         """Handle an incoming message event."""
-        if event.get("subtype") or event.get("bot_id"):
-            logger.info("Ignored message due to subtype, bot_id, or thread_ts.")
+        # Ignore bot messages
+        if event.get("bot_id"):
+            logger.info("Ignoring bot message.")
             return
 
+        # Allow file_share subtype (messages with images), ignore others
+        if event.get("subtype") and event.get("subtype") != "file_share":
+            logger.info("Ignoring message with subtype: %s", event.get("subtype"))
+            return
+
+        # Update parent message if this is a thread reply
         if event.get("thread_ts"):
-            try:
-                Message.objects.filter(
-                    slack_message_id=event.get("thread_ts"),
-                    conversation__slack_channel_id=event.get("channel"),
-                ).update(has_replies=True)
-            except Message.DoesNotExist:
-                logger.warning("Thread message not found.")
+            updated = Message.objects.filter(
+                slack_message_id=event.get("thread_ts"),
+                conversation__slack_channel_id=event.get("channel"),
+            ).update(has_replies=True)
+            if not updated:
+                logger.info(
+                    "Parent message for thread_ts %s not found in thread reply.",
+                    event.get("thread_ts"),
+                )
             return
 
         channel_id = event.get("channel")
@@ -49,25 +58,63 @@ def handle_event(self, event, client):
                 is_nest_bot_assistant_enabled=True,
             )
         except Conversation.DoesNotExist:
-            logger.warning("Conversation not found or assistant not enabled.")
+            logger.info("Conversation not found or bot not enabled for channel: %s", channel_id)
             return
 
-        if not self.question_detector.is_owasp_question(text):
+        # Check if message has valid images - only bypass question detector for valid images
+        from apps.slack.services.image_extraction import (
+            extract_images_then_maybe_reply,
+            is_valid_image_file,
+        )
+
+        image_files = [
+            f
+            for f in event.get("files", [])
+            if f.get("mimetype", "").startswith("image/") and is_valid_image_file(f)
+        ][:3]
+
+        # For text-only messages or messages without valid images, use question detector
+        if not image_files and not self.question_detector.is_owasp_question(text):
+            logger.info("Question detector rejected message")
             return
 
         try:
-            author = Member.objects.get(slack_user_id=user_id, workspace=conversation.workspace)
+            author = Member.objects.get(
+                slack_user_id=user_id,
+                workspace=conversation.workspace,
+            )
         except Member.DoesNotExist:
             user_info = client.users_info(user=user_id)
-            author = Member.update_data(user_info["user"], conversation.workspace, save=True)
-            logger.info("Created new member")
+            author = Member.update_data(
+                user_info["user"],
+                conversation.workspace,
+                save=True,
+            )
+            logger.info("Created new member for user_id %s", user_id)
 
         message = Message.update_data(
-            data=event, conversation=conversation, author=author, save=True
+            data=event,
+            conversation=conversation,
+            author=author,
+            save=True,
         )
 
-        django_rq.get_queue("ai").enqueue_in(
-            timedelta(minutes=QUEUE_RESPONSE_TIME_MINUTES),
-            generate_ai_reply_if_unanswered,
-            message.id,
-        )
+        # Handle messages with valid images
+        if image_files:
+            logger.info(
+                "Queueing image extraction for message %s with %s image(s)",
+                message.id,
+                len(image_files),
+            )
+            django_rq.get_queue("ai").enqueue(
+                extract_images_then_maybe_reply,
+                message.id,
+                image_files,
+            )
+        else:
+            logger.info("Queueing AI reply for message %s", message.id)
+            django_rq.get_queue("ai").enqueue_in(
+                timedelta(minutes=QUEUE_RESPONSE_TIME_MINUTES),
+                generate_ai_reply_if_unanswered,
+                message.id,
+            )

backend/apps/slack/models/message.py

@@ -84,6 +84,20 @@ def text(self) -> str:
         """Get the text of the message."""
         return self.raw_data.get("text", "")
 
+    @property
+    def text_with_images(self) -> str:
+        """Get message text combined with extracted image text."""
+        parts = [self.text] if self.text else []
+
+        if extractions := self.raw_data.get("image_extractions", []):
+            parts.extend(
+                f"\n[Image: {extraction.get('file_name', 'unnamed')}]\n"
+                f"{extraction['extracted_text']}"
+                for extraction in extractions
+                if extraction.get("status") == "success" and extraction.get("extracted_text")
+            )
+        return "\n\n".join(parts)
+
     @property
     def ts(self) -> str:
         """Get the message timestamp."""