Saving entire body of messsages

backend/apps/slack/management/commands/slack_sync_messages.py

@@ -322,32 +322,17 @@ def _create_message_from_data(
         parent_message: Message | None = None,
     ) -> Message | None:
         """Create Message instance using from_slack pattern."""
-        if message_data.get("subtype") in {
-            "channel_join",
-            "channel_leave",
-            "bot_message",
-        } or not any(
-            [
-                message_data.get("text"),
-                message_data.get("attachments"),
-                message_data.get("files"),
-                message_data.get("blocks"),
-            ]
-        ):
-            return None
-
-        try:
-            if not (slack_user_id := (message_data.get("user") or message_data.get("bot_id"))):
-                return None
+        slack_user_id = message_data.get("user")
+        bot_id = message_data.get("bot_id")
+        author = None
 
+        if slack_user_id:
             try:
                 author = Member.objects.get(
                     slack_user_id=slack_user_id, workspace=conversation.workspace
                 )
             except Member.DoesNotExist:
-                author = None
                 retry_count = 0
-
                 while retry_count < max_retries:
                     try:
                         time.sleep(delay)
@@ -362,45 +347,62 @@ def _create_message_from_data(
                             self.style.SUCCESS(f"Created new member: {slack_user_id}")
                         )
                         break
-                    except SlackApiError as e:
-                        if e.response["error"] == "ratelimited":
-                            retry_after = int(
-                                e.response.headers.get("Retry-After", delay * (retry_count + 1))
-                            )
 
+                    except SlackApiError as e:
+                        if e.response.get("error") == "ratelimited":
+                            retry_after = int(e.response.headers.get("Retry-After", delay))
                             retry_count += 1
+                            self.stdout.write(self.style.WARNING("Rate limited on user info"))
+                            time.sleep(retry_after)
+                        else:
                             self.stdout.write(
-                                self.style.WARNING(
-                                    f"Rate limited on user info. Retrying after {retry_after}s"
+                                self.style.ERROR(
+                                    f"Failed to fetch user data for {slack_user_id}: {e!s}"
                                 )
                             )
+                            break
+
+        elif bot_id:
+            try:
+                author = Member.objects.get(slack_user_id=bot_id, workspace=conversation.workspace)
+            except Member.DoesNotExist:
+                retry_count = 0
+                while retry_count < max_retries:
+                    try:
+                        time.sleep(delay)
+                        bot_info = client.bots_info(bot=bot_id)
+                        self._handle_slack_response(bot_info, "bots_info")
+
+                        bot_data = {
+                            "id": bot_id,
+                            "is_bot": True,
+                            "name": bot_info["bot"].get("name"),
+                            "real_name": bot_info["bot"].get("name"),
+                        }
+
+                        author = Member.update_data(bot_data, conversation.workspace, save=True)
+                        self.stdout.write(self.style.SUCCESS(f"Created bot member: {bot_id}"))
+                        break
+
+                    except SlackApiError as e:
+                        if e.response.get("error") == "ratelimited":
+                            retry_after = int(e.response.headers.get("Retry-After", delay))
+                            retry_count += 1
+                            self.stdout.write(self.style.WARNING("Rate limited on bot info"))
                             time.sleep(retry_after)
                         else:
                             self.stdout.write(
-                                self.style.WARNING(
-                                    f"Failed to fetch user data for {slack_user_id}"
-                                )
+                                self.style.WARNING(f"Failed to fetch bot data for {bot_id}.")
                             )
-                            return None
-
-                if not author:
-                    self.stdout.write(
-                        self.style.WARNING(
-                            f"Could not fetch user {slack_user_id}, skipping message"
-                        )
-                    )
-                    return None
+                            break
 
-            return Message.update_data(
-                data=message_data,
-                conversation=conversation,
-                author=author,
-                parent_message=parent_message,
-                save=False,
-            )
-        except Exception:
-            logger.exception("Error creating message from data")
-            return None
+        return Message.update_data(
+            data=message_data,
+            conversation=conversation,
+            author=author,
+            parent_message=parent_message,
+            save=False,
+        )
 
     def _handle_slack_response(self, response, api_method):
         """Handle Slack API response and raise exception if needed."""

backend/apps/slack/migrations/0016_message_raw_data_alter_message_author.py

@@ -0,0 +1,29 @@
+# Generated by Django 5.2.3 on 2025-06-18 18:38
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("slack", "0015_remove_message_is_thread_parent_message_has_replies_and_more"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="message",
+            name="raw_data",
+            field=models.JSONField(default=dict, verbose_name="Raw data"),
+        ),
+        migrations.AlterField(
+            model_name="message",
+            name="author",
+            field=models.ForeignKey(
+                blank=True,
+                null=True,
+                on_delete=django.db.models.deletion.CASCADE,
+                related_name="messages",
+                to="slack.member",
+            ),
+        ),
+    ]

backend/apps/slack/models/message.py

@@ -20,11 +20,14 @@ class Meta:
 
     created_at = models.DateTimeField(verbose_name="Created at")
     has_replies = models.BooleanField(verbose_name="Has replies", default=False)
+    raw_data = models.JSONField(verbose_name="Raw data", default=dict)
     slack_message_id = models.CharField(verbose_name="Slack message ID", max_length=50)
     text = models.TextField(verbose_name="Text")
 
     # FKs.
-    author = models.ForeignKey(Member, on_delete=models.CASCADE, related_name="messages")
+    author = models.ForeignKey(
+        Member, on_delete=models.CASCADE, related_name="messages", blank=True, null=True
+    )
     conversation = models.ForeignKey(
         Conversation, on_delete=models.CASCADE, related_name="messages"
     )
@@ -44,13 +47,14 @@ def from_slack(
         self,
         message_data: dict,
         conversation: Conversation,
-        author: Member,
+        author: "Member | None" = None,
         *,
         parent_message: "Message | None" = None,
     ) -> None:
         """Update instance based on Slack message data."""
         self.created_at = datetime.fromtimestamp(float(message_data["ts"]), tz=UTC)
         self.has_replies = message_data.get("reply_count", 0) > 0
+        self.raw_data = message_data
         self.slack_message_id = message_data.get("ts", "")
         self.text = message_data.get("text", "")
 
@@ -67,7 +71,7 @@ def bulk_save(messages: list["Message"], fields=None) -> None:
     def update_data(
         data: dict,
         conversation: Conversation,
-        author: Member,
+        author: Member | None = None,
         *,
         parent_message: "Message | None" = None,
         save: bool = True,

backend/tests/slack/commands/management/slack_sync_messages_test.py

@@ -231,63 +231,27 @@ def test_handle_successful_sync(
         assert "Processing channel: general" in output
         assert "Finished processing all workspaces" in output
 
-    def test_create_message_from_data_channel_join_subtype(self, command, mock_conversation):
-        """Test _create_message_from_data with channel_join subtype."""
-        message_data = {
-            "ts": TEST_MESSAGE_TS,
-            "subtype": "channel_join",
-            "text": "User joined channel",
-        }
-
-        mock_client = Mock()
-        result = command._create_message_from_data(
-            client=mock_client,
-            message_data=message_data,
-            conversation=mock_conversation,
-            delay=0.5,
-            max_retries=5,
-            parent_message=None,
-        )
-
-        assert result is None
-
-    def test_create_message_from_data_no_content(self, command, mock_conversation):
-        """Test _create_message_from_data with no text, attachments, or files."""
-        message_data = {
-            "ts": TEST_MESSAGE_TS,
-            "user": "U12345",
-        }
-
-        mock_client = Mock()
-        result = command._create_message_from_data(
-            client=mock_client,
-            message_data=message_data,
-            conversation=mock_conversation,
-            delay=0.5,
-            max_retries=5,
-            parent_message=None,
-        )
-
-        assert result is None
-
-    def test_create_message_from_data_no_user(self, command, mock_conversation):
-        """Test _create_message_from_data with no user or bot_id."""
+    def test_create_message_from_data_no_user_or_bot(self, command, mock_conversation):
+        """Test _create_message_from_data when no user or bot_id is provided."""
         message_data = {
             "ts": TEST_MESSAGE_TS,
             "text": "Hello world!",
         }
 
         mock_client = Mock()
-        result = command._create_message_from_data(
-            client=mock_client,
-            message_data=message_data,
-            conversation=mock_conversation,
-            delay=0.5,
-            max_retries=5,
-            parent_message=None,
-        )
+        mock_message = Mock(spec=Message)
 
-        assert result is None
+        with patch.object(Message, "update_data", return_value=mock_message):
+            result = command._create_message_from_data(
+                client=mock_client,
+                message_data=message_data,
+                conversation=mock_conversation,
+                delay=0.5,
+                max_retries=5,
+                parent_message=None,
+            )
+
+        assert result is not None
 
     @patch("apps.slack.management.commands.slack_sync_messages.time.sleep")
     def test_create_message_from_data_member_not_found(