diff --git a/backend/apps/owasp/migrations/0034_alter_chapter_leaders_and_more.py b/backend/apps/owasp/migrations/0034_alter_chapter_leaders_and_more.py new file mode 100644 index 0000000000..420642fee8 --- /dev/null +++ b/backend/apps/owasp/migrations/0034_alter_chapter_leaders_and_more.py @@ -0,0 +1,73 @@ +# Generated by Django 5.2 on 2025-05-12 20:50 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("github", "0026_alter_organization_company_and_more"), + ("owasp", "0033_merge_20250510_0037"), + ] + + operations = [ + migrations.AlterField( + model_name="chapter", + name="leaders", + field=models.ManyToManyField( + blank=True, + related_name="assigned_%(class)s", + to="github.user", + verbose_name="Leaders", + ), + ), + migrations.AlterField( + model_name="chapter", + name="suggested_leaders", + field=models.ManyToManyField( + blank=True, + related_name="matched_%(class)s", + to="github.user", + verbose_name="Suggested leaders", + ), + ), + migrations.AlterField( + model_name="committee", + name="leaders", + field=models.ManyToManyField( + blank=True, + related_name="assigned_%(class)s", + to="github.user", + verbose_name="Leaders", + ), + ), + migrations.AlterField( + model_name="committee", + name="suggested_leaders", + field=models.ManyToManyField( + blank=True, + related_name="matched_%(class)s", + to="github.user", + verbose_name="Suggested leaders", + ), + ), + migrations.AlterField( + model_name="project", + name="leaders", + field=models.ManyToManyField( + blank=True, + related_name="assigned_%(class)s", + to="github.user", + verbose_name="Leaders", + ), + ), + migrations.AlterField( + model_name="project", + name="suggested_leaders", + field=models.ManyToManyField( + blank=True, + related_name="matched_%(class)s", + to="github.user", + verbose_name="Suggested leaders", + ), + ), + ] diff --git a/backend/apps/slack/admin.py b/backend/apps/slack/admin.py index bdc8361c34..fe6491b787 100644 --- a/backend/apps/slack/admin.py +++ b/backend/apps/slack/admin.py @@ -2,17 +2,72 @@ from django.contrib import admin, messages -from apps.slack.models.channel import Channel +from apps.slack.models.conversation import Conversation from apps.slack.models.event import Event from apps.slack.models.member import Member from apps.slack.models.workspace import Workspace -class ChannelAdmin(admin.ModelAdmin): - list_filter = ("is_private",) +class ConversationAdmin(admin.ModelAdmin): + list_display = ( + "name", + "slack_channel_id", + "created_at", + "is_private", + "is_archived", + "is_general", + ) search_fields = ( "name", + "topic", + "purpose", + "slack_channel_id", + "slack_creator_id", + ) + list_filter = ( + "created_at", + "is_archived", + "is_channel", + "is_general", + "is_im", + "is_private", + ) + readonly_fields = ( "slack_channel_id", + "created_at", + "slack_creator_id", + ) + fieldsets = ( + ( + "Conversation Information", + { + "fields": ( + "slack_channel_id", + "name", + "created_at", + "slack_creator_id", + ) + }, + ), + ( + "Properties", + { + "fields": ( + "is_private", + "is_archived", + "is_general", + ) + }, + ), + ( + "Content", + { + "fields": ( + "topic", + "purpose", + ) + }, + ), ) @@ -29,6 +84,7 @@ class EventAdmin(admin.ModelAdmin): class MemberAdmin(admin.ModelAdmin): actions = ("approve_suggested_users",) + autocomplete_fields = ("user",) filter_horizontal = ("suggested_users",) search_fields = ( "slack_user_id", @@ -75,7 +131,7 @@ class WorkspaceAdmin(admin.ModelAdmin): ) -admin.site.register(Channel, ChannelAdmin) +admin.site.register(Conversation, ConversationAdmin) admin.site.register(Event, EventAdmin) admin.site.register(Member, MemberAdmin) admin.site.register(Workspace, WorkspaceAdmin) diff --git a/backend/apps/slack/management/commands/slack_sync_data.py b/backend/apps/slack/management/commands/slack_sync_data.py index 69828e4c03..4cd6e56f9b 100644 --- a/backend/apps/slack/management/commands/slack_sync_data.py +++ b/backend/apps/slack/management/commands/slack_sync_data.py @@ -1,18 +1,40 @@ """A command to populate Slack channels and members data based on workspaces's bot tokens.""" +import logging +import time + from django.core.management.base import BaseCommand from slack_sdk import WebClient from slack_sdk.errors import SlackApiError -from apps.slack.models import Channel, Member, Workspace +from apps.slack.models import Conversation, Member, Workspace + +logger = logging.getLogger(__name__) class Command(BaseCommand): help = "Populate channels and members for all Slack workspaces using their bot tokens" + def add_arguments(self, parser): + """Define command line arguments.""" + parser.add_argument( + "--batch-size", + type=int, + default=1000, + help="Number of conversations to retrieve per request", + ) + parser.add_argument( + "--delay", + type=float, + default=0.5, + help="Delay between API requests in seconds", + ) + def handle(self, *args, **options): - workspaces = Workspace.objects.all() + batch_size = options["batch_size"] + delay = options["delay"] + workspaces = Workspace.objects.all() if not workspaces.exists(): self.stdout.write(self.style.WARNING("No workspaces found in the database")) return @@ -25,56 +47,72 @@ def handle(self, *args, **options): continue client = WebClient(token=bot_token) - total_channels = 0 - total_members = 0 - self.stdout.write(f"Fetching channels for {workspace}...") + self.stdout.write(f"Fetching conversations for {workspace}...") + conversations = [] + total_channels = 0 try: cursor = None while True: response = client.conversations_list( - types="public_channel,private_channel", limit=1000, cursor=cursor + cursor=cursor, + exclude_archived=False, + limit=batch_size, + timeout=30, + types="public_channel,private_channel", ) self._handle_slack_response(response, "conversations_list") - for channel in response["channels"]: - # TODO(arkid15r): use bulk save. - Channel.update_data(workspace, channel) + conversations.extend( + member + for conversation_data in response["channels"] + if (member := Conversation.update_data(conversation_data, workspace)) + ) total_channels += len(response["channels"]) - cursor = response.get("response_metadata", {}).get("next_cursor") - if not cursor: + if not (cursor := response.get("response_metadata", {}).get("next_cursor")): break - self.stdout.write(self.style.SUCCESS(f"Populated {total_channels} channels")) + if delay: + time.sleep(delay) except SlackApiError as e: self.stdout.write( - self.style.ERROR(f"Failed to fetch channels: {e.response['error']}") + self.style.ERROR(f"Failed to fetch conversations: {e.response['error']}") ) + if conversations: + Conversation.bulk_save(conversations) + self.stdout.write(self.style.SUCCESS(f"Populated {total_channels} channels")) self.stdout.write(f"Fetching members for {workspace}...") + members = [] + total_members = 0 try: cursor = None while True: - response = client.users_list(limit=1000, cursor=cursor) + response = client.users_list( + cursor=cursor, + limit=batch_size, + timeout=30, + ) self._handle_slack_response(response, "users_list") - member_count = 0 - for member in response["members"]: - # TODO(arkid15r): use bulk save. - Member.update_data(workspace, member) - member_count += 1 - total_members += member_count + members.extend( + member + for member_data in response["members"] + if (member := Member.update_data(member_data, workspace)) + ) + total_members += len(response["members"]) cursor = response.get("response_metadata", {}).get("next_cursor") if not cursor: break - - self.stdout.write(self.style.SUCCESS(f"Populated {total_members} members")) except SlackApiError as e: self.stdout.write( self.style.ERROR(f"Failed to fetch members: {e.response['error']}") ) + if members: + Member.bulk_save(members) + self.stdout.write(self.style.SUCCESS(f"Populated {total_members} members")) self.stdout.write(self.style.SUCCESS("\nFinished processing all workspaces")) @@ -82,4 +120,5 @@ def _handle_slack_response(self, response, api_method): """Handle Slack API response and raise exception if needed.""" if not response["ok"]: error_message = f"{api_method} API call failed" - raise SlackApiError(error_message, response) + logger.error(error_message) + self.stdout.write(self.style.ERROR(error_message)) diff --git a/backend/apps/slack/migrations/0007_channel_created_at_channel_is_archived_and_more.py b/backend/apps/slack/migrations/0007_channel_created_at_channel_is_archived_and_more.py new file mode 100644 index 0000000000..391e576040 --- /dev/null +++ b/backend/apps/slack/migrations/0007_channel_created_at_channel_is_archived_and_more.py @@ -0,0 +1,69 @@ +# Generated by Django 5.2 on 2025-05-12 20:58 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("github", "0026_alter_organization_company_and_more"), + ("slack", "0006_member_is_bot_alter_member_user"), + ] + + operations = [ + migrations.AddField( + model_name="channel", + name="created_at", + field=models.DateTimeField(blank=True, null=True, verbose_name="Created at"), + ), + migrations.AddField( + model_name="channel", + name="is_archived", + field=models.BooleanField(default=False, verbose_name="Is archived"), + ), + migrations.AddField( + model_name="channel", + name="is_general", + field=models.BooleanField(default=False, verbose_name="Is general"), + ), + migrations.AddField( + model_name="channel", + name="purpose", + field=models.TextField(blank=True, default="", verbose_name="Purpose"), + ), + migrations.AddField( + model_name="channel", + name="slack_creator_id", + field=models.CharField(default="1", max_length=255, verbose_name="Creator ID"), + preserve_default=False, + ), + migrations.AddField( + model_name="channel", + name="topic", + field=models.TextField(blank=True, default="", verbose_name="Topic"), + ), + migrations.AlterField( + model_name="channel", + name="is_private", + field=models.BooleanField(default=False, verbose_name="Is private"), + ), + migrations.AlterField( + model_name="channel", + name="member_count", + field=models.PositiveIntegerField(default=0, verbose_name="Member count"), + ), + migrations.AlterField( + model_name="channel", + name="name", + field=models.CharField(default="", max_length=100, verbose_name="Name"), + ), + migrations.AlterField( + model_name="member", + name="suggested_users", + field=models.ManyToManyField( + blank=True, + related_name="suggested_slack_users", + to="github.user", + verbose_name="Github user suggestions", + ), + ), + ] diff --git a/backend/apps/slack/migrations/0008_channel_is_channel_channel_is_group_channel_is_im_and_more.py b/backend/apps/slack/migrations/0008_channel_is_channel_channel_is_group_channel_is_im_and_more.py new file mode 100644 index 0000000000..943826f750 --- /dev/null +++ b/backend/apps/slack/migrations/0008_channel_is_channel_channel_is_group_channel_is_im_and_more.py @@ -0,0 +1,37 @@ +# Generated by Django 5.2 on 2025-05-12 21:24 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("slack", "0007_channel_created_at_channel_is_archived_and_more"), + ] + + operations = [ + migrations.AddField( + model_name="channel", + name="is_channel", + field=models.BooleanField(default=False, verbose_name="Is channel"), + ), + migrations.AddField( + model_name="channel", + name="is_group", + field=models.BooleanField(default=False, verbose_name="Is group"), + ), + migrations.AddField( + model_name="channel", + name="is_im", + field=models.BooleanField(default=False, verbose_name="Is IM"), + ), + migrations.AddField( + model_name="channel", + name="is_mpim", + field=models.BooleanField(default=False, verbose_name="Is MPIM"), + ), + migrations.AddField( + model_name="channel", + name="is_shared", + field=models.BooleanField(default=False, verbose_name="Is shared"), + ), + ] diff --git a/backend/apps/slack/migrations/0009_rename_channel_conversation_and_more.py b/backend/apps/slack/migrations/0009_rename_channel_conversation_and_more.py new file mode 100644 index 0000000000..734853b37a --- /dev/null +++ b/backend/apps/slack/migrations/0009_rename_channel_conversation_and_more.py @@ -0,0 +1,24 @@ +# Generated by Django 5.2 on 2025-05-12 21:51 + +from django.db import migrations + + +class Migration(migrations.Migration): + dependencies = [ + ("slack", "0008_channel_is_channel_channel_is_group_channel_is_im_and_more"), + ] + + operations = [ + migrations.RenameModel( + old_name="Channel", + new_name="Conversation", + ), + migrations.AlterModelOptions( + name="conversation", + options={"verbose_name_plural": "Conversations"}, + ), + migrations.AlterModelTable( + name="conversation", + table="slack_conversations", + ), + ] diff --git a/backend/apps/slack/models/__init__.py b/backend/apps/slack/models/__init__.py index dfc517a754..e3c913a5dd 100644 --- a/backend/apps/slack/models/__init__.py +++ b/backend/apps/slack/models/__init__.py @@ -1,4 +1,4 @@ -from .channel import Channel +from .conversation import Conversation from .event import Event from .member import Member from .workspace import Workspace diff --git a/backend/apps/slack/models/channel.py b/backend/apps/slack/models/channel.py deleted file mode 100644 index aeec8477e8..0000000000 --- a/backend/apps/slack/models/channel.py +++ /dev/null @@ -1,37 +0,0 @@ -"""Slack app channel model.""" - -from django.db import models - -from apps.common.models import TimestampedModel -from apps.slack.models.workspace import Workspace - - -class Channel(TimestampedModel): - """Slack Channel model.""" - - class Meta: - db_table = "slack_channels" - verbose_name_plural = "Channels" - - is_private = models.BooleanField(verbose_name="Is Private", default=False) - member_count = models.PositiveIntegerField(verbose_name="Member Count", default=0) - name = models.CharField(verbose_name="Channel Name", max_length=100, default="") - slack_channel_id = models.CharField(verbose_name="Channel ID", max_length=50, unique=True) - workspace = models.ForeignKey(Workspace, on_delete=models.CASCADE, related_name="channels") - - def __str__(self): - """Channel human readable representation.""" - return f"#{self.name} - {self.workspace}" - - @staticmethod - def update_data(workspace, channel_data) -> None: - """Update instance based on Slack data.""" - Channel.objects.update_or_create( - slack_channel_id=channel_data["id"], - workspace=workspace, - defaults={ - "is_private": channel_data["is_private"], - "member_count": channel_data.get("num_members", 0), - "name": channel_data["name"], - }, - ) diff --git a/backend/apps/slack/models/conversation.py b/backend/apps/slack/models/conversation.py new file mode 100644 index 0000000000..b2e1412419 --- /dev/null +++ b/backend/apps/slack/models/conversation.py @@ -0,0 +1,90 @@ +"""Slack app channel model.""" + +from datetime import UTC, datetime + +from django.db import models + +from apps.common.models import BulkSaveModel, TimestampedModel +from apps.slack.models.workspace import Workspace + + +class Conversation(TimestampedModel): + """Slack Conversation model.""" + + class Meta: + db_table = "slack_conversations" + verbose_name_plural = "Conversations" + + created_at = models.DateTimeField(verbose_name="Created at", blank=True, null=True) + is_archived = models.BooleanField(verbose_name="Is archived", default=False) + is_channel = models.BooleanField(verbose_name="Is channel", default=False) + is_general = models.BooleanField(verbose_name="Is general", default=False) + is_group = models.BooleanField(verbose_name="Is group", default=False) + is_im = models.BooleanField(verbose_name="Is IM", default=False) + is_mpim = models.BooleanField(verbose_name="Is MPIM", default=False) + is_private = models.BooleanField(verbose_name="Is private", default=False) + is_shared = models.BooleanField(verbose_name="Is shared", default=False) + member_count = models.PositiveIntegerField(verbose_name="Member count", default=0) + name = models.CharField(verbose_name="Name", max_length=100, default="") + purpose = models.TextField(verbose_name="Purpose", blank=True, default="") + slack_channel_id = models.CharField(verbose_name="Channel ID", max_length=50, unique=True) + slack_creator_id = models.CharField(verbose_name="Creator ID", max_length=255) + topic = models.TextField(verbose_name="Topic", blank=True, default="") + workspace = models.ForeignKey(Workspace, on_delete=models.CASCADE, related_name="channels") + + def __str__(self): + """Channel human readable representation.""" + return f"{self.name} - {self.workspace}" + + def from_slack(self, conversation_data, workspace: Workspace) -> None: + """Update instance based on Slack conversation data.""" + self.created_at = datetime.fromtimestamp(int(conversation_data.get("created", 0)), tz=UTC) + + for attr_name in ( + "is_archived", + "is_channel", + "is_general", + "is_group", + "is_im", + "is_mpim", + "is_private", + "is_shared", + ): + setattr(self, attr_name, conversation_data.get(attr_name, False)) + + self.name = conversation_data.get("name", "") + self.purpose = conversation_data.get("purpose", {}).get("value", "") + self.slack_creator_id = conversation_data.get("creator", "") + self.topic = conversation_data.get("topic", {}).get("value", "") + + self.workspace = workspace + + @staticmethod + def bulk_save(conversations, fields=None): + """Bulk save conversations.""" + BulkSaveModel.bulk_save(Conversation, conversations, fields=fields) + + @staticmethod + def update_data(conversation_data, workspace, *, save=True): + """Update Channel data from Slack. + + Args: + workspace (Workspace): Workspace instance + conversation_data: Dictionary with conversation data from Slack API + save: Whether to save the model after updating + + Returns: + Updated or created Channel instance, or None if error + + """ + channel_id = conversation_data["id"] + try: + conversation = Conversation.objects.get(slack_channel_id=channel_id) + except Conversation.DoesNotExist: + conversation = Conversation(slack_channel_id=channel_id) + + conversation.from_slack(conversation_data, workspace) + if save: + conversation.save() + + return conversation diff --git a/backend/apps/slack/models/member.py b/backend/apps/slack/models/member.py index b2cd0c6e3e..23fdf9a4ed 100644 --- a/backend/apps/slack/models/member.py +++ b/backend/apps/slack/models/member.py @@ -2,7 +2,7 @@ from django.db import models -from apps.common.models import TimestampedModel +from apps.common.models import BulkSaveModel, TimestampedModel from .workspace import Workspace @@ -47,16 +47,32 @@ def __str__(self): """Member human readable representation.""" return f"{self.username or 'Unnamed'} ({self.slack_user_id})" + def from_slack(self, member_data, workspace: Workspace) -> None: + """Update instance based on Slack member data.""" + self.email = member_data["profile"].get("email", "") + self.is_bot = member_data["is_bot"] + self.real_name = member_data.get("real_name", "") + self.slack_user_id = member_data["id"] + self.username = member_data["name"] + + self.workspace = workspace + + @staticmethod + def bulk_save(members, fields=None): + """Bulk save members.""" + BulkSaveModel.bulk_save(Member, members, fields=fields) + @staticmethod - def update_data(workspace, member_data) -> None: + def update_data(member_data, workspace, *, save=True) -> None: """Update instance based on Slack data.""" - Member.objects.update_or_create( - slack_user_id=member_data["id"], - workspace=workspace, - defaults={ - "email": member_data["profile"].get("email", ""), - "is_bot": member_data["is_bot"], - "real_name": member_data.get("real_name", ""), - "username": member_data["name"], - }, - ) + member_id = member_data["id"] + try: + member = Member.objects.get(slack_user_id=member_id) + except Member.DoesNotExist: + member = Member(slack_user_id=member_id) + + member.from_slack(member_data, workspace) + if save: + member.save() + + return member diff --git a/backend/tests/slack/commands/management/__init__.py b/backend/tests/slack/commands/management/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/backend/tests/slack/models/__init__.py b/backend/tests/slack/models/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/backend/tests/slack/models/conversation_test.py b/backend/tests/slack/models/conversation_test.py new file mode 100644 index 0000000000..3dc8182203 --- /dev/null +++ b/backend/tests/slack/models/conversation_test.py @@ -0,0 +1,139 @@ +from datetime import UTC, datetime +from unittest.mock import Mock, patch + +from apps.slack.models.conversation import Conversation +from apps.slack.models.workspace import Workspace + + +class TestConversationModel: + def test_bulk_save(self): + mock_conversations = [Mock(id=None), Mock(id=1)] + with patch("apps.common.models.BulkSaveModel.bulk_save") as mock_bulk_save: + Conversation.bulk_save(mock_conversations) + mock_bulk_save.assert_called_once_with(Conversation, mock_conversations, fields=None) + + def test_update_data_new_conversation(self, mocker): + # Setup conversation data from Slack API + conversation_data = { + "id": "C12345", + "name": "general", + "created": "1605000000", + "is_private": False, + "is_archived": False, + "is_general": True, + "topic": {"value": "General topic"}, + "purpose": {"value": "General purpose"}, + "creator": "U12345", + } + + # Mock the DoesNotExist exception when getting conversation + mocker.patch( + "apps.slack.models.conversation.Conversation.objects.get", + side_effect=Conversation.DoesNotExist, + ) + + # Mock the from_slack method + mocker.patch.object(Conversation, "from_slack") + + # Mock the save method + mocker.patch.object(Conversation, "save") + + # Call update_data + result = Conversation.update_data(conversation_data, Workspace()) + + # Assertions + assert result is not None + assert result.slack_channel_id == "C12345" + assert result.from_slack.call_count == 1 + assert result.save.call_count == 1 + + def test_update_data_existing_conversation(self, mocker): + # Setup conversation data from Slack API + conversation_data = { + "id": "C12345", + "name": "general", + "created": "1605000000", + "is_private": False, + "is_archived": False, + "is_general": True, + "topic": {"value": "General topic"}, + "purpose": {"value": "General purpose"}, + "creator": "U12345", + } + + # Create a mock conversation object + mock_conversation = mocker.Mock(spec=Conversation) + mock_conversation.slack_channel_id = "C12345" + + # Mock the objects.get to return the mock conversation + mocker.patch( + "apps.slack.models.conversation.Conversation.objects.get", + return_value=mock_conversation, + ) + + # Call update_data + result = Conversation.update_data(conversation_data, Workspace()) + + # Assertions + assert result is not None + assert result.slack_channel_id == "C12345" + assert result.from_slack.call_count == 1 + assert result.save.call_count == 1 + + def test_update_data_no_save(self, mocker): + # Setup conversation data + conversation_data = {"id": "C12345", "name": "general"} + + # Mock Conversation.objects.get + mocker.patch( + "apps.slack.models.conversation.Conversation.objects.get", + side_effect=Conversation.DoesNotExist, + ) + + # Mock the save method + save_mock = mocker.patch.object(Conversation, "save") + + # Call update_data with save=False + result = Conversation.update_data(conversation_data, Workspace(), save=False) + + # Assertions + assert result is not None + assert save_mock.call_count == 0 + + def test_from_slack(self): + # Create test data + conversation_data = { + "name": "general", + "created": "1605000000", # Unix timestamp + "is_private": True, + "is_archived": True, + "is_general": True, + "topic": {"value": "General topic"}, + "purpose": {"value": "General purpose"}, + "creator": "U12345", + } + + # Create a conversation instance + conversation = Conversation() + + # Call from_slack + conversation.from_slack(conversation_data, Workspace()) + + # Assertions + assert conversation.name == "general" + assert conversation.created_at == datetime.fromtimestamp(1605000000, tz=UTC) + assert conversation.is_private is True + assert conversation.is_archived is True + assert conversation.is_general is True + assert conversation.topic == "General topic" + assert conversation.purpose == "General purpose" + assert conversation.slack_creator_id == "U12345" + + def test_str_method(self): + # Create a conversation with a name + conversation = Conversation( + name="test-channel", workspace=Workspace(name="test-workspace") + ) + + # Check __str__ returns the name + assert str(conversation) == "test-channel - test-workspace"