diff --git a/backend/Makefile b/backend/Makefile index 2b2d68c877..e2acb4ec1e 100644 --- a/backend/Makefile +++ b/backend/Makefile @@ -1,6 +1,6 @@ include backend/apps/ai/Makefile include backend/apps/github/Makefile -include backend/apps/nest/Makefile +include backend/apps/mentorship/Makefile include backend/apps/owasp/Makefile include backend/apps/slack/Makefile diff --git a/backend/apps/github/admin/issue.py b/backend/apps/github/admin/issue.py index 787781ffec..3c397dcd85 100644 --- a/backend/apps/github/admin/issue.py +++ b/backend/apps/github/admin/issue.py @@ -19,6 +19,7 @@ class IssueAdmin(admin.ModelAdmin): "repository", "created_at", "title", + "level", "custom_field_github_url", ) list_filter = ( diff --git a/backend/apps/github/migrations/0037_issue_level.py b/backend/apps/github/migrations/0037_issue_level.py new file mode 100644 index 0000000000..de309aade5 --- /dev/null +++ b/backend/apps/github/migrations/0037_issue_level.py @@ -0,0 +1,26 @@ +# Generated by Django 5.2.5 on 2025-09-25 08:53 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("github", "0036_user_has_public_member_page_alter_organization_name_and_more"), + ("mentorship", "0004_module_key_program_key_and_more"), + ] + + operations = [ + migrations.AddField( + model_name="issue", + name="level", + field=models.ForeignKey( + blank=True, + help_text="The difficulty level of this issue.", + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="issues", + to="mentorship.tasklevel", + ), + ), + ] diff --git a/backend/apps/github/models/issue.py b/backend/apps/github/models/issue.py index 693272a017..97a08ceae3 100644 --- a/backend/apps/github/models/issue.py +++ b/backend/apps/github/models/issue.py @@ -54,6 +54,14 @@ class Meta: null=True, related_name="created_issues", ) + level = models.ForeignKey( + "mentorship.TaskLevel", + null=True, + blank=True, + on_delete=models.SET_NULL, + related_name="issues", + help_text="The difficulty level of this issue.", + ) milestone = models.ForeignKey( "github.Milestone", on_delete=models.CASCADE, diff --git a/backend/apps/mentorship/Makefile b/backend/apps/mentorship/Makefile new file mode 100644 index 0000000000..a43f999761 --- /dev/null +++ b/backend/apps/mentorship/Makefile @@ -0,0 +1,5 @@ +mentorship-sync-module-issues: + @CMD="python manage.py sync_module_issues" $(MAKE) exec-backend-command + +mentorship-sync-issue-levels: + @CMD="python manage.py sync_issue_levels" $(MAKE) exec-backend-command diff --git a/backend/apps/mentorship/admin/module.py b/backend/apps/mentorship/admin/module.py index 1eb248c4dc..df25f85a93 100644 --- a/backend/apps/mentorship/admin/module.py +++ b/backend/apps/mentorship/admin/module.py @@ -13,6 +13,7 @@ class ModuleAdmin(admin.ModelAdmin): "program", "project", ) + autocomplete_fields = ("issues",) search_fields = ( "name", diff --git a/backend/apps/mentorship/admin/task.py b/backend/apps/mentorship/admin/task.py index 74662c10f1..e5ba082a8f 100644 --- a/backend/apps/mentorship/admin/task.py +++ b/backend/apps/mentorship/admin/task.py @@ -25,5 +25,7 @@ class TaskAdmin(admin.ModelAdmin): list_filter = ("status", "module") + ordering = ["-assigned_at"] + admin.site.register(Task, TaskAdmin) diff --git a/backend/apps/mentorship/management/__init__.py b/backend/apps/mentorship/management/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/backend/apps/mentorship/management/commands/__init__.py b/backend/apps/mentorship/management/commands/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/backend/apps/mentorship/management/commands/sync_issue_levels.py b/backend/apps/mentorship/management/commands/sync_issue_levels.py new file mode 100644 index 0000000000..a1abc1da38 --- /dev/null +++ b/backend/apps/mentorship/management/commands/sync_issue_levels.py @@ -0,0 +1,97 @@ +"""A command to sync issue level with Tasklevel.""" + +from django.core.management.base import BaseCommand +from django.db.models import Prefetch + +from apps.github.models.issue import Issue +from apps.github.models.label import Label +from apps.mentorship.models.task_level import TaskLevel +from apps.mentorship.utils import normalize_name + + +class Command(BaseCommand): + """Syncs the `level` field on Issues based on matching labels, respecting Module constraints. + + If any label matches a TaskLevel in the Issue's Module, that TaskLevel is assigned. + """ + + help = "Assigns a TaskLevel to each Issue by matching labels within the same Module." + + def _build_module_level_maps(self, all_levels): + """Build a mapping from module ID to a dictionary of data. + + The dictionary contains a 'label_to_level_map' for normalized label/level + names to TaskLevel objects. + """ + module_data_map = {} + for level in all_levels: + module_id = level.module_id + level_map_container = module_data_map.setdefault(module_id, {"label_to_level_map": {}}) + level_map = level_map_container["label_to_level_map"] + + normalized_level_name = normalize_name(level.name) + level_map[normalized_level_name] = level + + for label_name in level.labels: + normalized_label = normalize_name(label_name) + level_map[normalized_label] = level + return module_data_map + + def _find_best_match_level( + self, + issue_labels_normalized, + issue_mentorship_modules, + module_data_map, + ): + """Find the best matching TaskLevel for an issue based on its labels and modules.""" + for module in issue_mentorship_modules: + if module.id in module_data_map: + module_level_map = module_data_map[module.id]["label_to_level_map"] + for label_name in issue_labels_normalized: + if label_name in module_level_map: + return module_level_map[label_name] + return None + + def handle(self, *args, **options): + self.stdout.write("Starting...") + + # 1. Build a per-module map (normalized label → TaskLevel) + all_levels = TaskLevel.objects.select_related("module").order_by("name") + + if not all_levels.exists(): + self.stdout.write( + self.style.WARNING("No TaskLevel objects found in the database. Exiting.") + ) + return + + module_data_map = self._build_module_level_maps(all_levels) + self.stdout.write(f"Built label maps for {len(module_data_map)} modules.") + + # 2.match issue labels to TaskLevels + issues_to_update = [] + issues_query = Issue.objects.prefetch_related( + Prefetch("labels", queryset=Label.objects.only("name")), + "mentorship_modules", + ).select_related("level") + + for issue in issues_query: + issue_labels_normalized = {normalize_name(label.name) for label in issue.labels.all()} + + best_match_level = self._find_best_match_level( + issue_labels_normalized, + list(issue.mentorship_modules.all()), + module_data_map, + ) + + if issue.level != best_match_level: + issue.level = best_match_level + issues_to_update.append(issue) + + if issues_to_update: + updated_count = len(issues_to_update) + Issue.objects.bulk_update(issues_to_update, ["level"]) + self.stdout.write( + self.style.SUCCESS(f"Successfully updated the level for {updated_count} issues.") + ) + else: + self.stdout.write(self.style.SUCCESS("All issue levels are already up-to-date.")) diff --git a/backend/apps/mentorship/management/commands/sync_module_issues.py b/backend/apps/mentorship/management/commands/sync_module_issues.py new file mode 100644 index 0000000000..9a19fd6b6f --- /dev/null +++ b/backend/apps/mentorship/management/commands/sync_module_issues.py @@ -0,0 +1,237 @@ +"""A command to sync update relation between module and issue and create task.""" + +from urllib.parse import urlparse + +from django.core.management.base import BaseCommand +from django.db import transaction +from django.utils import timezone +from github.GithubException import GithubException + +from apps.github.auth import get_github_client +from apps.github.models.issue import Issue +from apps.mentorship.models.module import Module +from apps.mentorship.models.task import Task +from apps.mentorship.utils import normalize_name + + +class Command(BaseCommand): + """Efficiently syncs issues to mentorship modules based on matching labels.""" + + help = ( + "Syncs issues to modules by matching labels from all repositories " + "associated with the module's project and creates related tasks." + ) + ALLOWED_GITHUB_HOSTS = {"github.com", "www.github.com"} + REPO_PATH_PARTS = 2 + + def _extract_repo_full_name(self, repository): + """Extract repository full name from Repository model or URL string.""" + if hasattr(repository, "path"): + return repository.path + + repo_url = str(repository) if repository else "" + parsed = urlparse(repo_url) + if parsed.netloc in self.ALLOWED_GITHUB_HOSTS: + parts = parsed.path.strip("/").split("/") + if len(parts) >= self.REPO_PATH_PARTS: + return "/".join(parts[: self.REPO_PATH_PARTS]) + return None + return None + + def _get_status(self, issue, assignee): + """Map GitHub issue state + assignment to task status.""" + if issue.state.lower() == "closed": + return Task.Status.COMPLETED + + if assignee: + return Task.Status.IN_PROGRESS + + return Task.Status.TODO + + def _get_last_assigned_date(self, repo, issue_number, assignee_login): + """Find the most recent 'assigned' event for a specific user using PyGithub.""" + try: + gh_issue = repo.get_issue(number=issue_number) + last_dt = None + for event in gh_issue.get_events(): + if ( + event.event == "assigned" + and event.assignee + and event.assignee.login == assignee_login + ): + last_dt = event.created_at + + if last_dt and timezone.is_naive(last_dt): + return timezone.make_aware(last_dt, timezone.utc) + return last_dt # noqa: TRY300 + + except GithubException as e: + self.stderr.write( + self.style.ERROR(f"Unexpected error for {repo.name}#{issue_number}: {e}") + ) + + return None + + def _build_repo_label_to_issue_map(self): + """Build a map from (repository_id, normalized_label_name) to a set of issue IDs.""" + self.stdout.write("Building a repository-aware map of labels to issues...") + repo_label_to_issue_ids = {} + rows = ( + Issue.objects.filter(labels__isnull=False, repository__isnull=False) + .values_list("id", "repository_id", "labels__name") + .iterator(chunk_size=5000) + ) + for issue_id, repo_id, label_name in rows: + key = (repo_id, normalize_name(label_name)) + repo_label_to_issue_ids.setdefault(key, set()).add(issue_id) + + self.stdout.write( + f"Map built. Found issues for {len(repo_label_to_issue_ids)} unique repo-label pairs." + ) + return repo_label_to_issue_ids + + def _process_module( + self, + module, + repo_label_to_issue_ids, + gh_client, + repo_cache, + verbosity, + ): + """Process a single module to link issues and create tasks.""" + project_repos = list(module.project.repositories.all()) + linked_label_names = module.labels + num_tasks_created = 0 + + matched_issue_ids = set() + for repo in project_repos: + for label_name in linked_label_names: + normalized_label = normalize_name(label_name) + key = (repo.id, normalized_label) + issues_for_label = repo_label_to_issue_ids.get(key, set()) + matched_issue_ids.update(issues_for_label) + + with transaction.atomic(): + module.issues.set(matched_issue_ids) + + if matched_issue_ids: + issues = ( + Issue.objects.filter( + id__in=matched_issue_ids, + assignees__isnull=False, + ) + .select_related("repository") + .prefetch_related("assignees", "labels") + .distinct() + ) + + for issue in issues: + assignee = issue.assignees.first() + if not assignee: + continue + + status = self._get_status(issue, assignee) + task, created = Task.objects.get_or_create( + issue=issue, + assignee=assignee, + defaults={"module": module, "status": status}, + ) + + updates = {} + if task.module != module: + updates["module"] = module + if task.status != status: + updates["status"] = status + + # Only fetch assigned_at when needed. + if (created or task.assigned_at is None) and issue.repository: + repo_full_name = self._extract_repo_full_name(issue.repository) + if repo_full_name: + if repo_full_name not in repo_cache: + try: + repo_cache[repo_full_name] = gh_client.get_repo(repo_full_name) + except GithubException as e: + self.stderr.write( + self.style.ERROR( + f"Failed to fetch repo '{repo_full_name}': {e}" + ) + ) + repo_cache[repo_full_name] = None + repo = repo_cache.get(repo_full_name) + if repo: + assigned_date = self._get_last_assigned_date( + repo=repo, + issue_number=issue.number, + assignee_login=assignee.login, + ) + if assigned_date: + updates["assigned_at"] = assigned_date + + if created: + num_tasks_created += 1 + self.stdout.write( + self.style.SUCCESS( + f"Task created for user '{assignee.login}' on issue " + f"{issue.repository.name}#{issue.number} " + f"in module '{module.name}'" + ) + ) + + if updates: + for field, value in updates.items(): + setattr(task, field, value) + task.save(update_fields=list(updates.keys())) + + num_linked = len(matched_issue_ids) + if num_linked > 0: + repo_names = ", ".join([r.name for r in project_repos]) + log_message = ( + f"Updated module '{module.name}': set {num_linked} issues from " + f"repos: [{repo_names}]" + ) + if num_tasks_created > 0: + log_message += f" and created {num_tasks_created} tasks." + + self.stdout.write(self.style.SUCCESS(log_message)) + + if verbosity > 1 and num_tasks_created > 0: + self.stdout.write(self.style.SUCCESS(f" - Created {num_tasks_created} tasks.")) + return num_linked + + def handle(self, *_args, **options): + self.stdout.write("starting...") + verbosity = options["verbosity"] + gh_client = get_github_client() + repo_cache = {} + + repo_label_to_issue_ids = self._build_repo_label_to_issue_map() + + total_links_created = 0 + total_modules_updated = 0 + + self.stdout.write("Processing modules and linking issues...") + modules_to_process = ( + Module.objects.prefetch_related("project__repositories") + .exclude(project__repositories__isnull=True) + .exclude(labels__isnull=True) + .exclude(labels=[]) + ) + + for module in modules_to_process: + links_created = self._process_module( + module=module, + repo_label_to_issue_ids=repo_label_to_issue_ids, + gh_client=gh_client, + repo_cache=repo_cache, + verbosity=verbosity, + ) + if links_created > 0: + total_links_created += links_created + total_modules_updated += 1 + + self.stdout.write( + self.style.SUCCESS( + f"Completed. {total_links_created} issue links set " + f"across {total_modules_updated} modules." + ) + ) diff --git a/backend/apps/mentorship/migrations/0005_remove_task_level_module_issues_and_more.py b/backend/apps/mentorship/migrations/0005_remove_task_level_module_issues_and_more.py new file mode 100644 index 0000000000..92fd8e1acf --- /dev/null +++ b/backend/apps/mentorship/migrations/0005_remove_task_level_module_issues_and_more.py @@ -0,0 +1,37 @@ +# Generated by Django 5.2.5 on 2025-09-25 08:53 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("github", "0037_issue_level"), + ("mentorship", "0004_module_key_program_key_and_more"), + ] + + operations = [ + migrations.RemoveField( + model_name="task", + name="level", + ), + migrations.AddField( + model_name="module", + name="issues", + field=models.ManyToManyField( + blank=True, + help_text="Issues linked to this module via label matching.", + related_name="mentorship_modules", + to="github.issue", + verbose_name="Linked Issues", + ), + ), + migrations.AlterField( + model_name="task", + name="assigned_at", + field=models.DateTimeField( + blank=True, + help_text="Timestamp when the task was assigned to the mentee.", + null=True, + ), + ), + ] diff --git a/backend/apps/mentorship/models/module.py b/backend/apps/mentorship/models/module.py index 7309baca73..c9e13d2287 100644 --- a/backend/apps/mentorship/models/module.py +++ b/backend/apps/mentorship/models/module.py @@ -65,6 +65,14 @@ class Meta: ) # M2Ms. + issues = models.ManyToManyField( + "github.Issue", + verbose_name="Linked Issues", + related_name="mentorship_modules", + blank=True, + help_text="Issues linked to this module via label matching.", + ) + mentors = models.ManyToManyField( "mentorship.Mentor", verbose_name="Mentors", diff --git a/backend/apps/mentorship/models/task.py b/backend/apps/mentorship/models/task.py index a572b76bde..4160865310 100644 --- a/backend/apps/mentorship/models/task.py +++ b/backend/apps/mentorship/models/task.py @@ -25,7 +25,8 @@ class Status(models.TextChoices): COMPLETED = "COMPLETED", "Completed" assigned_at = models.DateTimeField( - auto_now_add=True, + null=True, + blank=True, help_text="Timestamp when the task was assigned to the mentee.", ) @@ -63,15 +64,6 @@ class Status(models.TextChoices): help_text="The GitHub issue this task corresponds to.", ) - level = models.ForeignKey( - "mentorship.TaskLevel", - null=True, - blank=True, - on_delete=models.SET_NULL, - related_name="tasks", - help_text="The difficulty level of this task.", - ) - module = models.ForeignKey( "mentorship.Module", on_delete=models.CASCADE, diff --git a/backend/apps/mentorship/utils.py b/backend/apps/mentorship/utils.py new file mode 100644 index 0000000000..8432cb97c2 --- /dev/null +++ b/backend/apps/mentorship/utils.py @@ -0,0 +1,6 @@ +"""Utility functions for the mentorship app.""" + + +def normalize_name(name): + """Normalize a string by stripping whitespace and converting to lowercase.""" + return (name or "").strip().casefold() diff --git a/docker-compose/local.yaml b/docker-compose/local.yaml index f292929754..32ac1913e5 100644 --- a/docker-compose/local.yaml +++ b/docker-compose/local.yaml @@ -29,7 +29,7 @@ services: - 8000:8000 volumes: - ../backend:/home/owasp - - backend-venv:/home/owasp/.venv + - backend-venv-mentorship:/home/owasp/.venv cache: command: > @@ -48,7 +48,7 @@ services: networks: - nest-network volumes: - - cache-data:/data + - cache-data-mentorship:/data db: container_name: nest-db @@ -65,7 +65,7 @@ services: networks: - nest-network volumes: - - db-data:/var/lib/postgresql/data + - db-data-mentorship:/var/lib/postgresql/data docs: container_name: nest-docs @@ -82,7 +82,7 @@ services: - 8001:8001 volumes: - ../docs:/home/owasp/docs - - docs-venv:/home/owasp/.venv + - docs-venv-mentorship:/home/owasp/.venv frontend: container_name: nest-frontend @@ -104,16 +104,16 @@ services: - 3000:3000 volumes: - ../frontend:/home/owasp - - frontend-next:/home/owasp/.next - - frontend-node-modules:/home/owasp/node_modules + - frontend-next-mentorship:/home/owasp/.next + - frontend-node-modules-mentorship:/home/owasp/node_modules networks: nest-network: volumes: - backend-venv: - cache-data: - db-data: - docs-venv: - frontend-next: - frontend-node-modules: + backend-venv-mentorship: + cache-data-mentorship: + db-data-mentorship: + docs-venv-mentorship: + frontend-next-mentorship: + frontend-node-modules-mentorship: