feat: API to replace email alias generation commands (#7012)

* feat: DraftAliasGenerator class

Encapsulates logic from generate_draft_aliases.py

* refactor: Avoid circular imports

* feat: Add draft_aliases API endpoint

* feat: Add @requires_api_token decorator

Stolen from feat/rpc-api

* feat: Add token auth to draft_aliases endpoint

* feat: draft-aliases-from-json.py script

Parses output from the draft_aliases API call

* chore: Remove unused cruft

* refactor: Avoid shadowing "draft" name

* fix: Suppress empty lists from DraftAliasGenerator

* refactor: Use a GET instead of POST

* feat: GroupAliasGenerator class

* feat: group aliases API view

* fix: Handle domains array correctly

* fix: Suppress empty group aliases

* refactor: Generalize aliases-from-json.py script

* refactor: Same output fmt for draft and group alias apis

* feat: Sort addresses for stability

* fix: Add "anything" virtual alias

* test: Test requires_api_token decorator

* feat: Harden is_valid_token against misconfig

* test: Test is_valid_token

* test: Test draft_aliases view

* test: Test group_aliases view

* test: Test DraftAliasGenerator

* fix: ise group is type "ise" in test data

* test: Fix logic in testManagementCommand

The test was incorrect - and fails when fixed. :-(

* test: Test GroupAliasGenerator

Test currently fails

* fix: Suppress empty -ads alias

* test: Fix group acronym copy/paste error

I *think* this must be what had been intended. The
code does not look like it ever dealt with GroupHistory,
so I'm pretty sure it wasn't meant to have the same
acronym used by two different Groups at different
times.

* test: Check draft .notify alias generation

* test: Cover get_draft_notify_emails()

ietf/api/ietf_utils.py

@@ -2,14 +2,75 @@
 
 # This is not utils.py because Tastypie implicitly consumes ietf.api.utils.
 # See ietf.api.__init__.py for details.
+from functools import wraps
+from typing import Callable, Optional, Union
 
 from django.conf import settings
+from django.http import HttpResponseForbidden
+
 
 def is_valid_token(endpoint, token):
     # This is where we would consider integration with vault
     # Settings implementation for now.
     if hasattr(settings, "APP_API_TOKENS"):
         token_store = settings.APP_API_TOKENS
-        if endpoint in token_store and token in token_store[endpoint]:
-            return True
+        if endpoint in token_store:
+            endpoint_tokens = token_store[endpoint]
+            # Be sure endpoints is a list or tuple so we don't accidentally use substring matching!
+            if not isinstance(endpoint_tokens, (list, tuple)):
+                endpoint_tokens = [endpoint_tokens]
+            if token in endpoint_tokens:
+                return True
     return False
+
+
+def requires_api_token(func_or_endpoint: Optional[Union[Callable, str]] = None):
+    """Validate API token before executing the wrapped method
+
+    Usage:
+        * Basic: endpoint defaults to the qualified name of the wrapped method. E.g., in ietf.api.views,
+
+                @requires_api_token
+                def my_view(request):
+                    ...
+
+            will require a token for "ietf.api.views.my_view"
+
+        * Custom endpoint: specify the endpoint explicitly
+
+                @requires_api_token("ietf.api.views.some_other_thing")
+                def my_view(request):
+                    ...
+
+            will require a token for "ietf.api.views.some_other_thing"
+    """
+
+    def decorate(f):
+        if _endpoint is None:
+            fname = getattr(f, "__qualname__", None)
+            if fname is None:
+                raise TypeError(
+                    "Cannot automatically decorate function that does not support __qualname__. "
+                    "Explicitly set the endpoint."
+                )
+            endpoint = "{}.{}".format(f.__module__, fname)
+        else:
+            endpoint = _endpoint
+
+        @wraps(f)
+        def wrapped(request, *args, **kwargs):
+            authtoken = request.META.get("HTTP_X_API_KEY", None)
+            if authtoken is None or not is_valid_token(endpoint, authtoken):
+                return HttpResponseForbidden()
+            return f(request, *args, **kwargs)
+
+        return wrapped
+
+    # Magic to allow decorator to be used with or without parentheses
+    if callable(func_or_endpoint):
+        func = func_or_endpoint
+        _endpoint = None
+        return decorate(func)
+    else:
+        _endpoint = func_or_endpoint
+        return decorate

ietf/api/tests.py

@@ -4,6 +4,7 @@
 import datetime
 import json
 import html
+import mock
 import os
 import sys
 
@@ -12,7 +13,8 @@
 
 from django.apps import apps
 from django.conf import settings
-from django.test import Client
+from django.http import HttpResponseForbidden
+from django.test import Client, RequestFactory
 from django.test.utils import override_settings
 from django.urls import reverse as urlreverse
 from django.utils import timezone
@@ -38,6 +40,8 @@
 from ietf.utils.models import DumpInfo
 from ietf.utils.test_utils import TestCase, login_testing_unauthorized, reload_db_objects
 
+from .ietf_utils import is_valid_token, requires_api_token
+
 OMITTED_APPS = (
     'ietf.secr.meetings',
     'ietf.secr.proceedings',
@@ -780,7 +784,74 @@ def test_api_get_session_matherials_no_agenda_meeting_url(self):
         url = urlreverse('ietf.meeting.views.api_get_session_materials', kwargs={'session_id': session.pk})
         r = self.client.get(url)
         self.assertEqual(r.status_code, 200)
+
+    @override_settings(APP_API_TOKENS={"ietf.api.views.email_aliases": ["valid-token"]})
+    @mock.patch("ietf.api.views.DraftAliasGenerator")
+    def test_draft_aliases(self, mock):
+        mock.return_value = (("alias1", ("a1", "a2")), ("alias2", ("a3", "a4")))
+        url = urlreverse("ietf.api.views.draft_aliases")
+        r = self.client.get(url, headers={"X-Api-Key": "valid-token"})
+        self.assertEqual(r.status_code, 200)
+        self.assertEqual(r.headers["Content-type"], "application/json")
+        self.assertEqual(
+            json.loads(r.content),
+            {
+                "aliases": [
+                    {"alias": "alias1", "domains": ["ietf"], "addresses": ["a1", "a2"]},
+                    {"alias": "alias2", "domains": ["ietf"], "addresses": ["a3", "a4"]},
+                ]}
+        )
+        # some invalid cases
+        self.assertEqual(
+            self.client.get(url, headers={}).status_code,
+            403,
+        )
+        self.assertEqual(
+            self.client.get(url, headers={"X-Api-Key": "something-else"}).status_code,
+            403,
+        )
+        self.assertEqual(
+            self.client.post(url, headers={"X-Api-Key": "something-else"}).status_code,
+            403,
+        )
+        self.assertEqual(
+            self.client.post(url, headers={"X-Api-Key": "valid-token"}).status_code,
+            405,
+        )
 
+    @override_settings(APP_API_TOKENS={"ietf.api.views.email_aliases": ["valid-token"]})
+    @mock.patch("ietf.api.views.GroupAliasGenerator")
+    def test_group_aliases(self, mock):
+        mock.return_value = (("alias1", ("ietf",), ("a1", "a2")), ("alias2", ("ietf", "iab"), ("a3", "a4")))
+        url = urlreverse("ietf.api.views.group_aliases")
+        r = self.client.get(url, headers={"X-Api-Key": "valid-token"})
+        self.assertEqual(r.status_code, 200)
+        self.assertEqual(r.headers["Content-type"], "application/json")
+        self.assertEqual(
+            json.loads(r.content),
+            {
+                "aliases": [
+                    {"alias": "alias1", "domains": ["ietf"], "addresses": ["a1", "a2"]},
+                    {"alias": "alias2", "domains": ["ietf", "iab"], "addresses": ["a3", "a4"]},
+                ]}
+        )
+        # some invalid cases
+        self.assertEqual(
+            self.client.get(url, headers={}).status_code,
+            403,
+        )
+        self.assertEqual(
+            self.client.get(url, headers={"X-Api-Key": "something-else"}).status_code,
+            403,
+        )
+        self.assertEqual(
+            self.client.post(url, headers={"X-Api-Key": "something-else"}).status_code,
+            403,
+        )
+        self.assertEqual(
+            self.client.post(url, headers={"X-Api-Key": "valid-token"}).status_code,
+            405,
+        )
 
 
 class DirectAuthApiTests(TestCase):
@@ -1133,3 +1204,85 @@ def test_no_such_document(self):
             url = urlreverse(self.target_view, kwargs={'name': name})
             r = self.client.get(url)
             self.assertEqual(r.status_code, 404)
+
+
+class TokenTests(TestCase):
+    @override_settings(APP_API_TOKENS={"known.endpoint": ["token in a list"], "oops": "token as a str"})
+    def test_is_valid_token(self):
+        # various invalid cases
+        self.assertFalse(is_valid_token("unknown.endpoint", "token in a list"))
+        self.assertFalse(is_valid_token("known.endpoint", "token"))
+        self.assertFalse(is_valid_token("known.endpoint", "token as a str"))
+        self.assertFalse(is_valid_token("oops", "token"))
+        self.assertFalse(is_valid_token("oops", "token in a list"))
+        # the only valid cases
+        self.assertTrue(is_valid_token("known.endpoint", "token in a list"))
+        self.assertTrue(is_valid_token("oops", "token as a str"))
+
+    @mock.patch("ietf.api.ietf_utils.is_valid_token")
+    def test_requires_api_token(self, mock_is_valid_token):
+        called = False
+
+        @requires_api_token
+        def fn_to_wrap(request, *args, **kwargs):
+            nonlocal called
+            called = True
+            return request, args, kwargs
+
+        req_factory = RequestFactory()
+        arg = object()
+        kwarg = object()
+
+        # No X-Api-Key header
+        mock_is_valid_token.return_value = False
+        val = fn_to_wrap(
+            req_factory.get("/some/url", headers={}),
+            arg,
+            kwarg=kwarg,
+        )
+        self.assertTrue(isinstance(val, HttpResponseForbidden))
+        self.assertFalse(mock_is_valid_token.called)
+        self.assertFalse(called)
+
+        # Bad X-Api-Key header (not resetting the mock, it was not used yet)
+        val = fn_to_wrap(
+            req_factory.get("/some/url", headers={"X-Api-Key": "some-value"}),
+            arg, 
+            kwarg=kwarg,
+        )
+        self.assertTrue(isinstance(val, HttpResponseForbidden))
+        self.assertTrue(mock_is_valid_token.called)
+        self.assertEqual(
+            mock_is_valid_token.call_args[0], 
+            (fn_to_wrap.__module__ + "." + fn_to_wrap.__qualname__, "some-value"),
+        )
+        self.assertFalse(called)
+
+        # Valid header
+        mock_is_valid_token.reset_mock()
+        mock_is_valid_token.return_value = True
+        request = req_factory.get("/some/url", headers={"X-Api-Key": "some-value"}) 
+        # Bad X-Api-Key header (not resetting the mock, it was not used yet)
+        val = fn_to_wrap(
+            request,
+            arg, 
+            kwarg=kwarg,
+        )
+        self.assertEqual(val, (request, (arg,), {"kwarg": kwarg}))
+        self.assertTrue(mock_is_valid_token.called)
+        self.assertEqual(
+            mock_is_valid_token.call_args[0], 
+            (fn_to_wrap.__module__ + "." + fn_to_wrap.__qualname__, "some-value"),
+        )
+        self.assertTrue(called)
+
+        # Test the endpoint setting
+        @requires_api_token("endpoint")
+        def another_fn_to_wrap(request):
+            return "yep"
+
+        val = another_fn_to_wrap(request)
+        self.assertEqual(
+            mock_is_valid_token.call_args[0], 
+            ("endpoint", "some-value"),
+        )

ietf/api/urls.py

@@ -22,8 +22,12 @@
     url(r'^v2/person/person', api_views.ApiV2PersonExportView.as_view()),
     #
     # --- Custom API endpoints, sorted alphabetically ---
+    # Email alias information for drafts
+    url(r'^doc/draft-aliases/$', api_views.draft_aliases),
     # GPRD: export of personal information for the logged-in person
     url(r'^export/personal-information/$', api_views.PersonalInformationExportView.as_view()),
+    # Email alias information for groups
+    url(r'^group/group-aliases/$', api_views.group_aliases),
     # Let IESG members set positions programmatically
     url(r'^iesg/position', views_ballot.api_set_position),
     # Let Meetecho set session video URLs

ietf/api/views.py

@@ -2,42 +2,39 @@
 # -*- coding: utf-8 -*-
 
 import json
-import pytz
 import re
 
-from jwcrypto.jwk import JWK
-
+import pytz
 from django.conf import settings
 from django.contrib.auth import authenticate
 from django.contrib.auth.decorators import login_required
 from django.contrib.auth.models import User
 from django.core.exceptions import ValidationError
 from django.core.validators import validate_email
-from django.http import HttpResponse, Http404
+from django.http import HttpResponse, Http404, JsonResponse
 from django.shortcuts import render, get_object_or_404
 from django.urls import reverse
 from django.utils.decorators import method_decorator
 from django.views.decorators.csrf import csrf_exempt
 from django.views.decorators.gzip import gzip_page
 from django.views.generic.detail import DetailView
-
+from jwcrypto.jwk import JWK
 from tastypie.exceptions import BadRequest
-from tastypie.utils.mime import determine_format, build_content_type
-from tastypie.utils import is_valid_jsonp_callback_value
 from tastypie.serializers import Serializer
-
-import debug                            # pyflakes:ignore
+from tastypie.utils import is_valid_jsonp_callback_value
+from tastypie.utils.mime import determine_format, build_content_type
 
 import ietf
-from ietf.person.models import Person, Email
 from ietf.api import _api_list
+from ietf.api.ietf_utils import is_valid_token, requires_api_token
 from ietf.api.serializer import JsonExportMixin
-from ietf.api.ietf_utils import is_valid_token
-from ietf.doc.utils import fuzzy_find_documents
-from ietf.ietfauth.views import send_account_creation_email
+from ietf.doc.utils import DraftAliasGenerator, fuzzy_find_documents
+from ietf.group.utils import GroupAliasGenerator
 from ietf.ietfauth.utils import role_required
+from ietf.ietfauth.views import send_account_creation_email
 from ietf.meeting.models import Meeting
 from ietf.nomcom.models import Volunteer, NomCom
+from ietf.person.models import Person, Email
 from ietf.stats.models import MeetingRegistration
 from ietf.utils import log
 from ietf.utils.decorators import require_api_key
@@ -453,3 +450,41 @@ def directauth(request):
 
     else:
         return HttpResponse(status=405)
+
+
+@requires_api_token("ietf.api.views.email_aliases")
+@csrf_exempt
+def draft_aliases(request):
+    if request.method == "GET":
+        return JsonResponse(
+            {
+                "aliases": [
+                    {
+                        "alias": alias,
+                        "domains": ["ietf"],
+                        "addresses": address_list,
+                    }
+                    for alias, address_list in DraftAliasGenerator()
+                ]
+            }
+        )
+    return HttpResponse(status=405)
+
+
+@requires_api_token("ietf.api.views.email_aliases")
+@csrf_exempt
+def group_aliases(request):
+    if request.method == "GET":
+        return JsonResponse(
+            {
+                "aliases": [
+                    {
+                        "alias": alias,
+                        "domains": domains,
+                        "addresses": address_list,
+                    } 
+                    for alias, domains, address_list in GroupAliasGenerator()
+                ]
+            }
+        )
+    return HttpResponse(status=405)