From d8d52eedbf6508d219fbabbfb0e9f36bc6ef0160 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Mon, 9 Sep 2024 12:00:54 -0300 Subject: [PATCH] feat: email ingestor api test endpoint (#7915) * feat: email ingestor api test endpoint * ci: add ingestion test token for sandbox * chore: fix comments --- dev/deploy-to-container/settings_local.py | 6 ++ ietf/api/tests.py | 97 ++++++++++++++++++++++- ietf/api/urls.py | 2 + ietf/api/views.py | 41 ++++++++-- 4 files changed, 138 insertions(+), 8 deletions(-) diff --git a/dev/deploy-to-container/settings_local.py b/dev/deploy-to-container/settings_local.py index 25eacc3004..ae698e20b6 100644 --- a/dev/deploy-to-container/settings_local.py +++ b/dev/deploy-to-container/settings_local.py @@ -71,5 +71,11 @@ DE_GFM_BINARY = '/usr/local/bin/de-gfm' +# No real secrets here, these are public testing values _only_ +APP_API_TOKENS = { + "ietf.api.views.ingest_email_test": ["ingestion-test-token"] +} + + # OIDC configuration SITE_URL = 'https://__HOSTNAME__' diff --git a/ietf/api/tests.py b/ietf/api/tests.py index 20c3e2cb44..4f2a7f7d3c 100644 --- a/ietf/api/tests.py +++ b/ietf/api/tests.py @@ -1022,7 +1022,9 @@ def test_role_holder_addresses(self): sorted(e.address for e in emails), ) - @override_settings(APP_API_TOKENS={"ietf.api.views.ingest_email": "valid-token"}) + @override_settings( + APP_API_TOKENS={"ietf.api.views.ingest_email": "valid-token", "ietf.api.views.ingest_email_test": "test-token"} + ) @mock.patch("ietf.api.views.iana_ingest_review_email") @mock.patch("ietf.api.views.ipr_ingest_response_email") @mock.patch("ietf.api.views.nomcom_ingest_feedback_email") @@ -1032,29 +1034,47 @@ def test_ingest_email( mocks = {mock_nomcom_ingest, mock_ipr_ingest, mock_iana_ingest} empty_outbox() url = urlreverse("ietf.api.views.ingest_email") + test_mode_url = urlreverse("ietf.api.views.ingest_email_test") # test various bad calls r = self.client.get(url) self.assertEqual(r.status_code, 403) self.assertFalse(any(m.called for m in mocks)) + r = self.client.get(test_mode_url) + self.assertEqual(r.status_code, 403) + self.assertFalse(any(m.called for m in mocks)) r = self.client.post(url) self.assertEqual(r.status_code, 403) self.assertFalse(any(m.called for m in mocks)) + r = self.client.post(test_mode_url) + self.assertEqual(r.status_code, 403) + self.assertFalse(any(m.called for m in mocks)) r = self.client.get(url, headers={"X-Api-Key": "valid-token"}) self.assertEqual(r.status_code, 405) self.assertFalse(any(m.called for m in mocks)) + r = self.client.get(test_mode_url, headers={"X-Api-Key": "test-token"}) + self.assertEqual(r.status_code, 405) + self.assertFalse(any(m.called for m in mocks)) r = self.client.post(url, headers={"X-Api-Key": "valid-token"}) self.assertEqual(r.status_code, 415) self.assertFalse(any(m.called for m in mocks)) + r = self.client.post(test_mode_url, headers={"X-Api-Key": "test-token"}) + self.assertEqual(r.status_code, 415) + self.assertFalse(any(m.called for m in mocks)) r = self.client.post( url, content_type="application/json", headers={"X-Api-Key": "valid-token"} ) self.assertEqual(r.status_code, 400) self.assertFalse(any(m.called for m in mocks)) + r = self.client.post( + test_mode_url, content_type="application/json", headers={"X-Api-Key": "test-token"} + ) + self.assertEqual(r.status_code, 400) + self.assertFalse(any(m.called for m in mocks)) r = self.client.post( url, @@ -1064,6 +1084,14 @@ def test_ingest_email( ) self.assertEqual(r.status_code, 400) self.assertFalse(any(m.called for m in mocks)) + r = self.client.post( + test_mode_url, + "this is not JSON!", + content_type="application/json", + headers={"X-Api-Key": "test-token"}, + ) + self.assertEqual(r.status_code, 400) + self.assertFalse(any(m.called for m in mocks)) r = self.client.post( url, @@ -1073,6 +1101,14 @@ def test_ingest_email( ) self.assertEqual(r.status_code, 400) self.assertFalse(any(m.called for m in mocks)) + r = self.client.post( + test_mode_url, + {"json": "yes", "valid_schema": False}, + content_type="application/json", + headers={"X-Api-Key": "test-token"}, + ) + self.assertEqual(r.status_code, 400) + self.assertFalse(any(m.called for m in mocks)) # bad destination message_b64 = base64.b64encode(b"This is a message").decode() @@ -1086,6 +1122,16 @@ def test_ingest_email( self.assertEqual(r.headers["Content-Type"], "application/json") self.assertEqual(json.loads(r.content), {"result": "bad_dest"}) self.assertFalse(any(m.called for m in mocks)) + r = self.client.post( + test_mode_url, + {"dest": "not-a-destination", "message": message_b64}, + content_type="application/json", + headers={"X-Api-Key": "test-token"}, + ) + self.assertEqual(r.status_code, 200) + self.assertEqual(r.headers["Content-Type"], "application/json") + self.assertEqual(json.loads(r.content), {"result": "bad_dest"}) + self.assertFalse(any(m.called for m in mocks)) # test that valid requests call handlers appropriately r = self.client.post( @@ -1102,6 +1148,19 @@ def test_ingest_email( self.assertFalse(any(m.called for m in (mocks - {mock_iana_ingest}))) mock_iana_ingest.reset_mock() + # the test mode endpoint should _not_ call the handler + r = self.client.post( + test_mode_url, + {"dest": "iana-review", "message": message_b64}, + content_type="application/json", + headers={"X-Api-Key": "test-token"}, + ) + self.assertEqual(r.status_code, 200) + self.assertEqual(r.headers["Content-Type"], "application/json") + self.assertEqual(json.loads(r.content), {"result": "ok"}) + self.assertFalse(any(m.called for m in mocks)) + mock_iana_ingest.reset_mock() + r = self.client.post( url, {"dest": "ipr-response", "message": message_b64}, @@ -1116,6 +1175,19 @@ def test_ingest_email( self.assertFalse(any(m.called for m in (mocks - {mock_ipr_ingest}))) mock_ipr_ingest.reset_mock() + # the test mode endpoint should _not_ call the handler + r = self.client.post( + test_mode_url, + {"dest": "ipr-response", "message": message_b64}, + content_type="application/json", + headers={"X-Api-Key": "test-token"}, + ) + self.assertEqual(r.status_code, 200) + self.assertEqual(r.headers["Content-Type"], "application/json") + self.assertEqual(json.loads(r.content), {"result": "ok"}) + self.assertFalse(any(m.called for m in mocks)) + mock_ipr_ingest.reset_mock() + # bad nomcom-feedback dest for bad_nomcom_dest in [ "nomcom-feedback", # no suffix @@ -1133,6 +1205,16 @@ def test_ingest_email( self.assertEqual(r.headers["Content-Type"], "application/json") self.assertEqual(json.loads(r.content), {"result": "bad_dest"}) self.assertFalse(any(m.called for m in mocks)) + r = self.client.post( + test_mode_url, + {"dest": bad_nomcom_dest, "message": message_b64}, + content_type="application/json", + headers={"X-Api-Key": "test-token"}, + ) + self.assertEqual(r.status_code, 200) + self.assertEqual(r.headers["Content-Type"], "application/json") + self.assertEqual(json.loads(r.content), {"result": "bad_dest"}) + self.assertFalse(any(m.called for m in mocks)) # good nomcom-feedback dest random_year = randrange(100000) @@ -1150,6 +1232,19 @@ def test_ingest_email( self.assertFalse(any(m.called for m in (mocks - {mock_nomcom_ingest}))) mock_nomcom_ingest.reset_mock() + # the test mode endpoint should _not_ call the handler + r = self.client.post( + test_mode_url, + {"dest": f"nomcom-feedback-{random_year}", "message": message_b64}, + content_type="application/json", + headers={"X-Api-Key": "test-token"}, + ) + self.assertEqual(r.status_code, 200) + self.assertEqual(r.headers["Content-Type"], "application/json") + self.assertEqual(json.loads(r.content), {"result": "ok"}) + self.assertFalse(any(m.called for m in mocks)) + mock_nomcom_ingest.reset_mock() + # test that exceptions lead to email being sent - assumes that iana-review handling is representative mock_iana_ingest.side_effect = EmailIngestionError("Error: don't send email") r = self.client.post( diff --git a/ietf/api/urls.py b/ietf/api/urls.py index 3c0fb872c9..396b3813d6 100644 --- a/ietf/api/urls.py +++ b/ietf/api/urls.py @@ -27,6 +27,8 @@ url(r'^doc/draft-aliases/$', api_views.draft_aliases), # email ingestor url(r'email/$', api_views.ingest_email), + # email ingestor + url(r'email/test/$', api_views.ingest_email_test), # GDPR: export of personal information for the logged-in person url(r'^export/personal-information/$', api_views.PersonalInformationExportView.as_view()), # Email alias information for groups diff --git a/ietf/api/views.py b/ietf/api/views.py index 62857bff54..f8662f9a0e 100644 --- a/ietf/api/views.py +++ b/ietf/api/views.py @@ -614,14 +614,16 @@ def as_emailmessage(self) -> Optional[EmailMessage]: return msg -@requires_api_token -@csrf_exempt -def ingest_email(request): - """Ingest incoming email +def ingest_email_handler(request, test_mode=False): + """Ingest incoming email - handler Returns a 4xx or 5xx status code if the HTTP request was invalid or something went wrong while processing it. If the request was valid, returns a 200. This may or may not indicate that the message was accepted. + + If test_mode is true, actual processing of a valid message will be skipped. In this + mode, a valid request with a valid destination will be treated as accepted. The + "bad_dest" error may still be returned. """ def _http_err(code, text): @@ -657,15 +659,18 @@ def _api_response(result): try: if dest == "iana-review": valid_dest = True - iana_ingest_review_email(message) + if not test_mode: + iana_ingest_review_email(message) elif dest == "ipr-response": valid_dest = True - ipr_ingest_response_email(message) + if not test_mode: + ipr_ingest_response_email(message) elif dest.startswith("nomcom-feedback-"): maybe_year = dest[len("nomcom-feedback-"):] if maybe_year.isdecimal(): valid_dest = True - nomcom_ingest_feedback_email(message, int(maybe_year)) + if not test_mode: + nomcom_ingest_feedback_email(message, int(maybe_year)) except EmailIngestionError as err: error_email = err.as_emailmessage() if error_email is not None: @@ -677,3 +682,25 @@ def _api_response(result): return _api_response("bad_dest") return _api_response("ok") + + +@requires_api_token +@csrf_exempt +def ingest_email(request): + """Ingest incoming email + + Hands off to ingest_email_handler() with test_mode=False. This allows @requires_api_token to + give the test endpoint a distinct token from the real one. + """ + return ingest_email_handler(request, test_mode=False) + + +@requires_api_token +@csrf_exempt +def ingest_email_test(request): + """Ingest incoming email test endpoint + + Hands off to ingest_email_handler() with test_mode=True. This allows @requires_api_token to + give the test endpoint a distinct token from the real one. + """ + return ingest_email_handler(request, test_mode=True)