From 1de41965bee819d85b5b76f6a2db5c2e5934a6b6 Mon Sep 17 00:00:00 2001 From: Russ Housley Date: Wed, 7 Aug 2024 10:18:13 -0400 Subject: [PATCH] fix: add celery tasks for idnits2 and bibxml file generation (#7204) * Add celery tasks for idnits2 and bibxml file generation * Update tests_tasks.py Fix typo: bad_vakue -> bad_value * Update utils.py Don't raise error inbibxml_for_all_drafts * chore: fixup merge * chore: more merge cleanup * chore: one last merge cleanup --------- Co-authored-by: Jennifer Richards --- ietf/doc/tasks.py | 2 + ietf/doc/tests_tasks.py | 119 ++++++++++++++++++++++------------------ ietf/doc/utils.py | 3 +- 3 files changed, 70 insertions(+), 54 deletions(-) diff --git a/ietf/doc/tasks.py b/ietf/doc/tasks.py index 209db035a4..f1de459dd8 100644 --- a/ietf/doc/tasks.py +++ b/ietf/doc/tasks.py @@ -105,6 +105,8 @@ def generate_draft_bibxml_files_task(days=7, process_all=False): If process_all is False (the default), processes only docs with new revisions in the last specified number of days. """ + if not process_all and days < 1: + raise ValueError("Must call with days >= 1 or process_all=True") ensure_draft_bibxml_path_exists() doc_events = NewRevisionDocEvent.objects.filter( type="new_revision", diff --git a/ietf/doc/tests_tasks.py b/ietf/doc/tests_tasks.py index 51a8556e69..b75f58656b 100644 --- a/ietf/doc/tests_tasks.py +++ b/ietf/doc/tests_tasks.py @@ -22,8 +22,6 @@ ) class TaskTests(TestCase): - settings_temp_path_overrides = TestCase.settings_temp_path_overrides + ["DERIVED_DIR"] - @mock.patch("ietf.doc.tasks.in_draft_expire_freeze") @mock.patch("ietf.doc.tasks.get_expired_drafts") @mock.patch("ietf.doc.tasks.expirable_drafts") @@ -63,8 +61,8 @@ def test_expire_ids_task( # test that an exception is raised in_draft_expire_freeze_mock.side_effect = RuntimeError - with self.assertRaises(RuntimeError): ( - expire_ids_task()) + with self.assertRaises(RuntimeError): + expire_ids_task() @mock.patch("ietf.doc.tasks.send_expire_warning_for_draft") @mock.patch("ietf.doc.tasks.get_soon_to_expire_drafts") @@ -98,37 +96,42 @@ def test_expire_last_calls_task(self, mock_get_expired, mock_expire): self.assertEqual(mock_expire.call_args_list[1], mock.call(docs[1])) self.assertEqual(mock_expire.call_args_list[2], mock.call(docs[2])) - @mock.patch("ietf.doc.tasks.generate_idnits2_rfc_status") - def test_generate_idnits2_rfc_status_task(self, mock_generate): + +class Idnits2SupportTests(TestCase): + settings_temp_path_overrides = TestCase.settings_temp_path_overrides + ['DERIVED_DIR'] + + @mock.patch("ietf.doc.tasks.generate_idnits2_rfcs_obsoleted") + def test_generate_idnits2_rfcs_obsoleted_task(self, mock_generate): mock_generate.return_value = "dåtå" - generate_idnits2_rfc_status_task() + generate_idnits2_rfcs_obsoleted_task() self.assertEqual(mock_generate.call_count, 1) self.assertEqual( "dåtå".encode("utf8"), - (Path(settings.DERIVED_DIR) / "idnits2-rfc-status").read_bytes(), + (Path(settings.DERIVED_DIR) / "idnits2-rfcs-obsoleted").read_bytes(), ) - - @mock.patch("ietf.doc.tasks.generate_idnits2_rfcs_obsoleted") - def test_generate_idnits2_rfcs_obsoleted_task(self, mock_generate): + + @mock.patch("ietf.doc.tasks.generate_idnits2_rfc_status") + def test_generate_idnits2_rfc_status_task(self, mock_generate): mock_generate.return_value = "dåtå" - generate_idnits2_rfcs_obsoleted_task() + generate_idnits2_rfc_status_task() self.assertEqual(mock_generate.call_count, 1) self.assertEqual( "dåtå".encode("utf8"), - (Path(settings.DERIVED_DIR) / "idnits2-rfcs-obsoleted").read_bytes(), + (Path(settings.DERIVED_DIR) / "idnits2-rfc-status").read_bytes(), ) - @mock.patch("ietf.doc.tasks.ensure_draft_bibxml_path_exists") - @mock.patch("ietf.doc.tasks.update_or_create_draft_bibxml_file") - def test_generate_draft_bibxml_files_task(self, mock_create, mock_ensure_path): + +class BIBXMLSupportTests(TestCase): + def setUp(self): + super().setUp() now = timezone.now() - very_old_event = NewRevisionDocEventFactory( + self.very_old_event = NewRevisionDocEventFactory( time=now - datetime.timedelta(days=1000), rev="17" ) - old_event = NewRevisionDocEventFactory( + self.old_event = NewRevisionDocEventFactory( time=now - datetime.timedelta(days=8), rev="03" ) - young_event = NewRevisionDocEventFactory( + self.young_event = NewRevisionDocEventFactory( time=now - datetime.timedelta(days=6), rev="06" ) # a couple that should always be ignored @@ -141,62 +144,72 @@ def test_generate_draft_bibxml_files_task(self, mock_create, mock_ensure_path): rev="09", doc__type_id="rfc", ) - # Get rid of the "00" events created by the factories -- they're just noise for this test NewRevisionDocEvent.objects.filter(rev="00").delete() - - # default args - look back 7 days - generate_draft_bibxml_files_task() - self.assertTrue(mock_ensure_path.called) - self.assertCountEqual( - mock_create.call_args_list, [mock.call(young_event.doc, young_event.rev)] - ) - mock_create.reset_mock() - mock_ensure_path.reset_mock() - - # shorter lookback - generate_draft_bibxml_files_task(days=5) - self.assertTrue(mock_ensure_path.called) - self.assertCountEqual(mock_create.call_args_list, []) - mock_create.reset_mock() - mock_ensure_path.reset_mock() - - # longer lookback - generate_draft_bibxml_files_task(days=9) + + @mock.patch("ietf.doc.tasks.ensure_draft_bibxml_path_exists") + @mock.patch("ietf.doc.tasks.update_or_create_draft_bibxml_file") + def test_generate_bibxml_files_for_all_drafts_task(self, mock_create, mock_ensure_path): + generate_draft_bibxml_files_task(process_all=True) self.assertTrue(mock_ensure_path.called) self.assertCountEqual( mock_create.call_args_list, [ - mock.call(young_event.doc, young_event.rev), - mock.call(old_event.doc, old_event.rev), + mock.call(self.young_event.doc, self.young_event.rev), + mock.call(self.old_event.doc, self.old_event.rev), + mock.call(self.very_old_event.doc, self.very_old_event.rev), ], ) mock_create.reset_mock() mock_ensure_path.reset_mock() - - # everything + + # everything should still be tried, even if there's an exception + mock_create.side_effect = RuntimeError generate_draft_bibxml_files_task(process_all=True) self.assertTrue(mock_ensure_path.called) self.assertCountEqual( mock_create.call_args_list, [ - mock.call(young_event.doc, young_event.rev), - mock.call(old_event.doc, old_event.rev), - mock.call(very_old_event.doc, very_old_event.rev), + mock.call(self.young_event.doc, self.young_event.rev), + mock.call(self.old_event.doc, self.old_event.rev), + mock.call(self.very_old_event.doc, self.very_old_event.rev), ], ) + + @mock.patch("ietf.doc.tasks.ensure_draft_bibxml_path_exists") + @mock.patch("ietf.doc.tasks.update_or_create_draft_bibxml_file") + def test_generate_bibxml_files_for_recent_drafts_task(self, mock_create, mock_ensure_path): + # default args - look back 7 days + generate_draft_bibxml_files_task() + self.assertTrue(mock_ensure_path.called) + self.assertCountEqual( + mock_create.call_args_list, [mock.call(self.young_event.doc, self.young_event.rev)] + ) mock_create.reset_mock() mock_ensure_path.reset_mock() - - # everything should still be tried, even if there's an exception - mock_create.side_effect = RuntimeError - generate_draft_bibxml_files_task(process_all=True) + + # shorter lookback + generate_draft_bibxml_files_task(days=5) + self.assertTrue(mock_ensure_path.called) + self.assertCountEqual(mock_create.call_args_list, []) + mock_create.reset_mock() + mock_ensure_path.reset_mock() + + # longer lookback + generate_draft_bibxml_files_task(days=9) self.assertTrue(mock_ensure_path.called) self.assertCountEqual( mock_create.call_args_list, [ - mock.call(young_event.doc, young_event.rev), - mock.call(old_event.doc, old_event.rev), - mock.call(very_old_event.doc, very_old_event.rev), + mock.call(self.young_event.doc, self.young_event.rev), + mock.call(self.old_event.doc, self.old_event.rev), ], ) + + @mock.patch("ietf.doc.tasks.ensure_draft_bibxml_path_exists") + @mock.patch("ietf.doc.tasks.update_or_create_draft_bibxml_file") + def test_generate_bibxml_files_for_recent_drafts_task_with_bad_value(self, mock_create, mock_ensure_path): + with self.assertRaises(ValueError): + generate_draft_bibxml_files_task(days=0) + self.assertFalse(mock_create.called) + self.assertFalse(mock_ensure_path.called) diff --git a/ietf/doc/utils.py b/ietf/doc/utils.py index cd0fbb43b0..a98b46cb50 100644 --- a/ietf/doc/utils.py +++ b/ietf/doc/utils.py @@ -1,4 +1,4 @@ -# Copyright The IETF Trust 2011-2020, All Rights Reserved +# Copyright The IETF Trust 2011-2024, All Rights Reserved # -*- coding: utf-8 -*- @@ -1228,6 +1228,7 @@ def fuzzy_find_documents(name, rev=None): FoundDocuments = namedtuple('FoundDocuments', 'documents matched_name matched_rev') return FoundDocuments(docs, name, rev) + def bibxml_for_draft(doc, rev=None): if rev is not None and rev != doc.rev: