feat: RFCs and subseries as explicit document objects

feat: RFCs and subseries as explicit document objects

.github/workflows/ci-run-tests.yml

@@ -4,6 +4,7 @@ on:
   pull_request:
     branches:
       - 'main'
+      - 'feat/rfc'
     paths:
       - 'client/**'
       - 'ietf/**'

dev/deploy-to-container/start.sh

@@ -36,7 +36,10 @@ echo "Running Datatracker checks..."
 # Migrate, adjusting to what the current state of the underlying database might be:
 
 echo "Running Datatracker migrations..."
-/usr/local/bin/python ./ietf/manage.py migrate --fake-initial --settings=settings_local
+/usr/local/bin/python ./ietf/manage.py migrate --settings=settings_local
+
+echo "Syncing with the rfc-index"
+./ietf/bin/rfc-editor-index-updates -d 1969-01-01
 
 echo "Starting Datatracker..."
 ./ietf/manage.py runserver 0.0.0.0:8000 --settings=settings_local

ietf/api/tests.py

@@ -24,7 +24,7 @@
 import ietf
 from ietf.doc.utils import get_unicode_document_content
 from ietf.doc.models import RelatedDocument, State
-from ietf.doc.factories import IndividualDraftFactory, WgDraftFactory
+from ietf.doc.factories import IndividualDraftFactory, WgDraftFactory, WgRfcFactory
 from ietf.group.factories import RoleFactory
 from ietf.meeting.factories import MeetingFactory, SessionFactory
 from ietf.meeting.models import Session
@@ -944,7 +944,7 @@ def do_draft_test(self, name):
         self.assertNotIn('previous', received, 'Rev 00 has no previous name when not replacing a draft')
 
         replaced = IndividualDraftFactory()
-        RelatedDocument.objects.create(relationship_id='replaces',source=draft,target=replaced.docalias.first())
+        RelatedDocument.objects.create(relationship_id='replaces',source=draft,target=replaced)
         received = self.getJson(dict(name=draft.name, rev='00'))
         self.assertEqual(received['previous'], f'{replaced.name}-{replaced.rev}',
                          'Rev 00 has a previous name when replacing a draft')
@@ -974,27 +974,27 @@ def test_draft_with_broken_history(self):
 
     def do_rfc_test(self, draft_name):
         draft = WgDraftFactory(name=draft_name, create_revisions=range(0,2))
-        draft.docalias.create(name=f'rfc{self.next_rfc_number():04}')
+        rfc = WgRfcFactory(group=draft.group, rfc_number=self.next_rfc_number())
+        draft.relateddocument_set.create(relationship_id="became_rfc", target=rfc)
         draft.set_state(State.objects.get(type_id='draft',slug='rfc'))
         draft.set_state(State.objects.get(type_id='draft-iesg', slug='pub'))
-        draft = reload_db_objects(draft)
-        rfc = draft
+        draft, rfc = reload_db_objects(draft, rfc)
 
-        number = rfc.rfc_number()
+        number = rfc.rfc_number
         received = self.getJson(dict(name=number))
         self.assertEqual(
             received,
             dict(
                 content_url=rfc.get_href(),
-                name=rfc.canonical_name(),
+                name=rfc.name,
                 previous=f'{draft.name}-{draft.rev}',
                 previous_url= draft.history_set.get(rev=draft.rev).get_href(),
             ),
             'Can look up an RFC by number',
         )
 
         num_received = received
-        received = self.getJson(dict(name=rfc.canonical_name()))
+        received = self.getJson(dict(name=rfc.name))
         self.assertEqual(num_received, received, 'RFC by canonical name gives same result as by number')
 
         received = self.getJson(dict(name=f'RfC {number}'))
@@ -1026,30 +1026,30 @@ def test_rfc(self):
 
     def test_rfc_with_tombstone(self):
         draft = WgDraftFactory(create_revisions=range(0,2))
-        draft.docalias.create(name='rfc3261') # See views_doc.HAS_TOMBSTONE
+        rfc = WgRfcFactory(rfc_number=3261,group=draft.group)# See views_doc.HAS_TOMBSTONE
+        draft.relateddocument_set.create(relationship_id="became_rfc", target=rfc)
         draft.set_state(State.objects.get(type_id='draft',slug='rfc'))
         draft.set_state(State.objects.get(type_id='draft-iesg', slug='pub'))
         draft = reload_db_objects(draft)
-        rfc = draft
 
         # Some old rfcs had tombstones that shouldn't be used for comparisons
-        received = self.getJson(dict(name=rfc.canonical_name()))
+        received = self.getJson(dict(name=rfc.name))
         self.assertTrue(received['previous'].endswith('00'))
 
     def do_rfc_with_broken_history_test(self, draft_name):
         draft = WgDraftFactory(rev='10', name=draft_name)
-        draft.docalias.create(name=f'rfc{self.next_rfc_number():04}')
+        rfc = WgRfcFactory(group=draft.group, rfc_number=self.next_rfc_number())
+        draft.relateddocument_set.create(relationship_id="became_rfc", target=rfc)
         draft.set_state(State.objects.get(type_id='draft',slug='rfc'))
         draft.set_state(State.objects.get(type_id='draft-iesg', slug='pub'))
         draft = reload_db_objects(draft)
-        rfc = draft
 
         received = self.getJson(dict(name=draft.name))
         self.assertEqual(
             received,
             dict(
                 content_url=rfc.get_href(),
-                name=rfc.canonical_name(),
+                name=rfc.name,
                 previous=f'{draft.name}-10',
                 previous_url= f'{settings.IETF_ID_ARCHIVE_URL}{draft.name}-10.txt',
             ),
@@ -1080,3 +1080,9 @@ def test_rfc_with_broken_history(self):
         # tricky draft names
         self.do_rfc_with_broken_history_test(draft_name='draft-gizmo-01')
         self.do_rfc_with_broken_history_test(draft_name='draft-oh-boy-what-a-draft-02-03')
+
+    def test_no_such_document(self):
+        for name in ['rfc0000', 'draft-ftei-oof-rab-00']:
+            url = urlreverse(self.target_view, kwargs={'name': name})
+            r = self.client.get(url)
+            self.assertEqual(r.status_code, 404)

ietf/api/views.py

@@ -317,45 +317,48 @@ def get_previous_url(name, rev=None):
     previous_url = ''
     if condition in ('historic version', 'current version'):
         doc = history if history else document
-        if found_rev:
-            doc.is_rfc = lambda: False
         previous_url = doc.get_href()
     elif condition == 'version dochistory not found':
         document.rev = found_rev
-        document.is_rfc = lambda: False
         previous_url = document.get_href()
     return previous_url
 
 
 def rfcdiff_latest_json(request, name, rev=None):
     response = dict()
     condition, document, history, found_rev = find_doc_for_rfcdiff(name, rev)
-
+    if document and document.type_id == "rfc":
+        draft = document.came_from_draft()
     if condition == 'no such document':
         raise Http404
     elif condition in ('historic version', 'current version'):
         doc = history if history else document
-        if not found_rev and doc.is_rfc():
-            response['content_url'] = doc.get_href()
-            response['name']=doc.canonical_name()
-            if doc.name != doc.canonical_name():
+        if doc.type_id == "rfc":
+                response['content_url'] = doc.get_href()
+                response['name']=doc.name
+                if draft:
+                    prev_rev = draft.rev
+                    if doc.rfc_number in HAS_TOMBSTONE and prev_rev != '00':
+                        prev_rev = f'{(int(draft.rev)-1):02d}'
+                    response['previous'] = f'{draft.name}-{prev_rev}'
+                    response['previous_url'] = get_previous_url(draft.name, prev_rev)            
+        elif doc.type_id == "draft" and not found_rev and doc.relateddocument_set.filter(relationship_id="became_rfc").exists():
+                rfc = doc.related_that_doc("became_rfc")[0]
+                response['content_url'] = rfc.get_href()
+                response['name']=rfc.name
                 prev_rev = doc.rev
-                # not sure what to do if non-numeric values come back, so at least log it
-                log.assertion('doc.rfc_number().isdigit()') # .rfc_number() is expensive...
-                log.assertion('doc.rev.isdigit()')
-                if int(doc.rfc_number()) in HAS_TOMBSTONE and prev_rev != '00':
+                if rfc.rfc_number in HAS_TOMBSTONE and prev_rev != '00':
                     prev_rev = f'{(int(doc.rev)-1):02d}'
                 response['previous'] = f'{doc.name}-{prev_rev}'
                 response['previous_url'] = get_previous_url(doc.name, prev_rev)
         else:
-            doc.is_rfc = lambda: False
             response['content_url'] = doc.get_href()
             response['rev'] = doc.rev
             response['name'] = doc.name
             if doc.rev == '00':
                 replaces_docs = (history.doc if condition=='historic version' else doc).related_that_doc('replaces')
                 if replaces_docs:
-                    replaces = replaces_docs[0].document
+                    replaces = replaces_docs[0]
                     response['previous'] = f'{replaces.name}-{replaces.rev}'
                     response['previous_url'] = get_previous_url(replaces.name, replaces.rev)
                 else:
@@ -374,7 +377,6 @@ def rfcdiff_latest_json(request, name, rev=None):
         response['name'] = document.name
         response['rev'] = found_rev
         document.rev = found_rev
-        document.is_rfc = lambda: False
         response['content_url'] = document.get_href()
         # not sure what to do if non-numeric values come back, so at least log it
         log.assertion('found_rev.isdigit()')

ietf/bin/rfc-editor-index-updates

@@ -79,12 +79,12 @@ if len(errata_data) < ietf.sync.rfceditor.MIN_ERRATA_RESULTS:
     sys.exit(1)
 
 new_rfcs = []
-for changes, doc, rfc_published in ietf.sync.rfceditor.update_docs_from_rfc_index(index_data, errata_data, skip_older_than_date=skip_date):
+for rfc_number, changes, doc, rfc_published in ietf.sync.rfceditor.update_docs_from_rfc_index(index_data, errata_data, skip_older_than_date=skip_date):
     if rfc_published:
         new_rfcs.append(doc)
 
     for c in changes:
-        log("RFC%s, %s: %s" % (doc.rfcnum, doc.name, c))
+        log("RFC%s, %s: %s" % (rfc_number, doc.name, c))
 
 sys.exit(0)
 
@@ -99,7 +99,7 @@ if newpid == 0:
         pipe("%s -a %s %s" % (settings.RSYNC_BINARY,settings.RFC_TEXT_RSYNC_SOURCE,settings.RFC_PATH))
         for rfc in new_rfcs:
             rebuild_reference_relations(rfc)
-            log("Updated references for %s"%rfc.canonical_name())
+            log("Updated references for %s"%rfc.name)
     except:
         subject = "Exception in updating references for new rfcs: %s : %s" % (sys.exc_info()[0],sys.exc_info()[1])
         msg = "%s\n%s\n----\n%s"%(sys.exc_info()[0],sys.exc_info()[1],traceback.format_tb(sys.exc_info()[2])) 

ietf/community/forms.py

@@ -30,6 +30,8 @@ def __init__(self, clist, rule_type, *args, **kwargs):
         super(SearchRuleForm, self).__init__(*args, **kwargs)
 
         def restrict_state(state_type, slug=None):
+            if "state" not in self.fields:
+                raise RuntimeError(f"Rule type {rule_type} cannot include state filtering")
             f = self.fields['state']
             f.queryset = f.queryset.filter(used=True).filter(type=state_type)
             if slug:
@@ -38,11 +40,15 @@ def restrict_state(state_type, slug=None):
                 f.initial = f.queryset[0].pk
                 f.widget = forms.HiddenInput()
 
+        if rule_type.endswith("_rfc"):
+            del self.fields["state"]  # rfc rules must not look at document states
+
         if rule_type in ["group", "group_rfc", "area", "area_rfc", "group_exp"]:
             if rule_type == "group_exp":
                 restrict_state("draft", "expired")
             else:
-                restrict_state("draft", "rfc" if rule_type.endswith("rfc") else "active")
+                if not rule_type.endswith("_rfc"):
+                    restrict_state("draft", "active")
 
             if rule_type.startswith("area"):
                 self.fields["group"].label = "Area"
@@ -70,7 +76,8 @@ def restrict_state(state_type, slug=None):
             del self.fields["text"]
 
         elif rule_type in ["author", "author_rfc", "shepherd", "ad"]:
-            restrict_state("draft", "rfc" if rule_type.endswith("rfc") else "active")
+            if not rule_type.endswith("_rfc"):
+                restrict_state("draft", "active")
 
             if rule_type.startswith("author"):
                 self.fields["person"].label = "Author"
@@ -84,7 +91,8 @@ def restrict_state(state_type, slug=None):
             del self.fields["text"]
 
         elif rule_type == "name_contains":
-            restrict_state("draft", "rfc" if rule_type.endswith("rfc") else "active")
+            if not rule_type.endswith("_rfc"):
+                restrict_state("draft", "active")
 
             del self.fields["person"]
             del self.fields["group"]

ietf/community/migrations/0003_track_rfcs.py

@@ -0,0 +1,50 @@
+# Generated by Django 4.2.3 on 2023-07-07 18:33
+
+from django.db import migrations
+
+
+def forward(apps, schema_editor):
+    """Track any RFCs that were created from tracked drafts"""
+    CommunityList = apps.get_model("community", "CommunityList")
+    RelatedDocument = apps.get_model("doc", "RelatedDocument")
+
+    # Handle individually tracked documents
+    for cl in CommunityList.objects.all():
+        for rfc in set(
+            RelatedDocument.objects.filter(
+                source__in=cl.added_docs.all(),
+                relationship__slug="became_rfc",
+            ).values_list("target__docs", flat=True)
+        ):
+            cl.added_docs.add(rfc)
+
+    # Handle rules - rules ending with _rfc should no longer filter by state.
+    # There are 9 CommunityLists with invalid author_rfc rules that are filtering
+    # by (draft, active) instead of (draft, rfc) state before migration. All but one
+    # also includes an author rule for (draft, active), so these will start following
+    # RFCs as well. The one exception will start tracking RFCs instead of I-Ds, which
+    # is probably what was intended, but will be a change in their user experience.
+    SearchRule = apps.get_model("community", "SearchRule")
+    rfc_rules = SearchRule.objects.filter(rule_type__endswith="_rfc")
+    rfc_rules.update(state=None)
+
+def reverse(apps, schema_editor):
+    Document = apps.get_model("doc", "Document")
+    for rfc in Document.objects.filter(type__slug="rfc"):
+        rfc.communitylist_set.clear()
+
+    # See the comment above regarding author_rfc
+    SearchRule = apps.get_model("community", "SearchRule")
+    State = apps.get_model("doc", "State")
+    SearchRule.objects.filter(rule_type__endswith="_rfc").update(
+        state=State.objects.get(type_id="draft", slug="rfc")
+    )
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("community", "0002_auto_20230320_1222"),
+        ("doc", "0014_move_rfc_docaliases"),
+    ]
+
+    operations = [migrations.RunPython(forward, reverse)]

ietf/community/tests.py

@@ -41,7 +41,7 @@ def test_rule_matching(self):
         clist = CommunityList.objects.create(user=User.objects.get(username="plain"))
 
         rule_group = SearchRule.objects.create(rule_type="group", group=draft.group, state=State.objects.get(type="draft", slug="active"), community_list=clist)
-        rule_group_rfc = SearchRule.objects.create(rule_type="group_rfc", group=draft.group, state=State.objects.get(type="draft", slug="rfc"), community_list=clist)
+        rule_group_rfc = SearchRule.objects.create(rule_type="group_rfc", group=draft.group, state=State.objects.get(type="rfc", slug="published"), community_list=clist)
         rule_area = SearchRule.objects.create(rule_type="area", group=draft.group.parent, state=State.objects.get(type="draft", slug="active"), community_list=clist)
 
         rule_state_iesg = SearchRule.objects.create(rule_type="state_iesg", state=State.objects.get(type="draft-iesg", slug="lc"), community_list=clist)
@@ -151,7 +151,7 @@ def test_manage_personal_list(self):
             "action": "add_rule",
             "rule_type": "author_rfc",
             "author_rfc-person": Person.objects.filter(documentauthor__document=draft).first().pk,
-            "author_rfc-state": State.objects.get(type="draft", slug="rfc").pk,
+            "author_rfc-state": State.objects.get(type="rfc", slug="published").pk,
         })
         self.assertEqual(r.status_code, 302)
         clist = CommunityList.objects.get(user__username="plain")
@@ -408,4 +408,4 @@ def test_notification(self):
         self.assertEqual(len(outbox), mailbox_before + 1)
         self.assertTrue(draft.name in outbox[-1]["Subject"])
 
-
+