Merge pull request #5740 from ietf-tools/feat/django4

feat!: django4

.github/workflows/build.yml

@@ -91,10 +91,10 @@ jobs:
           echo "pkg_version=$NEXT_VERSION" >> $GITHUB_OUTPUT
           echo "::notice::Release $NEXT_VERSION created using branch $GITHUB_REF_NAME"
         else
-          echo "Using TEST mode: 10.0.0-dev.$GITHUB_RUN_NUMBER"
+          echo "Using TEST mode: 11.0.0-dev.$GITHUB_RUN_NUMBER"
           echo "should_deploy=false" >> $GITHUB_OUTPUT
-          echo "pkg_version=10.0.0-dev.$GITHUB_RUN_NUMBER" >> $GITHUB_OUTPUT
-          echo "::notice::Non-production build 10.0.0-dev.$GITHUB_RUN_NUMBER created using branch $GITHUB_REF_NAME"
+          echo "pkg_version=11.0.0-dev.$GITHUB_RUN_NUMBER" >> $GITHUB_OUTPUT
+          echo "::notice::Non-production build 11.0.0-dev.$GITHUB_RUN_NUMBER created using branch $GITHUB_REF_NAME"
         fi
 
   # -----------------------------------------------------------------

.github/workflows/ci-run-tests.yml

@@ -4,6 +4,7 @@ on:
   pull_request:
     branches:
       - 'main'
+      - 'feat/django4'
     paths:
       - 'client/**'
       - 'ietf/**'

dev/deploy-to-container/settings_local.py

@@ -10,7 +10,7 @@
         'HOST': '__DBHOST__',
         'PORT': 5432,
         'NAME': 'datatracker',
-        'ENGINE': 'django.db.backends.postgresql_psycopg2',
+        'ENGINE': 'django.db.backends.postgresql',
         'USER': 'django',
         'PASSWORD': 'RkTkDPFnKpko',
     },

dev/diff/settings_local.py

@@ -10,7 +10,7 @@
         'HOST': '__DBHOST__',
         'PORT': 5432,
         'NAME': 'datatracker',
-        'ENGINE': 'django.db.backends.postgresql_psycopg2',
+        'ENGINE': 'django.db.backends.postgresql',
         'USER': 'django',
         'PASSWORD': 'RkTkDPFnKpko',
     },

dev/tests/settings_local.py

@@ -10,7 +10,7 @@
         'HOST': 'db',
         'PORT': 5432,
         'NAME': 'datatracker',
-        'ENGINE': 'django.db.backends.postgresql_psycopg2',
+        'ENGINE': 'django.db.backends.postgresql',
         'USER': 'django',
         'PASSWORD': 'RkTkDPFnKpko',
     },

docker/configs/settings_postgresqldb.py

@@ -3,7 +3,7 @@
         'HOST': 'db',
         'PORT': 5432,
         'NAME': 'datatracker',
-        'ENGINE': 'django.db.backends.postgresql_psycopg2',
+        'ENGINE': 'django.db.backends.postgresql',
         'USER': 'django',
         'PASSWORD': 'RkTkDPFnKpko',
     },

ietf/__init__.py

@@ -6,7 +6,7 @@
 
 # Version must stay in single quotes for automatic CI replace
 # Don't add patch number here:
-__version__ = '10.0.0-dev'
+__version__ = '11.0.0-dev'
 
 # Release hash must stay in single quotes for automatic CI replace
 __release_hash__ = ''

ietf/api/serializer.py

@@ -9,11 +9,12 @@
 from django.core.exceptions import ObjectDoesNotExist, FieldError
 from django.core.serializers.json import Serializer
 from django.http import HttpResponse
-from django.utils.encoding import smart_text
+from django.utils.encoding import smart_str
 from django.db.models import Field
-from django.db.models.query import QuerySet
 from django.db.models.signals import post_save, post_delete, m2m_changed
 
+from django_stubs_ext import QuerySetAny
+
 import debug                            # pyflakes:ignore
 
 
@@ -121,7 +122,7 @@ def end_object(self, obj):
         for name in expansions:
             try:
                 field = getattr(obj, name)
-                #self._current["_"+name] = smart_text(field)
+                #self._current["_"+name] = smart_str(field)
                 if not isinstance(field, Field):
                     options = self.options.copy()
                     options["expand"] = [ v[len(name)+2:] for v in options["expand"] if v.startswith(name+"__") ]
@@ -145,7 +146,7 @@ def end_object(self, obj):
                                 field_value = None
                         else:
                             field_value = field
-                        if isinstance(field_value, QuerySet) or isinstance(field_value, list):
+                        if isinstance(field_value, QuerySetAny) or isinstance(field_value, list):
                             self._current[name] = dict([ (rel.pk, self.expand_related(rel, name)) for rel in field_value ])
                         else:
                             if hasattr(field_value, "_meta"):
@@ -188,10 +189,10 @@ def handle_fk_field(self, obj, field):
                 related = related.natural_key()
             elif field.remote_field.field_name == related._meta.pk.name:
                 # Related to remote object via primary key
-                related = smart_text(related._get_pk_val(), strings_only=True)
+                related = smart_str(related._get_pk_val(), strings_only=True)
             else:
                 # Related to remote object via other field
-                related = smart_text(getattr(related, field.remote_field.field_name), strings_only=True)
+                related = smart_str(getattr(related, field.remote_field.field_name), strings_only=True)
         self._current[field.name] = related
 
     def handle_m2m_field(self, obj, field):
@@ -201,7 +202,7 @@ def handle_m2m_field(self, obj, field):
             elif self.use_natural_keys and hasattr(field.remote_field.to, 'natural_key'):
                 m2m_value = lambda value: value.natural_key()
             else:
-                m2m_value = lambda value: smart_text(value._get_pk_val(), strings_only=True)
+                m2m_value = lambda value: smart_str(value._get_pk_val(), strings_only=True)
             self._current[field.name] = [m2m_value(related)
                                for related in getattr(obj, field.name).iterator()]
 
@@ -221,7 +222,7 @@ class JsonExportMixin(object):
 #             obj = None
 # 
 #         if obj is None:
-#             raise Http404(_('%(name)s object with primary key %(key)r does not exist.') % {'name': force_text(self.model._meta.verbose_name), 'key': escape(object_id)})
+#             raise Http404(_('%(name)s object with primary key %(key)r does not exist.') % {'name': force_str(self.model._meta.verbose_name), 'key': escape(object_id)})
 # 
 #         content_type = 'application/json'
 #         return HttpResponse(serialize([ obj ], sort_keys=True, indent=3)[2:-2], content_type=content_type)
@@ -264,6 +265,6 @@ def json_view(self, request, filter=None, expand=None):
             qd = dict( ( k, json.loads(v)[0] )  for k,v in items )
         except (FieldError, ValueError) as e:
             return HttpResponse(json.dumps({"error": str(e)}, sort_keys=True, indent=3), content_type=content_type)
-        text = json.dumps({smart_text(self.model._meta): qd}, sort_keys=True, indent=3)
+        text = json.dumps({smart_str(self.model._meta): qd}, sort_keys=True, indent=3)
         return HttpResponse(text, content_type=content_type)
 

ietf/api/tests.py

@@ -691,7 +691,7 @@ def test_api_new_meeting_registration(self):
         self.assertEqual(set(missing_fields), set(drop_fields))
 
     def test_api_version(self):
-        DumpInfo.objects.create(date=timezone.datetime(2022,8,31,7,10,1,tzinfo=timezone.utc), host='testapi.example.com',tz='UTC')
+        DumpInfo.objects.create(date=timezone.datetime(2022,8,31,7,10,1,tzinfo=datetime.timezone.utc), host='testapi.example.com',tz='UTC')
         url = urlreverse('ietf.api.views.version')
         r = self.client.get(url)
         data = r.json()

ietf/api/urls.py

@@ -1,7 +1,7 @@
 # Copyright The IETF Trust 2017, All Rights Reserved
 
 from django.conf import settings
-from django.conf.urls import include
+from django.urls import include
 from django.views.generic import TemplateView
 
 from ietf import api

ietf/community/views.py

@@ -22,6 +22,7 @@
 from ietf.community.utils import states_of_significant_change, reset_name_contains_index_for_rule
 from ietf.doc.models import DocEvent, Document
 from ietf.doc.utils_search import prepare_document_table
+from ietf.utils.http import is_ajax
 from ietf.utils.response import permission_denied
 
 def view_list(request, username=None):
@@ -142,7 +143,7 @@ def track_document(request, name, username=None, acronym=None):
         if not doc in clist.added_docs.all():
             clist.added_docs.add(doc)
 
-        if request.is_ajax():
+        if is_ajax(request):
             return HttpResponse(json.dumps({ 'success': True }), content_type='application/json')
         else:
             return HttpResponseRedirect(clist.get_absolute_url())
@@ -162,7 +163,7 @@ def untrack_document(request, name, username=None, acronym=None):
         if clist.pk is not None:
             clist.added_docs.remove(doc)
 
-        if request.is_ajax():
+        if is_ajax(request):
             return HttpResponse(json.dumps({ 'success': True }), content_type='application/json')
         else:
             return HttpResponseRedirect(clist.get_absolute_url())

ietf/doc/mails.py

@@ -11,7 +11,7 @@
 from django.conf import settings
 from django.urls import reverse as urlreverse
 from django.utils import timezone
-from django.utils.encoding import force_text
+from django.utils.encoding import force_str
 
 import debug                            # pyflakes:ignore
 from ietf.doc.templatetags.mail_filters import std_level_prompt
@@ -175,7 +175,7 @@ def generate_ballot_writeup(request, doc):
     e.doc = doc
     e.rev = doc.rev
     e.desc = "Ballot writeup was generated"
-    e.text = force_text(render_to_string("doc/mail/ballot_writeup.txt", {'iana': iana, 'doc': doc }))
+    e.text = force_str(render_to_string("doc/mail/ballot_writeup.txt", {'iana': iana, 'doc': doc }))
 
     # caller is responsible for saving, if necessary
     return e
@@ -187,7 +187,7 @@ def generate_ballot_rfceditornote(request, doc):
     e.doc = doc
     e.rev = doc.rev
     e.desc = "RFC Editor Note for ballot was generated"
-    e.text = force_text(render_to_string("doc/mail/ballot_rfceditornote.txt"))
+    e.text = force_str(render_to_string("doc/mail/ballot_rfceditornote.txt"))
     e.save()
 
     return e
@@ -232,7 +232,7 @@ def generate_last_call_announcement(request, doc):
     e.doc = doc
     e.rev = doc.rev
     e.desc = "Last call announcement was generated"
-    e.text = force_text(mail)
+    e.text = force_str(mail)
 
     # caller is responsible for saving, if necessary
     return e
@@ -252,7 +252,7 @@ def generate_approval_mail(request, doc):
     e.doc = doc
     e.rev = doc.rev
     e.desc = "Ballot approval text was generated"
-    e.text = force_text(mail)
+    e.text = force_str(mail)
 
     # caller is responsible for saving, if necessary
     return e

ietf/doc/migrations/0004_alter_dochistory_ad_alter_dochistory_shepherd_and_more.py

@@ -0,0 +1,36 @@
+# Generated by Django 4.0.10 on 2023-05-16 20:36
+
+from django.db import migrations
+import django.db.models.deletion
+import ietf.utils.models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('person', '0001_initial'),
+        ('doc', '0003_remove_document_info_order'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='dochistory',
+            name='ad',
+            field=ietf.utils.models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='ad_%(class)s_set', to='person.person', verbose_name='area director'),
+        ),
+        migrations.AlterField(
+            model_name='dochistory',
+            name='shepherd',
+            field=ietf.utils.models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='shepherd_%(class)s_set', to='person.email'),
+        ),
+        migrations.AlterField(
+            model_name='document',
+            name='ad',
+            field=ietf.utils.models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='ad_%(class)s_set', to='person.person', verbose_name='area director'),
+        ),
+        migrations.AlterField(
+            model_name='document',
+            name='shepherd',
+            field=ietf.utils.models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='shepherd_%(class)s_set', to='person.email'),
+        ),
+    ]

ietf/doc/models.py

@@ -24,7 +24,7 @@
 from django.contrib.contenttypes.models import ContentType
 from django.conf import settings
 from django.utils import timezone
-from django.utils.encoding import force_text
+from django.utils.encoding import force_str
 from django.utils.html import mark_safe # type:ignore
 from django.contrib.staticfiles import finders
 
@@ -1134,7 +1134,7 @@ class DocHistory(DocumentInfo):
     name = models.CharField(max_length=255)
 
     def __str__(self):
-        return force_text(self.doc.name)
+        return force_str(self.doc.name)
 
     def get_related_session(self):
         return self.doc.get_related_session()
@@ -1196,7 +1196,7 @@ def document(self):
         return self.docs.first()
 
     def __str__(self):
-        return u"%s-->%s" % (self.name, ','.join([force_text(d.name) for d in self.docs.all() if isinstance(d, Document) ]))
+        return u"%s-->%s" % (self.name, ','.join([force_str(d.name) for d in self.docs.all() if isinstance(d, Document) ]))
     document_link = admin_link("document")
     class Meta:
         verbose_name = "document alias"

ietf/doc/templatetags/ietf_filters.py

@@ -13,8 +13,7 @@
 from django.template.defaultfilters import truncatewords_html, linebreaksbr, stringfilter, striptags
 from django.utils.safestring import mark_safe, SafeData
 from django.utils.html import strip_tags
-from django.utils.encoding import force_text
-from django.utils.encoding import force_str # pyflakes:ignore force_str is used in the doctests
+from django.utils.encoding import force_str
 from django.urls import reverse as urlreverse
 from django.core.cache import cache
 from django.core.exceptions import ValidationError
@@ -132,7 +131,7 @@ def bracketpos(pos,posslug):
 @register.filter
 def prettystdname(string, space=" "):
     from ietf.doc.utils import prettify_std_name
-    return prettify_std_name(force_text(string or ""), space)
+    return prettify_std_name(force_str(string or ""), space)
 
 @register.filter
 def rfceditor_info_url(rfcnum : str):

ietf/doc/tests.py

@@ -1779,7 +1779,7 @@ def test_document_ballot_needed_positions(self):
         self.client.login(username='ad', password='ad+password')
         r = self.client.post(urlreverse('ietf.doc.views_status_change.change_state',kwargs=dict(name=doc.name)),dict(new_state=iesgeval_pk))
         self.assertEqual(r.status_code, 302)
-        r = self.client.get(r._headers["location"][1])
+        r = self.client.get(r.headers["location"])
         self.assertContains(r, ">IESG Evaluation<")
         self.assertEqual(len(outbox), 2)
         self.assertIn('iesg-secretary',outbox[0]['To'])

ietf/doc/tests_draft.py

@@ -573,7 +573,7 @@ def test_request_resurrect(self):
         r = self.client.get(url)
         self.assertEqual(r.status_code, 200)
         q = PyQuery(r.content)
-        self.assertEqual(len(q('form [type=submit]')), 1)
+        self.assertEqual(len(q('#content form [type=submit]')), 1)
 
 
         # request resurrect
@@ -609,7 +609,7 @@ def test_resurrect(self):
         r = self.client.get(url)
         self.assertEqual(r.status_code, 200)
         q = PyQuery(r.content)
-        self.assertEqual(len(q('form [type=submit]')), 1)
+        self.assertEqual(len(q('#content form [type=submit]')), 1)
 
         # complete resurrect
         events_before = draft.docevent_set.count()

ietf/doc/tests_js.py

@@ -118,7 +118,7 @@ def _read_author_form(form_elt):
         # Must provide a "basis" (change reason)
         self.driver.find_element(By.ID, 'id_basis').send_keys('change testing')
         # Now click the 'submit' button and check that the update was accepted.
-        submit_button = self.driver.find_element(By.CSS_SELECTOR, 'button[type="submit"]')
+        submit_button = self.driver.find_element(By.CSS_SELECTOR, '#content button[type="submit"]')
         self.driver.execute_script("arguments[0].click();", submit_button)  # FIXME: no idea why this fails:
         # self.scroll_to_element(submit_button)
         # submit_button.click()
@@ -132,4 +132,4 @@ def _read_author_form(form_elt):
         self.assertEqual(
             list(draft.documentauthor_set.values_list('person', flat=True)),
             [first_auth.person.pk] + [auth.pk for auth in authors]
-        )
+        )

ietf/doc/urls.py

@@ -33,9 +33,9 @@
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
-from django.conf.urls import include
-from django.views.generic import RedirectView
 from django.conf import settings
+from django.urls import include
+from django.views.generic import RedirectView
 
 from ietf.doc import views_search, views_draft, views_ballot, views_status_change, views_doc, views_downref, views_stats, views_help, views_bofreq
 from ietf.utils.urls import url

ietf/doc/utils.py

@@ -17,14 +17,14 @@
 
 from django.conf import settings
 from django.contrib import messages
-from django.db.models import QuerySet
 from django.forms import ValidationError
 from django.http import Http404
 from django.template.loader import render_to_string
 from django.utils import timezone
 from django.utils.html import escape
 from django.urls import reverse as urlreverse
 
+from django_stubs_ext import QuerySetAny
 
 import debug                            # pyflakes:ignore
 from ietf.community.models import CommunityList
@@ -345,15 +345,15 @@ def augment_events_with_revision(doc, events):
     """Take a set of events for doc and add a .rev attribute with the
     revision they refer to by checking NewRevisionDocEvents."""
 
-    if isinstance(events, QuerySet):
+    if isinstance(events, QuerySetAny):
         qs = events.filter(newrevisiondocevent__isnull=False)
     else:
         qs = NewRevisionDocEvent.objects.filter(doc=doc)
     event_revisions = list(qs.order_by('time', 'id').values('id', 'rev', 'time'))
 
     if doc.type_id == "draft" and doc.get_state_slug() == "rfc":
         # add fake "RFC" revision
-        if isinstance(events, QuerySet):
+        if isinstance(events, QuerySetAny):
             e = events.filter(type="published_rfc").order_by('time').last()
         else:
             e = doc.latest_event(type="published_rfc")