ci: merge main to release (#7693)

ci: merge main to release

.github/dependabot.yml

@@ -9,14 +9,10 @@ updates:
     directory: "/"
     schedule:
       interval: "weekly"
-    reviewers:
-      - "ngpixel"
   - package-ecosystem: "docker"
     directory: "/docker"
     schedule:
       interval: "weekly"
-    reviewers:
-      - "ngpixel"
   - package-ecosystem: "pip"
     directory: "/"
     schedule:
@@ -27,8 +23,6 @@ updates:
     directory: "/"
     schedule:
       interval: "weekly"
-    reviewers:
-      - "ngpixel"
     groups:
       yarn:
         patterns:
@@ -37,8 +31,6 @@ updates:
     directory: "/playwright"
     schedule:
       interval: "weekly"
-    reviewers:
-      - "ngpixel"
     groups:
       npm:
         patterns:
@@ -47,8 +39,6 @@ updates:
     directory: "/dev/coverage-action"
     schedule:
       interval: "weekly"
-    reviewers:
-      - "ngpixel"
     groups:
       npm:
         patterns:
@@ -57,8 +47,6 @@ updates:
     directory: "/dev/deploy-to-container"
     schedule:
       interval: "weekly"
-    reviewers:
-      - "ngpixel"
     groups:
       npm:
         patterns:
@@ -67,8 +55,6 @@ updates:
     directory: "/dev/diff"
     schedule:
       interval: "weekly"
-    reviewers:
-      - "ngpixel"
     groups:
       npm:
         patterns:

docker/README.md

@@ -156,6 +156,11 @@ On Windows:
 docker compose down -v --rmi all
 docker image prune
 ```
+### Updating an older environment
+
+If you already have a clone, such as from a previous codesprint, and are updating that clone, before starting the datatracker from the updated image:
+* rm ietf/settings_local.py   # The startup script will put a new one, appropriate to the current release, in place
+* Execute the `Clean all` sequence above.
 
 ### Accessing PostgreSQL Port
 
@@ -197,4 +202,4 @@ drwxrwxr-x   5  100999  100999    4096 May 25 07:56 client
 (etc...)
 ```
 
-Try uninstalling Docker Desktop and installing Docker Compose manually. The Docker Compose bundled with Docker Desktop is incompatible with our software. See also [Rootless Docker: file ownership changes #3343](https://github.com/lando/lando/issues/3343), [Docker context desktop-linux has container permission issues #75](https://github.com/docker/desktop-linux/issues/75).
+Try uninstalling Docker Desktop and installing Docker Compose manually. The Docker Compose bundled with Docker Desktop is incompatible with our software. See also [Rootless Docker: file ownership changes #3343](https://github.com/lando/lando/issues/3343), [Docker context desktop-linux has container permission issues #75](https://github.com/docker/desktop-linux/issues/75).

docker/cleanall

@@ -1,12 +1,17 @@
 #!/bin/bash
 
+if test $(basename $PWD ) != "docker"
+then
+    echo "Run this from the docker directory" 1>&2
+    exit 1
+fi
+
 read -p "Stop and remove all containers, volumes and images for this project? [y/N] " -n 1 -r
 echo 
 if [[ $REPLY =~ ^[Yy]$ ]]
 then
     cd ..
     echo "Shutting down any instance still running and purge images..."
     docker compose down -v --rmi all
-    cd docker
     echo "Done!"
 fi

docker/cleandb

@@ -1,5 +1,11 @@
 #!/bin/bash
 
+if test $(basename $PWD ) != "docker"
+then
+    echo "Run this from the docker directory" 1>&2
+    exit 1
+fi
+
 cd ..
 echo "Shutting down any instance still running..."
 docker compose down
@@ -9,5 +15,5 @@ docker volume rm -f "${PROJNAME}_postgresdb-data"
 echo "Rebuilding the DB image..."
 docker compose pull db
 docker compose build --no-cache db
-cd docker
+
 echo "Done!"

ietf/doc/tests_ballot.py

@@ -806,7 +806,7 @@ def test_clear_ballot(self):
         ballot = create_ballot_if_not_open(None, draft, ad, 'approve')
         old_ballot_id = ballot.id
         draft.set_state(State.objects.get(used=True, type="draft-iesg", slug="iesg-eva")) 
-        url = urlreverse('ietf.doc.views_ballot.clear_ballot', kwargs=dict(name=draft.name,ballot_type_slug=draft.ballot_open('approve').ballot_type.slug))
+        url = urlreverse('ietf.doc.views_ballot.clear_ballot', kwargs=dict(name=draft.name,ballot_type_slug="approve"))
         login_testing_unauthorized(self, "secretary", url)
         r = self.client.get(url)
         self.assertEqual(r.status_code, 200)
@@ -816,6 +816,11 @@ def test_clear_ballot(self):
         self.assertIsNotNone(ballot)
         self.assertEqual(ballot.ballotpositiondocevent_set.count(),0)
         self.assertNotEqual(old_ballot_id, ballot.id)
+        # It's not valid to clear a ballot of a type where there's no matching state
+        url = urlreverse('ietf.doc.views_ballot.clear_ballot', kwargs=dict(name=draft.name,ballot_type_slug="statchg"))
+        r = self.client.post(url,{})
+        self.assertEqual(r.status_code, 404)
+
 
     def test_ballot_downref_approve(self):
         ad = Person.objects.get(name="Areað Irector")

ietf/doc/tests_status_change.py

@@ -484,7 +484,47 @@ def verify_relations(doc,target_name,status):
         verify_relations(doc,'rfc9998','tobcp' )
         verify_relations(doc,'rfc14'  ,'tohist')
         self.assertTrue(doc.latest_event(DocEvent,type="added_comment").desc.startswith('Affected RFC list changed.'))       
+
+    def test_clear_ballot(self):
+        doc = Document.objects.get(name='status-change-imaginary-mid-review')
+        url = urlreverse('ietf.doc.views_ballot.clear_ballot',kwargs=dict(name=doc.name, ballot_type_slug="statchg"))
+        login_testing_unauthorized(self, "secretary", url)
+
+        # Some additional setup
+        doc.relateddocument_set.create(target=Document.objects.get(name='rfc9999'),relationship_id='tois')
+        doc.relateddocument_set.create(target=Document.objects.get(name='rfc9998'),relationship_id='tohist')
+        create_ballot_if_not_open(None, doc, Person.objects.get(user__username="secretary"), "statchg")
+        doc.set_state(State.objects.get(slug='iesgeval',type='statchg'))
+        old_ballot = doc.ballot_open("statchg")
+        self.assertIsNotNone(old_ballot)
+
+        r = self.client.post(url, dict())
+        self.assertEqual(r.status_code,302)
+        new_ballot = doc.ballot_open("statchg")
+        self.assertIsNotNone(new_ballot)
+        self.assertNotEqual(new_ballot, old_ballot)
+        self.assertEqual(doc.get_state_slug("statchg"),"iesgeval")
+
+    def test_clear_deferred_ballot(self):
+        doc = Document.objects.get(name='status-change-imaginary-mid-review')
+        url = urlreverse('ietf.doc.views_ballot.clear_ballot',kwargs=dict(name=doc.name, ballot_type_slug="statchg"))
+        login_testing_unauthorized(self, "secretary", url)
+
+        # Some additional setup
+        doc.relateddocument_set.create(target=Document.objects.get(name='rfc9999'),relationship_id='tois')
+        doc.relateddocument_set.create(target=Document.objects.get(name='rfc9998'),relationship_id='tohist')
+        create_ballot_if_not_open(None, doc, Person.objects.get(user__username="secretary"), "statchg")
+        doc.set_state(State.objects.get(slug='defer',type='statchg'))
+        old_ballot = doc.ballot_open("statchg")
+        self.assertIsNotNone(old_ballot)
 
+        r = self.client.post(url, dict())
+        self.assertEqual(r.status_code,302)
+        new_ballot = doc.ballot_open("statchg")
+        self.assertIsNotNone(new_ballot)
+        self.assertNotEqual(new_ballot, old_ballot)
+        self.assertEqual(doc.get_state_slug("statchg"),"iesgeval")
+
     def setUp(self):
         super().setUp()
         IndividualRfcFactory(rfc_number=14,std_level_id='unkn') # draft was never issued

ietf/doc/views_ballot.py

@@ -399,11 +399,22 @@ def send_ballot_comment(request, name, ballot_id):
 def clear_ballot(request, name, ballot_type_slug):
     """Clear all positions and discusses on every open ballot for a document."""
     doc = get_object_or_404(Document, name=name)
+    # If there's no appropriate ballot type state, clearing would be an invalid action.
+    # This will need to be updated if we ever allow defering IRTF ballots
+    if ballot_type_slug == "approve":
+        state_machine = "draft-iesg"
+    elif ballot_type_slug in ["statchg","conflrev"]:
+        state_machine = ballot_type_slug
+    else:
+        state_machine = None
+    state_slug = state_machine and doc.get_state_slug(state_machine)
+    if state_machine is None or state_slug is None:
+        raise Http404
     if request.method == 'POST':
         by = request.user.person
         if close_ballot(doc, by, ballot_type_slug):
             create_ballot_if_not_open(request, doc, by, ballot_type_slug)
-        if doc.get_state('draft-iesg').slug == 'defer':
+        if state_slug == "defer":
             do_undefer_ballot(request,doc)
         return redirect("ietf.doc.views_doc.document_main", name=doc.name)
 

ietf/urls.py

@@ -5,6 +5,7 @@
 from django.contrib import admin
 from django.contrib.sitemaps import views as sitemap_views
 from django.contrib.staticfiles.urls import staticfiles_urlpatterns
+from django.http import HttpResponse
 from django.urls import include, path
 from django.views import static as static_view
 from django.views.generic import TemplateView
@@ -35,6 +36,7 @@
 
 urlpatterns = [
     url(r'^$', views_search.frontpage),
+    url(r'^health/', lambda _: HttpResponse()),
     url(r'^accounts/', include('ietf.ietfauth.urls')),
     url(r'^admin/', admin.site.urls),
     url(r'^admin/docs/', include('django.contrib.admindocs.urls')),

ietf/utils/tests.py

@@ -679,3 +679,12 @@ class TestForm(Form):
         self.assertTrue(changed_form.has_changed())
         unchanged_form = TestForm(initial={'test_field': [1]}, data={'test_field': [1]})
         self.assertFalse(unchanged_form.has_changed())
+
+
+class HealthTests(TestCase):
+    def test_health(self):
+        self.assertEqual(
+            self.client.get("/health/").status_code,
+            200,
+        )
+

k8s/README.md

@@ -0,0 +1,5 @@
+# Kustomize deployment
+
+## Run locally
+
+The `secrets.yaml` file is provided as a reference only and must be referenced manually in the `kustomization.yaml` file.

k8s/auth.yaml

@@ -19,40 +19,11 @@ spec:
         runAsNonRoot: true
       containers:
         # -----------------------------------------------------
-        # ScoutAPM Container
+        # Auth Container
         # -----------------------------------------------------
-        - name: scoutapm
-          image: "scoutapp/scoutapm:version-1.4.0"
-          imagePullPolicy: IfNotPresent
-          # Replace command with one that will shut down on a TERM signal
-          # The ./core-agent start command line is from the scoutapm docker image
-          command:
-            - "sh"
-            - "-c"
-            - >-
-              trap './core-agent shutdown --tcp 0.0.0.0:6590' TERM;
-              ./core-agent start --daemonize false --log-level debug --tcp 0.0.0.0:6590 &
-              wait $!
-          livenessProbe:
-            exec:
-              command:
-                - "sh"
-                - "-c"
-                - "./core-agent probe --tcp 0.0.0.0:6590 | grep -q 'Agent found'"
-          securityContext:
-            readOnlyRootFilesystem: true
-            runAsUser: 65534 # "nobody" user by default
-            runAsGroup: 65534  # "nogroup" group by default
-        # -----------------------------------------------------
-        # Datatracker Container
-        # -----------------------------------------------------
-        - name: datatracker
+        - name: auth
           image: "ghcr.io/ietf-tools/datatracker:$APP_IMAGE_TAG"
           imagePullPolicy: Always
-          ports:
-            - containerPort: 8000
-              name: http
-              protocol: TCP
           volumeMounts:
             - name: dt-vol
               mountPath: /a
@@ -72,8 +43,16 @@ spec:
             - name: "DEPLOY_UID"
               value: "$DEPLOY_UID"
           envFrom:
-            - configMapRef:
-                name: django-config
+            - secretRef:
+                name: dt-secrets-env
+          startupProbe:
+            httpGet:
+              port: 8000
+              path: /health/
+            initialDelaySeconds: 10
+            periodSeconds: 5
+            failureThreshold: 30
+            timeoutSeconds: 3
           securityContext:
             allowPrivilegeEscalation: false
             capabilities:
@@ -82,6 +61,53 @@ spec:
             readOnlyRootFilesystem: true
             runAsUser: 1000
             runAsGroup: 1000
+        # -----------------------------------------------------
+        # Nginx Container
+        # -----------------------------------------------------
+        - name: nginx
+          image: "ghcr.io/nginxinc/nginx-unprivileged:1.27"
+          imagePullPolicy: IfNotPresent
+          ports:
+            - containerPort: 8080
+              name: http
+              protocol: TCP
+          livenessProbe:
+            httpGet:
+              port: 8080
+              path: /health/nginx
+          securityContext:
+            readOnlyRootFilesystem: true
+          volumeMounts:
+            - name: nginx-tmp
+              mountPath: /tmp
+            - name: dt-cfg
+              mountPath: /etc/nginx/conf.d/auth.conf
+              subPath: nginx-auth.conf
+        # -----------------------------------------------------
+        # ScoutAPM Container
+        # -----------------------------------------------------
+        - name: scoutapm
+          image: "scoutapp/scoutapm:version-1.4.0"
+          imagePullPolicy: IfNotPresent
+          # Replace command with one that will shut down on a TERM signal
+          # The ./core-agent start command line is from the scoutapm docker image
+          command:
+            - "sh"
+            - "-c"
+            - >-
+              trap './core-agent shutdown --tcp 0.0.0.0:6590' TERM;
+              ./core-agent start --daemonize false --log-level debug --tcp 0.0.0.0:6590 &
+              wait $!
+          livenessProbe:
+            exec:
+              command:
+                - "sh"
+                - "-c"
+                - "./core-agent probe --tcp 0.0.0.0:6590 | grep -q 'Agent found'"
+          securityContext:
+            readOnlyRootFilesystem: true
+            runAsUser: 65534 # "nobody" user by default
+            runAsGroup: 65534  # "nogroup" group by default
       volumes:
         # To be overriden with the actual shared volume
         - name: dt-vol
@@ -97,6 +123,9 @@ spec:
         - name: dt-cfg
           configMap:
             name: files-cfgmap
+        - name: nginx-tmp
+          emptyDir:
+            sizeLimit: "500Mi"
       dnsPolicy: ClusterFirst
       restartPolicy: Always
       terminationGracePeriodSeconds: 60
@@ -108,9 +137,13 @@ metadata:
 spec:
   type: ClusterIP
   ports:
-    - port: 8080
+    - port: 80
       targetPort: http
       protocol: TCP
       name: http
+    - port: 8080
+      targetPort: http
+      protocol: TCP
+      name: http-old
   selector:
     app: auth