chore: Prevent test suite artifact creation in work directory (#6438)

* chore: Prevent test suite artifact creation in work directory

Also remove a few other stale test assets while I'm here.

* Try and fix CI

* Change IDSUBMIT_REPOSITORY_PATH

* Make the dir

* test: clean up media/nomcom directories

* test: de-dup settings_temp_path_overrides

* chore: remove debug

* refactor: avoid premature import of test_utils

* refactor: drop useless lambda wrapper

---------

Co-authored-by: Jennifer Richards <[email protected]>

dev/tests/settings_local.py

@@ -17,8 +17,8 @@
 }
 
 IDSUBMIT_IDNITS_BINARY = "/usr/local/bin/idnits"
-IDSUBMIT_REPOSITORY_PATH = "test/id/"
-IDSUBMIT_STAGING_PATH = "test/staging/"
+IDSUBMIT_REPOSITORY_PATH = "/assets/ietfdata/doc/draft/repository"
+IDSUBMIT_STAGING_PATH = "/assets/www6s/staging/"
 
 AGENDA_PATH = '/assets/www6s/proceedings/'
 MEETINGHOST_LOGO_PATH = AGENDA_PATH

docker/configs/settings_local.py

@@ -8,7 +8,7 @@
 from ietf.settings_postgresqldb import DATABASES   # pyflakes:ignore
 
 IDSUBMIT_IDNITS_BINARY = "/usr/local/bin/idnits"
-IDSUBMIT_STAGING_PATH = "test/staging/"
+IDSUBMIT_STAGING_PATH = "/assets/www6s/staging/"
 
 AGENDA_PATH = '/assets/www6s/proceedings/'
 MEETINGHOST_LOGO_PATH = AGENDA_PATH
@@ -53,7 +53,7 @@
 FTP_DIR = '/assets/ftp'
 
 NOMCOM_PUBLIC_KEYS_DIR = 'data/nomcom_keys/public_keys/'
-SLIDE_STAGING_PATH = 'test/staging/'
+SLIDE_STAGING_PATH = '/assets/www6s/staging/'
 
 DE_GFM_BINARY = '/usr/local/bin/de-gfm'
 

docker/scripts/app-create-dirs.sh

@@ -1,13 +1,6 @@
 #!/bin/bash
 
 for sub in \
-    test/id \
-    test/staging \
-    test/archive \
-    test/rfc \
-    test/media \
-    test/wiki/ietf \
-    data/nomcom_keys/public_keys \
     /assets/archive/id \
     /assets/collection \
     /assets/collection/draft-archive \
@@ -27,6 +20,7 @@ for sub in \
     /assets/ietfdata/derived \
     /assets/ietfdata/derived/bibxml \
     /assets/ietfdata/derived/bibxml/bibxml-ids \
+    /assets/ietfdata/doc/draft/repository \
     /assets/www6s \
     /assets/www6s/staging \
     /assets/www6s/wg-descriptions \

ietf/meeting/tests_js.py

@@ -5,7 +5,7 @@
 import time
 import datetime
 import shutil
-import os
+import tempfile
 import re
 
 from django.utils import timezone
@@ -939,13 +939,8 @@ def tearDown(self):
     def tempdir(self, label):
         # Borrowed from  test_utils.TestCase
         slug = slugify(self.__class__.__name__.replace('.','-'))
-        dirname = "tmp-{label}-{slug}-dir".format(**locals())
-        if 'VIRTUAL_ENV' in os.environ:
-            dirname = os.path.join(os.environ['VIRTUAL_ENV'], dirname)
-        path = os.path.abspath(dirname)
-        if not os.path.exists(path):
-            os.mkdir(path)
-        return path
+        suffix = "-{label}-{slug}-dir".format(**locals())
+        return tempfile.mkdtemp(suffix=suffix)
 
     def displayed_interims(self, groups=None):
         sessions = add_event_info_to_session_qs(

ietf/person/factories.py

@@ -8,7 +8,7 @@
 import faker.config
 import os
 import random
-import shutil
+from PIL import Image
 
 from unidecode import unidecode
 from unicodedata import normalize
@@ -103,10 +103,9 @@ def default_photo(obj, create, extracted, **kwargs): # pylint: disable=no-self-a
             media_name = "%s/%s.jpg" % (settings.PHOTOS_DIRNAME, photo_name)
             obj.photo = media_name
             obj.photo_thumb = media_name
-            photosrc = os.path.join(settings.TEST_DATA_DIR, "profile-default.jpg")
             photodst = os.path.join(settings.PHOTOS_DIR,  photo_name + '.jpg')
-            if not os.path.exists(photodst):
-                shutil.copy(photosrc, photodst)
+            img = Image.new('RGB', (200, 200))
+            img.save(photodst)
             def delete_file(file):
                 os.unlink(file)
             atexit.register(delete_file, photodst)

ietf/settings_test.py

@@ -9,9 +9,12 @@
 #   ./manage.py test --settings=settings_test doc.ChangeStateTestCase
 #
 
-import os 
+import atexit
+import os
+import shutil
+import tempfile
 from ietf.settings import *                                          # pyflakes:ignore
-from ietf.settings import TEST_CODE_COVERAGE_CHECKER, BASE_DIR, PHOTOS_DIRNAME
+from ietf.settings import TEST_CODE_COVERAGE_CHECKER
 import debug                            # pyflakes:ignore
 debug.debug = True
 
@@ -48,11 +51,20 @@ def __getitem__(self, item):
 if TEST_CODE_COVERAGE_CHECKER and not TEST_CODE_COVERAGE_CHECKER._started: # pyflakes:ignore
     TEST_CODE_COVERAGE_CHECKER.start()                          # pyflakes:ignore
 
-NOMCOM_PUBLIC_KEYS_DIR=os.path.abspath("tmp-nomcom-public-keys-dir")
 
-MEDIA_ROOT = os.path.join(os.path.dirname(BASE_DIR), 'test/media/') # pyflakes:ignore
-MEDIA_URL = '/test/media/'
-PHOTOS_DIR = MEDIA_ROOT + PHOTOS_DIRNAME                            # pyflakes:ignore
+def tempdir_with_cleanup(**kwargs):
+    """Utility to create a temporary dir and arrange cleanup"""
+    _dir = tempfile.mkdtemp(**kwargs)
+    atexit.register(shutil.rmtree, _dir)
+    return _dir
+
+
+NOMCOM_PUBLIC_KEYS_DIR = tempdir_with_cleanup(suffix="-nomcom-public-keys-dir")
+
+MEDIA_ROOT = tempdir_with_cleanup(suffix="-media")
+PHOTOS_DIRNAME = "photo"
+PHOTOS_DIR = os.path.join(MEDIA_ROOT, PHOTOS_DIRNAME)
+os.mkdir(PHOTOS_DIR)
 
 # Undo any developer-dependent middleware when running the tests
 MIDDLEWARE = [ c for c in MIDDLEWARE if not c in DEV_MIDDLEWARE ] # pyflakes:ignore

ietf/utils/test_utils.py

@@ -34,7 +34,7 @@
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
-import os
+import tempfile
 import re
 import email
 import html5lib
@@ -239,13 +239,8 @@ def normalize(x):
 
     def tempdir(self, label):
         slug = slugify(self.__class__.__name__.replace('.','-'))
-        dirname = "tmp-{label}-{slug}-dir".format(**locals())
-        if 'VIRTUAL_ENV' in os.environ:
-            dirname = os.path.join(os.environ['VIRTUAL_ENV'], dirname)
-        path = os.path.abspath(dirname)
-        if not os.path.exists(path):
-            os.mkdir(path)
-        return path
+        suffix = "-{label}-{slug}-dir".format(**locals())
+        return tempfile.mkdtemp(suffix=suffix)
 
     def assertNoFormPostErrors(self, response, error_css_selector=".is-invalid"):
         """Try to fish out form errors, if none found at least check the
@@ -306,7 +301,7 @@ def setUp(self):
 
         # Replace settings paths with temporary directories.
         self._ietf_temp_dirs = {}  # trashed during tearDown, DO NOT put paths you care about in this
-        for setting in self.settings_temp_path_overrides:
+        for setting in set(self.settings_temp_path_overrides):
             self._ietf_temp_dirs[setting] = self.tempdir(slugify(setting))
         self._ietf_saved_context = django.test.utils.override_settings(**self._ietf_temp_dirs)
         self._ietf_saved_context.enable()