From a2283abb0162c63471202ec3d396174e292732eb Mon Sep 17 00:00:00 2001 From: Vinit Kumar Date: Mon, 10 Jul 2023 00:43:53 +0530 Subject: [PATCH 01/10] fix: issues with filer image crash when trying to generate thumbnail Thumbnail generation can fail when the image is not present or moved or if we have a reference in database but no image in storage. - Github Issue: Fixes #1379 Authored-by: Vinit Kumar Signed-off-by: Vinit Kumar --- filer/admin/fileadmin.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/filer/admin/fileadmin.py b/filer/admin/fileadmin.py index 6b30429ab..9c8e292d8 100644 --- a/filer/admin/fileadmin.py +++ b/filer/admin/fileadmin.py @@ -8,6 +8,7 @@ from easy_thumbnails.files import get_thumbnailer from easy_thumbnails.options import ThumbnailOptions +from easy_thumbnails.exceptions import InvalidImageFormatError from .. import settings from ..models import BaseImage, File @@ -174,10 +175,13 @@ def icon_view(self, request, file_id: int, size: int) -> HttpResponse: if not isinstance(file, BaseImage): raise Http404() - thumbnailer = get_thumbnailer(file) - thumbnail_options = ThumbnailOptions({'size': (size, size), "crop": True}) - thumbnail = thumbnailer.get_thumbnail(thumbnail_options, generate=True) - return HttpResponseRedirect(thumbnail.url) + try: + thumbnailer = get_thumbnailer(file) + thumbnail_options = ThumbnailOptions({'size': (size, size), "crop": True}) + thumbnail = thumbnailer.get_thumbnail(thumbnail_options, generate=True) + return HttpResponseRedirect(thumbnail.url) + except InvalidImageFormatError: + raise Http404 FileAdmin.fieldsets = FileAdmin.build_fieldsets() From c79046d0b68f88acf2981db2e6d4457ac3f16c2a Mon Sep 17 00:00:00 2001 From: Vinit Kumar Date: Mon, 10 Jul 2023 10:37:43 +0530 Subject: [PATCH 02/10] Update filer/admin/fileadmin.py Co-authored-by: Fabian Braun --- filer/admin/fileadmin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/filer/admin/fileadmin.py b/filer/admin/fileadmin.py index 9c8e292d8..57b2a8b74 100644 --- a/filer/admin/fileadmin.py +++ b/filer/admin/fileadmin.py @@ -181,7 +181,7 @@ def icon_view(self, request, file_id: int, size: int) -> HttpResponse: thumbnail = thumbnailer.get_thumbnail(thumbnail_options, generate=True) return HttpResponseRedirect(thumbnail.url) except InvalidImageFormatError: - raise Http404 + return HttpResponseRedirect(staticfiles_storage.url('filer/icons/file-missing.svg')) FileAdmin.fieldsets = FileAdmin.build_fieldsets() From 29b6b008b360e4cab4b4029594dd3bec4aee4d64 Mon Sep 17 00:00:00 2001 From: Fabian Braun Date: Tue, 11 Jul 2023 21:49:13 +0200 Subject: [PATCH 03/10] Add tests --- filer/admin/fileadmin.py | 3 ++- tests/test_admin.py | 43 +++++++++++++++++++++++++++++++++++++++- tests/test_validation.py | 4 ++-- 3 files changed, 46 insertions(+), 4 deletions(-) diff --git a/filer/admin/fileadmin.py b/filer/admin/fileadmin.py index 57b2a8b74..f2b1f4123 100644 --- a/filer/admin/fileadmin.py +++ b/filer/admin/fileadmin.py @@ -1,14 +1,15 @@ from django import forms from django.contrib.admin.utils import unquote +from django.contrib.staticfiles.storage import staticfiles_storage from django.http import Http404, HttpResponse, HttpResponseRedirect from django.shortcuts import get_object_or_404 from django.urls import path, reverse from django.utils.safestring import mark_safe from django.utils.translation import gettext as _ +from easy_thumbnails.exceptions import InvalidImageFormatError from easy_thumbnails.files import get_thumbnailer from easy_thumbnails.options import ThumbnailOptions -from easy_thumbnails.exceptions import InvalidImageFormatError from .. import settings from ..models import BaseImage, File diff --git a/tests/test_admin.py b/tests/test_admin.py index cabb6c715..aa47b4ac9 100644 --- a/tests/test_admin.py +++ b/tests/test_admin.py @@ -21,7 +21,7 @@ from filer.models.filemodels import File from filer.models.foldermodels import Folder, FolderPermission from filer.models.virtualitems import FolderRoot -from filer.settings import FILER_IMAGE_MODEL +from filer.settings import DEFERRED_THUMBNAIL_SIZES, FILER_IMAGE_MODEL from filer.templatetags.filer_admin_tags import file_icon_url from filer.thumbnail_processors import normalize_subject_location from filer.utils.loader import load_model @@ -254,9 +254,50 @@ class FilerImageAdminUrlsTests(TestCase): def setUp(self): self.superuser = create_superuser() self.client.login(username='admin', password='secret') + self.img = create_image() + self.image_name = 'test_file.jpg' + self.filename = os.path.join(settings.FILE_UPLOAD_TEMP_DIR, self.image_name) + self.img.save(self.filename, 'JPEG') + self.file_object = Image.objects.create( + file=django.core.files.File(open(self.filename, 'rb'), name=self.image_name) + ) def tearDown(self): self.client.logout() + os.remove(self.filename) + + def test_icon_view_sizes(self): + """Tests if redirects are issued for accepted thumbnail sizes and 404 otherwise""" + test_set = tuple((size, 302) for size in DEFERRED_THUMBNAIL_SIZES) + test_set += (50, 404), (90, 404), (320, 404) + for size, expected_status in test_set: + url = reverse('admin:filer_file_fileicon', kwargs={ + 'file_id': self.file_object.pk, + 'size': size, + }) + response = self.client.get(url) + self.assertEqual(response.status_code, expected_status) + if response.status_code == 302: # redirect + # Redirects to a media file + self.assertIn("/media/", response["Location"]) + # Does not redirect to a static file + self.assertNotIn("/static/", response["Location"]) + + def test_missing_file(self): + image = Image.objects.create( + owner=self.superuser, + original_filename="some-image.jpg", + ) + url = reverse('admin:filer_file_fileicon', kwargs={ + 'file_id': image.pk, + 'size': 80, + }) + # Make file unaccessible + + response = self.client.get(url) + + self.assertEqual(response.status_code, 302) + self.assertIn("icons/file-missing.svg", response["Location"]) class FilerClipboardAdminUrlsTests(TestCase): diff --git a/tests/test_validation.py b/tests/test_validation.py index 8b63cdae6..542bbc1ad 100644 --- a/tests/test_validation.py +++ b/tests/test_validation.py @@ -1,14 +1,14 @@ import os +import django.core from django.apps import apps from django.conf import settings -import django.core from django.test import TestCase from django.urls import reverse from django.utils.crypto import get_random_string from filer.models import File, Folder -from filer.validation import validate_upload, FileValidationError +from filer.validation import FileValidationError, validate_upload from tests.helpers import create_superuser From be899ee8d10bdacfac20d14c785e5a3b7a1146fb Mon Sep 17 00:00:00 2001 From: Fabian Braun Date: Tue, 11 Jul 2023 22:05:27 +0200 Subject: [PATCH 04/10] Add tests on mime types --- filer/apps.py | 10 +++++++++- tests/test_utils.py | 9 +++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/filer/apps.py b/filer/apps.py index fef1f6c53..6f6f57a30 100644 --- a/filer/apps.py +++ b/filer/apps.py @@ -1,3 +1,5 @@ +import mimetypes + from django.apps import AppConfig from django.core.exceptions import ImproperlyConfigured from django.utils.translation import gettext_lazy as _ @@ -14,8 +16,14 @@ def register_optional_heif_supprt(self): from .settings import IMAGE_EXTENSIONS, IMAGE_MIME_TYPES + # Register with easy_thumbnails register_heif_opener() - IMAGE_EXTENSIONS += [".heic", ".heics", ".heif", ".heifs", ".hif"] + HEIF_EXTENSIONS = [".heic", ".heics", ".heif", ".heifs", ".hif"] + # Add extensions to python mimetypes which filer uses + for ext in HEIF_EXTENSIONS: + mimetypes.add_type("image/heic", ext) + # Mark them as images + IMAGE_EXTENSIONS += HEIF_EXTENSIONS IMAGE_MIME_TYPES.append("heic") except (ModuleNotFoundError, ImportError): # No heif support installed diff --git a/tests/test_utils.py b/tests/test_utils.py index a6ba470bd..02d8d3948 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,3 +1,4 @@ +import mimetypes import os from zipfile import ZipFile @@ -5,6 +6,7 @@ from django.core.files import File as DjangoFile from django.test.testcases import TestCase +from filer.settings import IMAGE_EXTENSIONS from filer.utils.loader import load_object from filer.utils.zip import unzip from tests.helpers import create_image @@ -62,3 +64,10 @@ def tearDown(self): def test_unzipping_works(self): result = unzip(self.zipfilename) self.assertEqual(result[0][0].name, self.file.name) + + +class MimeTypesTestCase(TestCase): + def test_mime_types_known(self): + for ext in IMAGE_EXTENSIONS: + self.assertIsNotNone(mimetypes.guess_type(f"file{ext}")[0], + f"Mime type for extension {ext} unknown") From 3f4e675cd3ead211b733c1a6a72231d81819b0b2 Mon Sep 17 00:00:00 2001 From: Fabian Braun Date: Tue, 11 Jul 2023 22:08:57 +0200 Subject: [PATCH 05/10] Fix: Explicitly add image/webp to known mime types --- filer/apps.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/filer/apps.py b/filer/apps.py index 6f6f57a30..9f970039f 100644 --- a/filer/apps.py +++ b/filer/apps.py @@ -60,5 +60,8 @@ def resolve_validators(self): self.FILE_VALIDATORS[mime_type] = functions def ready(self): + # Make webp mime type known to python (needed for python < 3.11) + mimetypes.add_type("image/webp", ".webp") + # self.resolve_validators() self.register_optional_heif_supprt() From 7400543a2882b418b475096e401195528ca4ba81 Mon Sep 17 00:00:00 2001 From: Fabian Braun Date: Tue, 11 Jul 2023 22:13:59 +0200 Subject: [PATCH 06/10] Fix: always close file in tests --- tests/test_admin.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/test_admin.py b/tests/test_admin.py index aa47b4ac9..4375559d6 100644 --- a/tests/test_admin.py +++ b/tests/test_admin.py @@ -258,9 +258,8 @@ def setUp(self): self.image_name = 'test_file.jpg' self.filename = os.path.join(settings.FILE_UPLOAD_TEMP_DIR, self.image_name) self.img.save(self.filename, 'JPEG') - self.file_object = Image.objects.create( - file=django.core.files.File(open(self.filename, 'rb'), name=self.image_name) - ) + with django.core.files.File(open(self.filename, 'rb'), name=self.image_name) as upload: + self.file_object = Image.objects.create(file=upload) def tearDown(self): self.client.logout() From 806fac5937b54d14b187577921513e9b14089011 Mon Sep 17 00:00:00 2001 From: Fabian Braun Date: Tue, 11 Jul 2023 22:15:34 +0200 Subject: [PATCH 07/10] Add comment --- tests/test_utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_utils.py b/tests/test_utils.py index 02d8d3948..2c6e85dae 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -68,6 +68,7 @@ def test_unzipping_works(self): class MimeTypesTestCase(TestCase): def test_mime_types_known(self): + """Ensure that for all IMAGE_EXTENSIONS the mime types can be identified""" for ext in IMAGE_EXTENSIONS: self.assertIsNotNone(mimetypes.guess_type(f"file{ext}")[0], f"Mime type for extension {ext} unknown") From 9a4b8b97a920aaa10fda1fc36d86c403226c9a46 Mon Sep 17 00:00:00 2001 From: Fabian Braun Date: Tue, 11 Jul 2023 22:22:50 +0200 Subject: [PATCH 08/10] Add heif support to tests --- tests/requirements/base.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/requirements/base.txt b/tests/requirements/base.txt index 3bbe8ffa8..998ca24d2 100644 --- a/tests/requirements/base.txt +++ b/tests/requirements/base.txt @@ -1,6 +1,6 @@ # requirements from setup.py Pillow - +pillow-heif # other requirements coverage isort From f32820e037ecced419aef2829682399d9a152c79 Mon Sep 17 00:00:00 2001 From: Fabian Braun Date: Tue, 11 Jul 2023 22:33:10 +0200 Subject: [PATCH 09/10] Fix: put open in a with block in tests --- tests/test_admin.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_admin.py b/tests/test_admin.py index 4375559d6..19737fb5b 100644 --- a/tests/test_admin.py +++ b/tests/test_admin.py @@ -258,8 +258,8 @@ def setUp(self): self.image_name = 'test_file.jpg' self.filename = os.path.join(settings.FILE_UPLOAD_TEMP_DIR, self.image_name) self.img.save(self.filename, 'JPEG') - with django.core.files.File(open(self.filename, 'rb'), name=self.image_name) as upload: - self.file_object = Image.objects.create(file=upload) + with open(self.filename, 'rb') as upload: + self.file_object = Image.objects.create(file=django.core.files.File(upload, name=self.image_name)) def tearDown(self): self.client.logout() From 2fcbc8f83a6ed509bdd3d715f441ced4b716e0c9 Mon Sep 17 00:00:00 2001 From: Fabian Braun Date: Wed, 12 Jul 2023 08:33:42 +0200 Subject: [PATCH 10/10] Update changelog --- CHANGELOG.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 9e42a310d..445056c94 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -2,6 +2,14 @@ CHANGELOG ========= +unreleased +================== + +* Fix a bug that creates a server error when requesting a thumbnail from an + invalid or missing file +* Fix a bug that on some systems webp images were not recognized +* Add missing css map files + 3.0.0 (2023-07-05) ==================