Ktximporter: forward Basis Universal compressed images to BasisImporter

doc/building-plugins.dox

@@ -227,7 +227,7 @@ enable them, do the following:
 
 -   For @ref Trade::BasisImporter "BasisImporter" or
     @ref Trade::BasisImageConverter "BasisImageConverter", [download commit
-    `2f43afcc` of the Basis Universal repo](https://github.com/BinomialLLC/basis_universal/archive/2f43afcc97d0a5dafdb73b4e24e123cf9687a418.tar.gz),
+    `77b7df8e` of the Basis Universal repo](https://github.com/BinomialLLC/basis_universal/archive/77b7df8e5df3532a42ef3c76de0c14cc005d0f65.tar.gz),
     extract it into `src/external/basis-universal` (note the dash instead of an
     underscore) and set `WITH_BASISIMPORTER` / `WITH_BASISIMAGECONVERTER` to
     `ON` in `package/debian/rules`

modules/FindBasisUniversal.cmake

@@ -31,6 +31,7 @@
 #   Copyright © 2010, 2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019,
 #               2020, 2021 Vladimír Vondruš <[email protected]>
 #   Copyright © 2019 Jonathan Hale <[email protected]>
+#   Copyright © 2021 Pablo Escobar <[email protected]>
 #
 #   Permission is hereby granted, free of charge, to any person obtaining a
 #   copy of this software and associated documentation files (the "Software"),
@@ -51,13 +52,50 @@
 #   DEALINGS IN THE SOFTWARE.
 #
 
+# Several places in this find module assume that the C language is enabled:
+#   - test_big_endian() assumes C is enabled, configuration fails without.
+#     CMake says to call enable_language() in the highest directory using the
+#     language, so we can't do that here.
+#   - both the transcoder and encoder link to .c files that would just not be
+#     compiled without the language enabled, or the source file language being
+#     changed to CXX
+# Currently both BasisImporter and BasisImageConverter call enable_language(C).
+
 list(FIND BasisUniversal_FIND_COMPONENTS "Encoder" _index)
 if(${_index} GREATER -1)
     list(APPEND BasisUniversal_FIND_COMPONENTS "Transcoder")
     list(REMOVE_DUPLICATES BasisUniversal_FIND_COMPONENTS)
 endif()
 
+# Figure out endianness for Basis Universal. test_big_endian() fails on
+# Emscripten, but WebAssembly is always little-endian. On CMake 3.8 and below,
+# test_big_endian() requires C support which breaks compilation in funny ways
+# (see comment below) so we skip that.
+if(NOT CORRADE_TARGET_EMSCRIPTEN AND NOT CMAKE_VERSION VERSION_LESS 3.9)
+    include(TestBigEndian)
+    test_big_endian(BIG_ENDIAN)
+endif()
+
 macro(_basis_setup_source_file source)
+    # Compile any .c files as C++. Otherwise the files are just ignored because
+    # the C language is not enabled by project() or enable_language(). Calling
+    # enable_language in a find module is not a good idea, and even if we do
+    # this higher up in one of the Basis* plugins, some compilers will require
+    # the C99 standard being enabled and/or complain about -std=c++11 (done by
+    # CORRADE_CXX_STANDARD) being set on a C compiler. Doing both (enabling C
+    # and setting LANGUAGE CXX) still results in static libraries compiling
+    # with C and producing above-mentioned errors, possibly to do with
+    # LINKER_LANGUAGE. What a horrible mess.
+    set_property(SOURCE ${source} PROPERTY LANGUAGE
+        CXX)
+
+    # Tell Basis if we're on a big endian system. It currently doesn't figure
+    # this out by itself.
+    if(BIG_ENDIAN)
+        set_property(SOURCE ${source} APPEND PROPERTY COMPILE_DEFINITIONS
+            BASISD_IS_BIG_ENDIAN=1)
+    endif()
+
     # Basis shouldn't override the MSVC iterator debug level as it would make
     # it inconsistent with the rest of the code
     if(CORRADE_TARGET_WINDOWS)
@@ -73,7 +111,7 @@ macro(_basis_setup_source_file source)
             " -w")
     # Clang supports -w, but it doesn't have any effect on all the
     # -Wall -Wold-style-cast etc flags specified before. -Wno-everything does.
-    # Funnily enough this is not an issue on Emscripten.;
+    # Funnily enough this is not an issue on Emscripten.
     elseif(CMAKE_CXX_COMPILER_ID MATCHES "(Apple)?Clang" AND NOT CMAKE_CXX_SIMULATE_ID STREQUAL "MSVC")
         set_property(SOURCE ${source} APPEND_STRING PROPERTY COMPILE_FLAGS
             " -Wno-everything")
@@ -125,20 +163,30 @@ foreach(_component ${BasisUniversal_FIND_COMPONENTS})
                         "Set BASIS_UNIVERSAL_DIR to the root of a directory containing basis_universal source.")
                 endif()
 
+                # @todo Disable file loading at compile time and get rid of the
+                # BMP/JPG/PNG libraries, we don't use those at all. Hopefully
+                # this becomes a preprocessor define upstream at some point.
+                # Alternatively, look into creating stubs for the library
+                # functions used by basis_universal.
                 set(BasisUniversalEncoder_SOURCES
+                    ${BasisUniversalEncoder_DIR}/apg_bmp.c
                     ${BasisUniversalEncoder_DIR}/basisu_astc_decomp.cpp
                     ${BasisUniversalEncoder_DIR}/basisu_backend.cpp
                     ${BasisUniversalEncoder_DIR}/basisu_basis_file.cpp
+                    ${BasisUniversalEncoder_DIR}/basisu_bc7enc.cpp
                     ${BasisUniversalEncoder_DIR}/basisu_comp.cpp
                     ${BasisUniversalEncoder_DIR}/basisu_enc.cpp
                     ${BasisUniversalEncoder_DIR}/basisu_etc.cpp
                     ${BasisUniversalEncoder_DIR}/basisu_frontend.cpp
                     ${BasisUniversalEncoder_DIR}/basisu_global_selector_palette_helpers.cpp
                     ${BasisUniversalEncoder_DIR}/basisu_gpu_texture.cpp
+                    ${BasisUniversalEncoder_DIR}/basisu_kernels_sse.cpp
                     ${BasisUniversalEncoder_DIR}/basisu_pvrtc1_4.cpp
                     ${BasisUniversalEncoder_DIR}/basisu_resampler.cpp
                     ${BasisUniversalEncoder_DIR}/basisu_resample_filters.cpp
                     ${BasisUniversalEncoder_DIR}/basisu_ssim.cpp
+                    ${BasisUniversalEncoder_DIR}/basisu_uastc_enc.cpp
+                    ${BasisUniversalEncoder_DIR}/jpgd.cpp
                     ${BasisUniversalEncoder_DIR}/lodepng.cpp)
 
                 foreach(_file ${BasisUniversalEncoder_SOURCES})
@@ -173,8 +221,6 @@ foreach(_component ${BasisUniversal_FIND_COMPONENTS})
                 # The rest is documented in the BasisImageConverter plugin itself.
                 set_property(TARGET BasisUniversal::Encoder APPEND PROPERTY
                     INTERFACE_LINK_LIBRARIES BasisUniversal::Transcoder)
-                set_property(TARGET BasisUniversal::Encoder APPEND PROPERTY
-                    INTERFACE_COMPILE_DEFINITIONS "BASISU_NO_ITERATOR_DEBUG_LEVEL")
             endif()
         else()
             set(BasisUniversal_Encoder_FOUND TRUE)
@@ -227,6 +273,27 @@ foreach(_component ${BasisUniversal_FIND_COMPONENTS})
                 set(BasisUniversalTranscoder_SOURCES
                     ${BasisUniversalTranscoder_DIR}/basisu_transcoder.cpp)
 
+                set(BasisUniversalTranscoder_DEFINITIONS "BASISU_NO_ITERATOR_DEBUG_LEVEL")
+
+                # Not linking to zstddeclib.c because together with Encoder
+                # linking to zstd.c this would lead to duplicate symbol
+                # errors.
+                # @todo Unused functions *should* be removed by LTO but is
+                # there a better way?
+                find_path(BasisUniversalZstd_DIR NAMES zstd.c
+                    HINTS "${BASIS_UNIVERSAL_DIR}/zstd" "${BASIS_UNIVERSAL_DIR}"
+                    NO_CMAKE_FIND_ROOT_PATH)
+                if(BasisUniversalZstd_DIR)
+                    list(APPEND BasisUniversalTranscoder_SOURCES
+                        ${BasisUniversalZstd_DIR}/zstd.c)
+                else()
+                    # If zstd wasn't found, disable Zstandard supercompression
+                    # support at compile time. The zstd.h include is hidden
+                    # behind this definition as well.
+                    list(APPEND BasisUniversalTranscoder_DEFINITIONS
+                        "BASISD_SUPPORT_KTX2_ZSTD=0")
+                endif()
+
                 foreach(_file ${BasisUniversalTranscoder_SOURCES})
                     _basis_setup_source_file(${_file})
                 endforeach()
@@ -243,7 +310,7 @@ foreach(_component ${BasisUniversal_FIND_COMPONENTS})
                 set_property(TARGET BasisUniversal::Transcoder APPEND PROPERTY
                     INTERFACE_INCLUDE_DIRECTORIES ${BasisUniversalTranscoder_INCLUDE_DIR})
                 set_property(TARGET BasisUniversal::Transcoder APPEND PROPERTY
-                    INTERFACE_COMPILE_DEFINITIONS "BASISU_NO_ITERATOR_DEBUG_LEVEL")
+                    INTERFACE_COMPILE_DEFINITIONS ${BasisUniversalTranscoder_DEFINITIONS})
                 set_property(TARGET BasisUniversal::Transcoder APPEND PROPERTY
                     INTERFACE_SOURCES "${BasisUniversalTranscoder_SOURCES}")
             endif()

package/archlinux/magnum-plugins-git/PKGBUILD

@@ -1,7 +1,7 @@
 # Author: mosra <[email protected]>
 pkgname=magnum-plugins-git
 pkgver=2020.06.r119.g15b8cac9
-_basis_pkgver=2f43afcc97d0a5dafdb73b4e24e123cf9687a418
+_basis_pkgver=1_15_update2
 pkgrel=1
 pkgdesc="Plugins for the Magnum C++11/C++14 graphics engine (Git version)"
 arch=('i686' 'x86_64')
@@ -12,11 +12,9 @@ makedepends=('cmake' 'git' 'ninja')
 provides=('magnum-plugins')
 conflicts=('magnum-plugins')
 source=("git+git://github.com/mosra/magnum-plugins.git"
-        # A commit that's before the UASTC support (which is not implemented
-        # yet, because latest versions crash even on trivial tests)
-        "https://github.com/BinomialLLC/basis_universal/archive/${_basis_pkgver}.tar.gz")
+        "https://github.com/BinomialLLC/basis_universal/archive/v${_basis_pkgver}.tar.gz")
 sha1sums=('SKIP'
-          'b8d3995292c2c0bbedea943250087b0a9a92ca96')
+          'b9615d48ebfc62a53f333ebf8a582558a058b0e9')
 
 pkgver() {
     cd "$srcdir/${pkgname%-git}"

package/ci/appveyor.yml

@@ -82,12 +82,9 @@ install:
 - IF "%TARGET%" == "desktop" IF "%APPVEYOR_BUILD_WORKER_IMAGE%" == "Visual Studio 2017" IF "%COMPILER:~0,4%" == "msvc" appveyor DownloadFile https://ci.magnum.graphics/freetype-2.10.4-windows-2016.zip && 7z x freetype-2.10.4-windows-2016.zip -o%APPVEYOR_BUILD_FOLDER%\deps
 
 # Basis Universal
-- set BASIS_VERSION=8565af680d1bd2ad56ab227ca7d96c56dfbe93ed
-- IF NOT EXIST %APPVEYOR_BUILD_FOLDER%\basis_universal-%BASIS_VERSION%.zip appveyor DownloadFile https://github.com/BinomialLLC/basis_universal/archive/%BASIS_VERSION%.zip
-- 7z x %BASIS_VERSION%.zip && ren basis_universal-%BASIS_VERSION% basis_universal
-# https://github.com/BinomialLLC/basis_universal/pull/106
-# TODO: remove once we update to UASTC-enabled Basis
-- bash -c "cd basis_universal && patch -p1 < ../package/ci/basisu-msvc2019-16.6.patch"
+- set BASIS_VERSION=1_15_update2
+- IF NOT EXIST %APPVEYOR_BUILD_FOLDER%\v%BASIS_VERSION%.zip appveyor DownloadFile https://github.com/BinomialLLC/basis_universal/archive/v%BASIS_VERSION%.zip
+- 7z x v%BASIS_VERSION%.zip && ren basis_universal-%BASIS_VERSION% basis_universal
 
 # SPIRV-Tools, for MSVC 2019, 2017 and clang-cl only
 # This line REQUIRES the COMPILER variable to be set on rt builds, otherwise it

package/ci/circleci.yml

@@ -148,9 +148,8 @@ commands:
     steps:
     - run:
         name: Install Basis Universal
-        # Version before UASTC is a thing, as our tests crash with the new one
         command: |
-          export BASIS_VERSION=8565af680d1bd2ad56ab227ca7d96c56dfbe93ed
+          export BASIS_VERSION=v1_15_update2
           mkdir -p $HOME/basis_universal && cd $HOME/basis_universal
           wget -nc https://github.com/BinomialLLC/basis_universal/archive/$BASIS_VERSION.tar.gz
           tar --strip-components 1 -xzf $BASIS_VERSION.tar.gz

package/ci/travis.yml

@@ -196,7 +196,7 @@ install:
 - if [ "$TRAVIS_OS_NAME" == "linux" ] && [ "$TARGET" == "desktop-sanitizers" ]; then cd $HOME ; wget https://ci.magnum.graphics/spirv-tools-2020.4-ubuntu-16.04-gcc5.zip && cd $HOME/deps && unzip $HOME/spirv-tools-2020.4-ubuntu-16.04-gcc5.zip && cd $TRAVIS_BUILD_DIR ; fi
 
 # Basis Universal
-- export BASIS_VERSION=8565af680d1bd2ad56ab227ca7d96c56dfbe93ed && wget -nc https://github.com/BinomialLLC/basis_universal/archive/$BASIS_VERSION.zip && unzip -q $BASIS_VERSION; mv basis_universal-$BASIS_VERSION $HOME/basis_universal
+- export BASIS_VERSION=1_15_update2 && wget -nc https://github.com/BinomialLLC/basis_universal/archive/v$BASIS_VERSION.zip && unzip -q v$BASIS_VERSION; mv basis_universal-$BASIS_VERSION $HOME/basis_universal
 
 script:
 - if [ "$TRAVIS_OS_NAME" == "linux" ] && ( [ "$TARGET" == "desktop" ] || [ "$TARGET" == "desktop-sanitizers" ] ); then ./package/ci/unix-desktop.sh; fi

package/homebrew/magnum-plugins.rb

@@ -23,11 +23,16 @@ class MagnumPlugins < Formula
   depends_on "spirv-tools" => :recommended
 
   def install
-    # Bundle Basis Universal, a commit that's before the UASTC support (which
-    # is not implemented yet). The repo has massive useless files in its
-    # history, so we're downloading just a snapshot instead of a git clone.
-    # Also, WHY THE FUCK curl needs -L and -o?! why can't it just work?!
-    system "curl", "-L", "https://github.com/BinomialLLC/basis_universal/archive/2f43afcc97d0a5dafdb73b4e24e123cf9687a418.tar.gz", "-o", "src/external/basis-universal.tar.gz"
+    # Bundle Basis Universal, v1_15_update2 for HEAD builds, a commit that's
+    # before the UASTC support (which was not implemented yet) on 2020.06.
+    # The repo has massive useless files in its history, so we're downloading
+    # just a snapshot instead of a git clone. Also, WHY THE FUCK curl needs -L
+    # and -o?! why can't it just work?!
+    if build.head?
+      system "curl", "-L", "https://github.com/BinomialLLC/basis_universal/archive/v1_15_update2.tar.gz", "-o", "src/external/basis-universal.tar.gz"
+    else
+      system "curl", "-L", "https://github.com/BinomialLLC/basis_universal/archive/2f43afcc97d0a5dafdb73b4e24e123cf9687a418.tar.gz", "-o", "src/external/basis-universal.tar.gz"
+    end
     cd "src/external" do
       system "mkdir", "basis-universal"
       system "tar", "xzvf", "basis-universal.tar.gz", "-C", "basis-universal", "--strip-components=1"

src/MagnumPlugins/BasisImageConverter/BasisImageConverter.conf

@@ -1,17 +1,20 @@
+provides=BasisKtxImageConverter
+
 # [configuration_]
 [configuration]
-# All following options correspond to options of the basisu tool, grouped in
-# the same way. Names follow the Basis C++ API and may differ from what the
-# tool exposes.
+# All following options correspond to parameters of the `basis_compressor` C++
+# API and may differ from what the basisu tool exposes.
 
 # Options
 quality_level=128
-# sRGB images should have this enabled, turn this flag off for linear images
-perceptual=true
+# Treat images as sRGB color data, rather than linear intensity. Leave blank to
+# determine from the image format.
+perceptual=
 debug=false
+validate=false
 debug_images=false
 compute_stats=false
-compression_level=1
+compression_level=2
 
 # More options
 max_endpoint_clusters=512
@@ -21,7 +24,14 @@ y_flip=true
 # `mip_srgb` and enabling `no_selector_rdo` & `no_endpoint_rdo`
 check_for_alpha=true
 force_alpha=false
-separate_rg_to_color_alpha=false
+# Remap color channels before compression. Must be empty or 4 characters long,
+# valid characters are r,g,b,a. This replaced separate_rg_to_color_alpha,
+# for the same effect use 'rrrg'.
+swizzle=
+renormalize=false
+resample_width=
+resample_height=
+resample_factor=
 # Number of threads Basis should use during compression, 0 sets it to the
 # value returned by std::thread::hardware_concurrency(), 1 disables
 # multithreading. This value is clamped to std::thread::hardware_concurrency()
@@ -30,13 +40,18 @@ threads=1
 disable_hierarchical_endpoint_codebooks=false
 
 # Mipmap generation options
+# Generate mipmaps from the base image. If you pass custom mip levels into
+# openData, this option will be ignored. Leave blank to determine from the
+# number of levels passed to convertToData.
 mip_gen=false
-# Generate mipmaps assuming sRGB input, turn this flag off for linear images
-mip_srgb=true
+# Filter mipmaps assuming sRGB color data, rather than linear intensity. Leave
+# blank to determine from the image format.
+mip_srgb=
 mip_scale=1.0
 mip_filter=kaiser
 mip_renormalize=false
 mip_wrapping=true
+mip_fast=true
 mip_smallest_dimension=1
 
 # Backend endpoint/selector RDO codec options
@@ -53,6 +68,23 @@ global_palette_bits=8
 global_modifier_bits=8
 hybrid_selector_codebook_quality_threshold=2.0
 
+# UASTC options
+uastc=false
+pack_uastc_level=2
+pack_uastc_flags=
+rdo_uastc=false
+rdo_uastc_quality_scalar=1.0
+rdo_uastc_dict_size=4096
+rdo_uastc_max_smooth_block_error_scale=10.0
+rdo_uastc_smooth_block_max_std_dev=18.0
+rdo_uastc_max_allowed_rms_increase_ratio=10.0
+rdo_uastc_skip_block_rms_threshold=8.0
+rdo_uastc_favor_simpler_modes_in_rdo_mode=true
+
+# KTX2 options
+ktx2_uastc_supercompression=true
+ktx2_zstd_supercompression_level=6
+
 # Set various fields in the Basis file header
 userdata0=0
 userdata1=0