Add multithreading to libFLAC

.github/workflows/distcheck.yml

@@ -35,9 +35,9 @@ jobs:
         run: |
           make
           unxz --keep test/abi/abi-libFLAC-1.4.0.dump.xz
-          unxz --keep test/abi/abi-libFLAC++-1.4.0.dump.xz
-          abi-compliance-checker -l flac -old test/abi/abi-libFLAC-1.4.0.dump -new test/abi/abi-descriptor-libFLAC-1.4.3.xml
-          abi-compliance-checker -l flac++ -old test/abi/abi-libFLAC++-1.4.0.dump -new test/abi/abi-descriptor-libFLAC++-1.4.3.xml
+          unxz --keep test/abi/abi-libFLAC++-1.4.4.dump.xz
+          abi-compliance-checker -l flac -old test/abi/abi-libFLAC-1.4.0.dump -new test/abi/abi-descriptor-libFLAC-1.4.4.xml
+          abi-compliance-checker -l flac++ -old test/abi/abi-libFLAC++-1.4.4.dump -new test/abi/abi-descriptor-libFLAC++-1.4.4.xml
 
       - name: Check with flac test files
         run: ./src/flac/flac -t test-files/subset/*.flac test-files/uncommon/0[5-9]*.flac test-files/uncommon/10*.flac
@@ -56,3 +56,4 @@ jobs:
           name: flac-${{ github.sha }}-${{ github.run_id }}-logs
           path: |
             ./flac-**/**/*.log
+            ./compat_reports

CMakeLists.txt

@@ -24,6 +24,7 @@ option(INSTALL_PKGCONFIG_MODULES "Install PkgConfig modules" ON)
 option(INSTALL_CMAKE_CONFIG_MODULE "Install CMake package-config module" ON)
 option(WITH_OGG "ogg support (default: test for libogg)" ON)
 option(BUILD_SHARED_LIBS "Build shared instead of static libraries" OFF)
+option(ENABLE_MULTITHREADING "Enable multithreading if pthreads is available" ON)
 
 set(VERSION ${PROJECT_VERSION})
 
@@ -113,6 +114,7 @@ check_include_file("inttypes.h" HAVE_INTTYPES_H)
 check_include_file("stdint.h" HAVE_STDINT_H)
 check_include_file("stdbool.h" HAVE_STDBOOL_H)
 check_include_file("arm_neon.h" FLAC__HAS_NEONINTRIN)
+check_include_file("semaphore.h" HAVE_SEMAPHORE_H)
 
 if(NOT HAVE_STDINT_H OR NOT HAVE_STDBOOL_H)
     message(SEND_ERROR "Header stdint.h and/or stdbool.h not found")
@@ -198,6 +200,15 @@ if(CMAKE_BUILD_TYPE STREQUAL Debug OR CMAKE_BUILD_TYPE STREQUAL RelWithDebInfo)
     add_definitions(-DFLAC__OVERFLOW_DETECT)
 endif()
 
+if(ENABLE_MULTITHREADING AND HAVE_SEMAPHORE_H)
+    set(CMAKE_THREAD_PREFER_PTHREAD TRUE)
+    set(THREADS_PREFER_PTHREAD_FLAG TRUE)
+    find_package(Threads)
+    if(CMAKE_USE_PTHREADS_INIT)
+        set(HAVE_PTHREAD 1)
+    endif()
+endif()
+
 add_subdirectory("src")
 add_subdirectory("microbench")
 if(BUILD_DOCS)

config.cmake.h.in

@@ -88,6 +88,9 @@
 /* Define to 1 if you have the <memory.h> header file. */
 #cmakedefine HAVE_MEMORY_H
 
+/* Define if multithreading is enabled */
+#cmakedefine HAVE_PTHREAD 1
+
 /* Define to 1 if you have the <stdint.h> header file. */
 #cmakedefine HAVE_STDINT_H
 

configure.ac

@@ -53,7 +53,7 @@ AM_PROG_CC_C_O
 AC_C_INLINE
 AC_C_TYPEOF
 
-AC_CHECK_HEADERS([stdint.h stdbool.h inttypes.h byteswap.h sys/auxv.h sys/param.h sys/ioctl.h termios.h x86intrin.h cpuid.h arm_neon.h])
+AC_CHECK_HEADERS([stdint.h stdbool.h inttypes.h byteswap.h sys/auxv.h sys/param.h sys/ioctl.h termios.h x86intrin.h cpuid.h arm_neon.h semaphore.h])
 
 if test "x$ac_cv_header_stdint_h" != xyes -o "x$ac_cv_header_stdbool_h" != xyes; then
 AC_MSG_ERROR("Header stdint.h and/or stdbool.h not found")
@@ -371,6 +371,29 @@ AC_ARG_ENABLE([version-from-git],
 	AS_HELP_STRING([--disable-version-from-git], [Do not use git tag, commit hash and commit date for version number]),
        [ enable_version_from_git=$enableval ], [ enable_version_from_git=yes ])
 
+dnl Build with multithreading?
+AC_ARG_ENABLE([multithreading],
+	AS_HELP_STRING([--disable-multithreading], [Disable multithreading in libFLAC]))
+
+HAVE_PTHREAD=no
+if test "x$enable_multithreading" != "xno" ; then
+	if test "x$ac_cv_header_semaphore_h" != "xyes"; then
+		AC_MSG_WARN("Header semaphore.h, needed for multithreading, is not found")
+	else
+		AX_PTHREAD([
+			HAVE_PTHREAD=yes
+			AC_DEFINE(HAVE_PTHREAD,1,[Define if pthread is enabled])
+			LIBS="$PTHREAD_LIBS $LIBS"
+			CFLAGS="$CFLAGS $PTHREAD_CFLAGS"
+			CXXFLAGS="$CXXFLAGS $PTHREAD_CFLAGS"
+			CC="$PTHREAD_CC"
+			CXX="$PTHREAD_CXX"
+			],[HAVE_PTHREAD=no])
+		if test "x${HAVE_PTHREAD}" == "xno"; then
+			AC_MSG_WARN("pthread support, needed for multithreading, is not found")
+		fi
+	fi
+fi
 
 dnl check for i18n(internationalization); these are from libiconv/gettext
 AM_ICONV
@@ -597,6 +620,7 @@ fi
 	echo "    Compiler is Clang : ....................... ${xiph_cv_c_compiler_clang}"
 	echo "    Asm optimizations : ....................... ${asm_optimisation}"
 	echo "    Ogg/FLAC support : ........................ ${have_ogg}"
+	echo "    Multithreading   : ........................ ${HAVE_PTHREAD}"
 	echo "    Stack protector  : ........................ ${enable_stack_smash_protection}"
 	echo "    Fuzzing support (Clang only) : ............ ${have_oss_fuzzers}"
 echo

include/FLAC++/encoder.h

@@ -148,6 +148,7 @@ namespace FLAC {
 			virtual bool set_metadata(::FLAC__StreamMetadata **metadata, uint32_t num_blocks);    ///< See FLAC__stream_encoder_set_metadata()
 			virtual bool set_metadata(FLAC::Metadata::Prototype **metadata, uint32_t num_blocks); ///< See FLAC__stream_encoder_set_metadata()
 			virtual bool set_limit_min_bitrate(bool value);                 ///< See FLAC__stream_encoder_set_limit_min_bitrate()
+			virtual uint32_t set_num_threads(uint32_t value);                       ///< See FLAC__stream_encoder_set_num_threads()
 
 			/* get_state() is not virtual since we want subclasses to be able to return their own state */
 			State get_state() const;                                   ///< See FLAC__stream_encoder_get_state()
@@ -171,6 +172,7 @@ namespace FLAC {
 			virtual uint32_t get_rice_parameter_search_dist() const;   ///< See FLAC__stream_encoder_get_rice_parameter_search_dist()
 			virtual FLAC__uint64 get_total_samples_estimate() const;   ///< See FLAC__stream_encoder_get_total_samples_estimate()
 			virtual bool     get_limit_min_bitrate() const;            ///< See FLAC__stream_encoder_get_limit_min_bitrate()
+			virtual uint32_t get_num_threads() const;                  ///< See FLAC__stream_encoder_get_num_threads()
 
 			virtual ::FLAC__StreamEncoderInitStatus init();            ///< See FLAC__stream_encoder_init_stream()
 			virtual ::FLAC__StreamEncoderInitStatus init_ogg();        ///< See FLAC__stream_encoder_init_ogg_stream()

include/FLAC++/export.h

@@ -91,8 +91,8 @@
 /** These \#defines will mirror the libtool-based library version number, see
  * http://www.gnu.org/software/libtool/manual/libtool.html#Libtool-versioning
  */
-#define FLACPP_API_VERSION_CURRENT 10
-#define FLACPP_API_VERSION_REVISION 1 /**< see above */
+#define FLACPP_API_VERSION_CURRENT 11
+#define FLACPP_API_VERSION_REVISION 0 /**< see above */
 #define FLACPP_API_VERSION_AGE 0 /**< see above */
 
 /* \} */

include/FLAC/export.h

@@ -95,9 +95,9 @@
 /** These \#defines will mirror the libtool-based library version number, see
  * http://www.gnu.org/software/libtool/manual/libtool.html#Libtool-versioning
  */
-#define FLAC_API_VERSION_CURRENT 13
+#define FLAC_API_VERSION_CURRENT 14
 #define FLAC_API_VERSION_REVISION 0 /**< see above */
-#define FLAC_API_VERSION_AGE 1 /**< see above */
+#define FLAC_API_VERSION_AGE 2 /**< see above */
 
 #ifdef __cplusplus
 extern "C" {

include/FLAC/stream_encoder.h

@@ -288,6 +288,11 @@ typedef enum {
 extern FLAC_API const char * const FLAC__StreamEncoderStateString[];
 
 
+#define FLAC__STREAM_ENCODER_SET_NUM_THREADS_OK 0
+#define FLAC__STREAM_ENCODER_SET_NUM_THREADS_NOT_COMPILED_WITH_MULTITHREADING_ENABLED 1
+#define FLAC__STREAM_ENCODER_SET_NUM_THREADS_ALREADY_INITIALIZED 2
+#define FLAC__STREAM_ENCODER_SET_NUM_THREADS_TOO_MANY_THREADS 3
+
 /** Possible return values for the FLAC__stream_encoder_init_*() functions.
  */
 typedef enum {
@@ -1102,6 +1107,52 @@ FLAC_API FLAC__bool FLAC__stream_encoder_set_min_residual_partition_order(FLAC__
  */
 FLAC_API FLAC__bool FLAC__stream_encoder_set_max_residual_partition_order(FLAC__StreamEncoder *encoder, uint32_t value);
 
+/** Set the maximum number of threads to use during encoding.
+ *  Set to a value different than 1 to enable multithreaded encoding.
+ *
+ *  Note that enabling multithreading encoding (i.e., setting a value
+ *  different than 1) results in the behaviour of
+ *  FLAC__stream_encoder_finish(), FLAC__stream_encoder_process(),
+ *  FLAC__stream_encoder_process_interleaved() subtly changing.
+ *  For example, calling one of the process functions with enough
+ *  samples to fill a block might not always result in a call to
+ *  the write callback with a frame coding these samples. Instead,
+ *  blocks with samples are distributed among worker threads,
+ *  meaning that the first few calls to those functions will
+ *  return very quickly, while later calls might block if all
+ *  threads are occupied. Also, certain calls to the process
+ *  functions will results in several calls to the write callback,
+ *  while subsequent calls might again return very quickly with no
+ *  calls to the write callback.
+ *
+ *  Also, a call to FLAC__stream_encoder_finish() blocks while
+ *  waiting for all threads to finish, and therefore might take much
+ *  longer than when not multithreading and result in multiple calls
+ *  to the write callback.
+ *
+ *  Calls to the write callback are guaranteed to be in the correct
+ *  order.
+ *
+ *  Currently, passing a value of 0 is synonymous with a value of 1,
+ *  but this might change in the future.
+ *
+ * \default \c 1
+ * \param  encoder  An encoder instance to set.
+ * \param  value    See above.
+ * \assert
+ *    \code encoder != NULL \endcode
+ * \retval uint32_t
+ *    - \c FLAC__STREAM_ENCODER_SET_NUM_THREADS_OK if the number of threads was set correctly,
+ *    - \c FLAC__STREAM_ENCODER_SET_NUM_THREADS_NOT_COMPILED_WITH_MULTITHREADING_ENABLED when
+ *    multithreading was not enabled at compilation,
+ *    - \c FLAC__STREAM_ENCODER_SET_NUM_THREADS_ALREADY_INITIALIZED when the encoder was
+ *    already initialized,
+ *    - \c FLAC__STREAM_ENCODER_SET_NUM_THREADS_TOO_MANY_THREADS when
+ *    the number of threads was larger than the maximum allowed number of threads (currently
+ *    128).
+ */
+FLAC_API uint32_t FLAC__stream_encoder_set_num_threads(FLAC__StreamEncoder *encoder, uint32_t value);
+
 /** Deprecated.  Setting this value has no effect.
  *
  * \default \c 0
@@ -1435,6 +1486,16 @@ FLAC_API uint32_t FLAC__stream_encoder_get_min_residual_partition_order(const FL
  */
 FLAC_API uint32_t FLAC__stream_encoder_get_max_residual_partition_order(const FLAC__StreamEncoder *encoder);
 
+/** Get maximum number of threads setting.
+ *
+ * \param  encoder  An encoder instance to query.
+ * \assert
+ *    \code encoder != NULL \endcode
+ * \retval uint32_t
+ *    See FLAC__stream_encoder_set_num_threads().
+ */
+FLAC_API uint32_t FLAC__stream_encoder_get_num_threads(const FLAC__StreamEncoder *encoder);
+
 /** Get the Rice parameter search distance setting.
  *
  * \param  encoder  An encoder instance to query.