FFmpeg5 and CBS library

.github/workflows/build-ffmpeg.yml

@@ -512,6 +512,53 @@ jobs:
           cp ./COPYING.GPLv2 $root_path/ffmpeg_build/LICENSE
           echo "::endgroup::"
 
+      - name: cbs
+        env:
+          root_path: ${{ steps.root.outputs.root_path }}
+        run: |
+          mkdir -p build
+          cd build
+
+          echo "::group::configure extra flags for cross compilation"
+          extra_configure=""
+          if [[ ${{ matrix.os_type }} == "linux" && ${{ matrix.arch }} == "aarch64" ]]; then
+            extra_configure=$(cat <<VAREOF
+              -DCMAKE_C_COMPILER=${{ matrix.host }}-gcc
+              -DCMAKE_CXX_COMPILER=${{ matrix.host }}-g++
+          VAREOF
+          )
+          elif [[ ${{ matrix.os_type }} == "macos" && ${{ matrix.arch }} == "aarch64" ]]; then
+            export CFLAGS="${{ steps.cross.outputs.CFLAGS }}"
+            export CXXFLAGS="-std=c++11 ${CFLAGS}"
+            extra_configure=$(cat <<- VAREOF
+              -DCMAKE_SYSTEM_PROCESSOR=arm64
+              -DCMAKE_OSX_ARCHITECTURES=arm64
+              -DCMAKE_C_COMPILER=clang
+              -DCMAKE_C_COMPILER_TARGET=${{ matrix.host }}
+              -DCMAKE_CXX_COMPILER=clang++
+              -DCMAKE_CXX_COMPILER_TARGET=${{ matrix.host }}
+          VAREOF
+          )
+          fi
+          echo "$extra_configure"
+          echo "::endgroup::"
+
+          echo "::group::configure"
+          PATH="$root_path/bin:$PATH" cmake -G "${{ matrix.cmake_generator }}" \
+            $extra_configure \
+            -DCMAKE_INSTALL_PREFIX="$root_path/ffmpeg_build" \
+            -DFFMPEG_CBS=ON \
+            ..
+          echo "::endgroup::"
+
+          echo "::group::make"
+          PATH="$root_path/bin:$PATH" make -j$(nproc || sysctl -n hw.logicalcpu)
+          echo "::endgroup::"
+
+          echo "::group::make install"
+          make install
+          echo "::endgroup::"
+
       - name: Upload Artifacts
         uses: actions/upload-artifact@v3
         with:

.gitignore

@@ -1,2 +1,6 @@
+# Distribution / packaging
+build/
+
 # JetBrains project folder
 .idea/
+

.gitmodules

@@ -17,7 +17,7 @@
 [submodule "ffmpeg_sources/ffmpeg"]
 	path = ffmpeg_sources/ffmpeg
 	url = https://github.com/FFmpeg/FFmpeg
-	branch = release/4.4
+	branch = release/5.1
 [submodule "ffmpeg_sources/SVT-AV1"]
 	path = ffmpeg_sources/SVT-AV1
 	url = https://gitlab.com/AOMediaCodec/SVT-AV1.git

CMakeLists.txt

@@ -0,0 +1,11 @@
+cmake_minimum_required(VERSION 3.2)
+
+project(build-deps
+        DESCRIPTION "Pre-build dependencies for LizardByte projects"
+        )
+
+option(FFMPEG_CBS "Enable CBS library configuration" ON)
+
+if (FFMPEG_CBS)
+    include (${CMAKE_CURRENT_SOURCE_DIR}/cmake/ffmpeg_cbs.cmake)
+endif()

cmake/apply_git_patch.cmake

@@ -0,0 +1,23 @@
+#
+# This macro applies patch to git repository if patch is applicable
+# Arguments are path to git repository and path to the git patch
+#
+macro(apply_git_patch REPO_PATH PATCH_PATH)
+    execute_process(COMMAND git apply -v --ignore-whitespace --check ${PATCH_PATH}
+        WORKING_DIRECTORY ${REPO_PATH}
+        RESULT_VARIABLE SUCCESS
+        COMMAND_ECHO STDOUT)
+
+    if(${SUCCESS} EQUAL 0)
+        message("Applying git patch ${PATCH_PATH} in ${REPO_PATH} repository")
+        execute_process(COMMAND git apply -v --ignore-whitespace ${PATCH_PATH}
+            WORKING_DIRECTORY ${REPO_PATH}
+            RESULT_VARIABLE SUCCESS
+            COMMAND_ECHO STDOUT)
+
+        if(${SUCCESS} EQUAL 1)
+            # We don't stop here because it can happen in case of parallel builds
+            message(WARNING "\nError: failed to apply the patch patch: ${PATCH_PATH}\n")
+        endif()
+    endif()
+endmacro()

cmake/ffmpeg_cbs.cmake

@@ -0,0 +1,134 @@
+cmake_minimum_required(VERSION 3.2)
+
+project(cbs
+        DESCRIPTION "FFmpeg code subset to expose coded bitstream (CBS) internal APIs for Sunshine"
+        VERSION 0.1
+        )
+
+set(CMAKE_GENERATED_SRC_PATH ${CMAKE_BINARY_DIR}/generated-src)
+
+# Apply patches
+include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/apply_git_patch.cmake)
+apply_git_patch(${CMAKE_SOURCE_DIR}/ffmpeg_sources/ffmpeg ${CMAKE_SOURCE_DIR}/ffmpeg_patches/cbs/explicit_intmath.patch)
+apply_git_patch(${CMAKE_SOURCE_DIR}/ffmpeg_sources/ffmpeg ${CMAKE_SOURCE_DIR}/ffmpeg_patches/cbs/remove_register.patch)
+apply_git_patch(${CMAKE_SOURCE_DIR}/ffmpeg_sources/ffmpeg ${CMAKE_SOURCE_DIR}/ffmpeg_patches/cbs/size_specifier.patch)
+
+file(COPY ${CMAKE_SOURCE_DIR}/ffmpeg_sources/ffmpeg DESTINATION ${CMAKE_GENERATED_SRC_PATH})
+
+set(FFMPEG_GENERATED_SRC_PATH ${CMAKE_GENERATED_SRC_PATH}/ffmpeg)
+set(AVCODEC_GENERATED_SRC_PATH ${CMAKE_GENERATED_SRC_PATH}/ffmpeg/libavcodec)
+set(CBS_INCLUDE_PATH ${CMAKE_BINARY_DIR}/include/cbs)
+
+# Configure FFmpeg to generate platform-specific config
+if(NOT EXISTS ${FFMPEG_GENERATED_SRC_PATH}/config.h)
+    message("Running FFmpeg configure")
+    # Explicit shell otherwise Windows runs in the wrong terminal
+    # The output config.h needs to have `CONFIG_CBS_` flags enabled
+    execute_process(COMMAND sh ./configure
+            --disable-autodetect
+            --disable-iconv
+            --enable-gpl
+            --enable-static
+            --enable-avcodec
+            --enable-avutil
+        WORKING_DIRECTORY ${FFMPEG_GENERATED_SRC_PATH}
+        COMMAND_ECHO STDOUT
+        )
+else()
+    message("FFmpeg config.h found, skipping")
+endif()
+
+# Headers needed to link for Sunshine
+configure_file(${AVCODEC_GENERATED_SRC_PATH}/arm/mathops.h ${CBS_INCLUDE_PATH}/arm/mathops.h COPYONLY)
+configure_file(${AVCODEC_GENERATED_SRC_PATH}/x86/mathops.h ${CBS_INCLUDE_PATH}/x86/mathops.h COPYONLY)
+configure_file(${AVCODEC_GENERATED_SRC_PATH}/av1.h ${CBS_INCLUDE_PATH}/av1.h COPYONLY)
+configure_file(${AVCODEC_GENERATED_SRC_PATH}/cbs_av1.h ${CBS_INCLUDE_PATH}/cbs_av1.h COPYONLY)
+configure_file(${AVCODEC_GENERATED_SRC_PATH}/cbs_bsf.h ${CBS_INCLUDE_PATH}/cbs_bsf.h COPYONLY)
+configure_file(${AVCODEC_GENERATED_SRC_PATH}/cbs.h ${CBS_INCLUDE_PATH}/cbs.h COPYONLY)
+configure_file(${AVCODEC_GENERATED_SRC_PATH}/cbs_h2645.h ${CBS_INCLUDE_PATH}/cbs_h2645.h COPYONLY)
+configure_file(${AVCODEC_GENERATED_SRC_PATH}/cbs_h264.h ${CBS_INCLUDE_PATH}/cbs_h264.h COPYONLY)
+configure_file(${AVCODEC_GENERATED_SRC_PATH}/cbs_h265.h ${CBS_INCLUDE_PATH}/cbs_h265.h COPYONLY)
+configure_file(${AVCODEC_GENERATED_SRC_PATH}/cbs_jpeg.h ${CBS_INCLUDE_PATH}/cbs_jpeg.h COPYONLY)
+configure_file(${AVCODEC_GENERATED_SRC_PATH}/cbs_mpeg2.h ${CBS_INCLUDE_PATH}/cbs_mpeg2.h COPYONLY)
+configure_file(${AVCODEC_GENERATED_SRC_PATH}/cbs_sei.h ${CBS_INCLUDE_PATH}/cbs_sei.h COPYONLY)
+configure_file(${AVCODEC_GENERATED_SRC_PATH}/cbs_vp9.h ${CBS_INCLUDE_PATH}/cbs_vp9.h COPYONLY)
+configure_file(${AVCODEC_GENERATED_SRC_PATH}/codec_desc.h ${CBS_INCLUDE_PATH}/codec_desc.h COPYONLY)
+configure_file(${AVCODEC_GENERATED_SRC_PATH}/codec_id.h ${CBS_INCLUDE_PATH}/codec_id.h COPYONLY)
+configure_file(${AVCODEC_GENERATED_SRC_PATH}/codec_par.h ${CBS_INCLUDE_PATH}/codec_par.h COPYONLY)
+configure_file(${AVCODEC_GENERATED_SRC_PATH}/defs.h ${CBS_INCLUDE_PATH}/defs.h COPYONLY)
+configure_file(${AVCODEC_GENERATED_SRC_PATH}/get_bits.h ${CBS_INCLUDE_PATH}/get_bits.h COPYONLY)
+configure_file(${AVCODEC_GENERATED_SRC_PATH}/h264_levels.h ${CBS_INCLUDE_PATH}/h264_levels.h COPYONLY)
+configure_file(${AVCODEC_GENERATED_SRC_PATH}/h2645_parse.h ${CBS_INCLUDE_PATH}/h2645_parse.h COPYONLY)
+configure_file(${AVCODEC_GENERATED_SRC_PATH}/h264.h ${CBS_INCLUDE_PATH}/h264.h COPYONLY)
+configure_file(${AVCODEC_GENERATED_SRC_PATH}/hevc.h ${CBS_INCLUDE_PATH}/hevc.h COPYONLY)
+configure_file(${AVCODEC_GENERATED_SRC_PATH}/mathops.h ${CBS_INCLUDE_PATH}/mathops.h COPYONLY)
+configure_file(${AVCODEC_GENERATED_SRC_PATH}/packet.h ${CBS_INCLUDE_PATH}/packet.h COPYONLY)
+configure_file(${AVCODEC_GENERATED_SRC_PATH}/sei.h ${CBS_INCLUDE_PATH}/sei.h COPYONLY)
+configure_file(${AVCODEC_GENERATED_SRC_PATH}/vlc.h ${CBS_INCLUDE_PATH}/vlc.h COPYONLY)
+configure_file(${FFMPEG_GENERATED_SRC_PATH}/config.h ${CMAKE_BINARY_DIR}/include/config.h COPYONLY)
+configure_file(${FFMPEG_GENERATED_SRC_PATH}/libavutil/x86/asm.h ${CMAKE_BINARY_DIR}/include/libavutil/x86/asm.h COPYONLY)
+configure_file(${FFMPEG_GENERATED_SRC_PATH}/libavutil/x86/intmath.h ${CMAKE_BINARY_DIR}/include/libavutil/x86/intmath.h COPYONLY)
+configure_file(${FFMPEG_GENERATED_SRC_PATH}/libavutil/arm/intmath.h ${CMAKE_BINARY_DIR}/include/libavutil/arm/intmath.h COPYONLY)
+configure_file(${FFMPEG_GENERATED_SRC_PATH}/libavutil/intmath.h ${CMAKE_BINARY_DIR}/include/libavutil/intmath.h COPYONLY)
+
+set(CBS_SOURCE_FILES
+    ${CBS_INCLUDE_PATH}/arm/mathops.h
+    ${CBS_INCLUDE_PATH}/x86/mathops.h
+    ${CBS_INCLUDE_PATH}/av1.h
+    ${CBS_INCLUDE_PATH}/cbs_av1.h
+    ${CBS_INCLUDE_PATH}/cbs_bsf.h
+    ${CBS_INCLUDE_PATH}/cbs.h
+    ${CBS_INCLUDE_PATH}/cbs_h2645.h
+    ${CBS_INCLUDE_PATH}/cbs_h264.h
+    ${CBS_INCLUDE_PATH}/cbs_h265.h
+    ${CBS_INCLUDE_PATH}/cbs_jpeg.h
+    ${CBS_INCLUDE_PATH}/cbs_mpeg2.h
+    ${CBS_INCLUDE_PATH}/cbs_sei.h
+    ${CBS_INCLUDE_PATH}/cbs_vp9.h
+    ${CBS_INCLUDE_PATH}/codec_desc.h
+    ${CBS_INCLUDE_PATH}/codec_id.h
+    ${CBS_INCLUDE_PATH}/codec_par.h
+    ${CBS_INCLUDE_PATH}/defs.h
+    ${CBS_INCLUDE_PATH}/get_bits.h
+    ${CBS_INCLUDE_PATH}/h264_levels.h
+    ${CBS_INCLUDE_PATH}/h2645_parse.h
+    ${CBS_INCLUDE_PATH}/h264.h
+    ${CBS_INCLUDE_PATH}/hevc.h
+    ${CBS_INCLUDE_PATH}/mathops.h
+    ${CBS_INCLUDE_PATH}/packet.h
+    ${CBS_INCLUDE_PATH}/sei.h
+    ${CBS_INCLUDE_PATH}/vlc.h
+    ${CMAKE_BINARY_DIR}/include/config.h
+    ${CMAKE_BINARY_DIR}/include/libavutil/x86/asm.h
+    ${CMAKE_BINARY_DIR}/include/libavutil/x86/intmath.h
+    ${CMAKE_BINARY_DIR}/include/libavutil/arm/intmath.h
+    ${CMAKE_BINARY_DIR}/include/libavutil/intmath.h
+
+    ${AVCODEC_GENERATED_SRC_PATH}/cbs.c
+    ${AVCODEC_GENERATED_SRC_PATH}/cbs_h2645.c
+    ${AVCODEC_GENERATED_SRC_PATH}/cbs_av1.c
+    ${AVCODEC_GENERATED_SRC_PATH}/cbs_vp9.c
+    ${AVCODEC_GENERATED_SRC_PATH}/cbs_mpeg2.c
+    ${AVCODEC_GENERATED_SRC_PATH}/cbs_jpeg.c
+    ${AVCODEC_GENERATED_SRC_PATH}/cbs_sei.c
+    ${AVCODEC_GENERATED_SRC_PATH}/h264_levels.c
+    ${AVCODEC_GENERATED_SRC_PATH}/h2645_parse.c
+    ${FFMPEG_GENERATED_SRC_PATH}/libavutil/intmath.c
+)
+
+include_directories(
+    ${CBS_INCLUDE_PATH}
+    ${FFMPEG_GENERATED_SRC_PATH}
+)
+
+add_library(cbs ${CBS_SOURCE_FILES})
+target_compile_options(cbs PRIVATE -Wall -Wno-incompatible-pointer-types -Wno-format -Wno-format-extra-args)
+
+install(DIRECTORY ${CMAKE_BINARY_DIR}/include
+        DESTINATION ${CMAKE_INSTALL_PREFIX})
+install(FILES ${CMAKE_BINARY_DIR}/libcbs.a
+        DESTINATION ${CMAKE_INSTALL_PREFIX}/lib)
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/cmake/libcbs.pc.in
+        ${CMAKE_BINARY_DIR}/libcbs.pc @ONLY)
+install(FILES ${CMAKE_BINARY_DIR}/libcbs.pc
+        DESTINATION ${CMAKE_INSTALL_PREFIX}/lib/pkgconfig)

cmake/libcbs.pc.in

@@ -0,0 +1,12 @@
+# Based on https://www.scivision.dev/cmake-generate-pkg-config/
+
+prefix=@CMAKE_INSTALL_PREFIX@
+exec_prefix="${prefix}"
+libdir="${prefix}/lib"
+includedir="${prefix}/include"
+
+Name: @PROJECT_NAME@
+Description: @CMAKE_PROJECT_DESCRIPTION@
+Version: @PROJECT_VERSION@
+Cflags: -I"${includedir}"
+Libs: -L"${libdir}" -lcbs

ffmpeg_patches/cbs/explicit_intmath.patch

@@ -0,0 +1,39 @@
+Explicitly imports intmath as our subset of sources doesn't have it included
+
+diff --git a/libavcodec/cbs_av1.c b/libavcodec/cbs_av1.c
+index 1229480567..600a24c47a 100644
+--- a/libavcodec/cbs_av1.c
++++ b/libavcodec/cbs_av1.c
+@@ -17,6 +17,7 @@
+  */
+
+ #include "libavutil/avassert.h"
++#include "libavutil/intmath.h"
+ #include "libavutil/opt.h"
+ #include "libavutil/pixfmt.h"
+
+diff --git a/libavcodec/cbs_h2645.c b/libavcodec/cbs_h2645.c
+index 12e38c80b5..03970520e7 100644
+--- a/libavcodec/cbs_h2645.c
++++ b/libavcodec/cbs_h2645.c
+@@ -18,6 +18,7 @@
+
+ #include "libavutil/attributes.h"
+ #include "libavutil/avassert.h"
++#include "libavutil/intmath.h"
+
+ #include "bytestream.h"
+ #include "cbs.h"
+diff --git a/libavcodec/cbs_sei_syntax_template.c b/libavcodec/cbs_sei_syntax_template.c
+index 0ef7b42ed9..b6242367c8 100644
+--- a/libavcodec/cbs_sei_syntax_template.c
++++ b/libavcodec/cbs_sei_syntax_template.c
+@@ -16,6 +16,8 @@
+  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+  */
+
++#include "libavutil/intmath.h"
++
+ static int FUNC(filler_payload)
+     (CodedBitstreamContext *ctx, RWContext *rw,
+      SEIRawFillerPayload *current, SEIMessageState *state)

ffmpeg_patches/cbs/remove_register.patch

@@ -0,0 +1,64 @@
+Register storage specifier is an error for modern compiler and generally ignored anyway.
+
+diff --git a/libavcodec/get_bits.h b/libavcodec/get_bits.h
+index 992765dc92..fffa461bbe 100644
+--- a/libavcodec/get_bits.h
++++ b/libavcodec/get_bits.h
+@@ -327,8 +327,8 @@ static inline int get_xbits(GetBitContext *s, int n)
+
+     return ((((uint32_t)(sign ^ cache)) >> (32 - n)) ^ sign) - sign;
+ #else
+-    register int sign;
+-    register int32_t cache;
++    int sign;
++    int32_t cache;
+     OPEN_READER(re, s);
+     av_assert2(n>0 && n<=25);
+     UPDATE_CACHE(re, s);
+@@ -343,8 +343,8 @@ static inline int get_xbits(GetBitContext *s, int n)
+ #if !CACHED_BITSTREAM_READER
+ static inline int get_xbits_le(GetBitContext *s, int n)
+ {
+-    register int sign;
+-    register int32_t cache;
++    int sign;
++    int32_t cache;
+     OPEN_READER(re, s);
+     av_assert2(n>0 && n<=25);
+     UPDATE_CACHE_LE(re, s);
+@@ -358,7 +358,7 @@ static inline int get_xbits_le(GetBitContext *s, int n)
+
+ static inline int get_sbits(GetBitContext *s, int n)
+ {
+-    register int tmp;
++    int tmp;
+ #if CACHED_BITSTREAM_READER
+     av_assert2(n>0 && n<=25);
+     tmp = sign_extend(get_bits(s, n), n);
+@@ -378,7 +378,7 @@ static inline int get_sbits(GetBitContext *s, int n)
+  */
+ static inline unsigned int get_bits(GetBitContext *s, int n)
+ {
+-    register unsigned int tmp;
++    unsigned int tmp;
+ #if CACHED_BITSTREAM_READER
+
+     av_assert2(n>0 && n<=32);
+@@ -429,7 +429,7 @@ static inline unsigned int get_bits_le(GetBitContext *s, int n)
+
+     return get_val(s, n, 1);
+ #else
+-    register int tmp;
++    int tmp;
+     OPEN_READER(re, s);
+     av_assert2(n>0 && n<=25);
+     UPDATE_CACHE_LE(re, s);
+@@ -445,7 +445,7 @@ static inline unsigned int get_bits_le(GetBitContext *s, int n)
+  */
+ static inline unsigned int show_bits(GetBitContext *s, int n)
+ {
+-    register unsigned int tmp;
++    unsigned int tmp;
+ #if CACHED_BITSTREAM_READER
+     if (n > s->bits_left)
+ #ifdef BITSTREAM_READER_LE