Add scalabilityMode support for AV1/VP9. (#90)

cloudwebrtc · cloudwebrtc · commit 7159977e07d3 · 2023-09-14T21:19:42.000+08:00
* add scalabilityMode for AV1.

 * fix bug for scalability-mode.

 * add scalability-mode support for VP9.

 * wip: ScalabilityModes for android.

 * update.

 * wip.

 * wip.

 * wip.

 * wip.

 * done.

 * fix

 * update.
diff --git a/api/video_codecs/video_encoder_factory.h b/api/video_codecs/video_encoder_factory.h
@@ -85,13 +85,8 @@ class VideoEncoderFactory {
   virtual CodecSupport QueryCodecSupport(
       const SdpVideoFormat& format,
       absl::optional<std::string> scalability_mode) const {
-    // Default implementation, query for supported formats and check if the
-    // specified format is supported. Returns false if scalability_mode is
-    // specified.
     CodecSupport codec_support;
-    if (!scalability_mode) {
-      codec_support.is_supported = format.IsCodecInList(GetSupportedFormats());
-    }
+    codec_support.is_supported = format.IsCodecInList(GetSupportedFormats());
     return codec_support;
   }
 
diff --git a/sdk/android/api/org/webrtc/HardwareVideoEncoderFactory.java b/sdk/android/api/org/webrtc/HardwareVideoEncoderFactory.java
@@ -143,11 +143,11 @@ public VideoCodecInfo[] getSupportedCodecs() {
         // supported by the decoder.
         if (type == VideoCodecMimeType.H264 && isH264HighProfileSupported(codec)) {
           supportedCodecInfos.add(new VideoCodecInfo(
-              name, MediaCodecUtils.getCodecProperties(type, /* highProfile= */ true)));
+              name, MediaCodecUtils.getCodecProperties(type, /* highProfile= */ true), new ArrayList<>()));
         }
 
         supportedCodecInfos.add(new VideoCodecInfo(
-            name, MediaCodecUtils.getCodecProperties(type, /* highProfile= */ false)));
+            name, MediaCodecUtils.getCodecProperties(type, /* highProfile= */ false), new ArrayList<>()));
       }
     }
 
diff --git a/sdk/android/api/org/webrtc/LibaomAv1Encoder.java b/sdk/android/api/org/webrtc/LibaomAv1Encoder.java
@@ -9,6 +9,7 @@
  */
 
 package org.webrtc;
+import java.util.List;
 
 public class LibaomAv1Encoder extends WrappedNativeVideoEncoder {
   @Override
@@ -22,4 +23,10 @@ public long createNativeVideoEncoder() {
   public boolean isHardwareEncoder() {
     return false;
   }
+
+  static List<String> scalabilityModes() {
+    return nativeGetSupportedScalabilityModes();
+  }
+
+  static native List<String> nativeGetSupportedScalabilityModes();
 }
diff --git a/sdk/android/api/org/webrtc/LibvpxVp9Encoder.java b/sdk/android/api/org/webrtc/LibvpxVp9Encoder.java
@@ -9,6 +9,7 @@
  */
 
 package org.webrtc;
+import java.util.List;
 
 public class LibvpxVp9Encoder extends WrappedNativeVideoEncoder {
   @Override
@@ -24,4 +25,10 @@ public boolean isHardwareEncoder() {
   }
 
   static native boolean nativeIsSupported();
+
+  static List<String> scalabilityModes() {
+    return nativeGetSupportedScalabilityModes();
+  }
+
+  static native List<String> nativeGetSupportedScalabilityModes();
 }
diff --git a/sdk/android/api/org/webrtc/RtpParameters.java b/sdk/android/api/org/webrtc/RtpParameters.java
@@ -76,6 +76,8 @@ public static class Encoding {
     // If non-null, scale the width and height down by this factor for video. If null,
     // implementation default scaling factor will be used.
     @Nullable public Double scaleResolutionDownBy;
+    // Scalability modes are used to represent simulcast and SVC layers.
+    @Nullable public String scalabilityMode;
     // SSRC to be used by this encoding.
     // Can't be changed between getParameters/setParameters.
     public Long ssrc;
@@ -93,8 +95,8 @@ public Encoding(String rid, boolean active, Double scaleResolutionDownBy) {
     @CalledByNative("Encoding")
     Encoding(String rid, boolean active, double bitratePriority, @Priority int networkPriority,
         Integer maxBitrateBps, Integer minBitrateBps, Integer maxFramerate,
-        Integer numTemporalLayers, Double scaleResolutionDownBy, Long ssrc,
-        boolean adaptiveAudioPacketTime) {
+        Integer numTemporalLayers, Double scaleResolutionDownBy, String scalabilityMode,
+        Long ssrc, boolean adaptiveAudioPacketTime) {
       this.rid = rid;
       this.active = active;
       this.bitratePriority = bitratePriority;
@@ -104,6 +106,7 @@ public Encoding(String rid, boolean active, Double scaleResolutionDownBy) {
       this.maxFramerate = maxFramerate;
       this.numTemporalLayers = numTemporalLayers;
       this.scaleResolutionDownBy = scaleResolutionDownBy;
+      this.scalabilityMode = scalabilityMode;
       this.ssrc = ssrc;
       this.adaptiveAudioPacketTime = adaptiveAudioPacketTime;
     }
@@ -160,6 +163,12 @@ Double getScaleResolutionDownBy() {
       return scaleResolutionDownBy;
     }
 
+    @Nullable
+    @CalledByNative("Encoding")
+    String getScalabilityMode() {
+      return scalabilityMode;
+    }
+
     @CalledByNative("Encoding")
     Long getSsrc() {
       return ssrc;
diff --git a/sdk/android/api/org/webrtc/VideoCodecInfo.java b/sdk/android/api/org/webrtc/VideoCodecInfo.java
@@ -14,6 +14,8 @@
 import java.util.Arrays;
 import java.util.Locale;
 import java.util.Map;
+import java.util.List;
+import java.util.ArrayList;
 
 /**
  * Represent a video codec as encoded in SDP.
@@ -34,20 +36,24 @@ public class VideoCodecInfo {
 
   public final String name;
   public final Map<String, String> params;
+  public final List<String> scalabilityModes;
+
   @Deprecated public final int payload;
 
   @CalledByNative
-  public VideoCodecInfo(String name, Map<String, String> params) {
+  public VideoCodecInfo(String name, Map<String, String> params, List<String> scalabilityModes) {
     this.payload = 0;
     this.name = name;
     this.params = params;
+    this.scalabilityModes = scalabilityModes;
   }
 
   @Deprecated
   public VideoCodecInfo(int payload, String name, Map<String, String> params) {
     this.payload = payload;
     this.name = name;
     this.params = params;
+    this.scalabilityModes = new ArrayList<>();
   }
 
   @Override
@@ -83,4 +89,9 @@ String getName() {
   Map getParams() {
     return params;
   }
+
+  @CalledByNative
+  List<String> getScalabilityModes() {
+    return scalabilityModes;
+  }
 }
diff --git a/sdk/android/instrumentationtests/src/org/webrtc/AndroidVideoDecoderInstrumentationTest.java b/sdk/android/instrumentationtests/src/org/webrtc/AndroidVideoDecoderInstrumentationTest.java
@@ -48,7 +48,7 @@ public AndroidVideoDecoderInstrumentationTest(String codecName, boolean useEglCo
     if (codecName.equals("H264")) {
       this.codecType = H264Utils.DEFAULT_H264_BASELINE_PROFILE_CODEC;
     } else {
-      this.codecType = new VideoCodecInfo(codecName, new HashMap<>());
+      this.codecType = new VideoCodecInfo(codecName, new HashMap<>(), new ArrayList<>());
     }
     this.useEglContext = useEglContext;
   }
diff --git a/sdk/android/instrumentationtests/src/org/webrtc/DefaultVideoEncoderFactoryTest.java b/sdk/android/instrumentationtests/src/org/webrtc/DefaultVideoEncoderFactoryTest.java
@@ -47,7 +47,7 @@ public void setUp() {
   @SmallTest
   @Test
   public void getSupportedCodecs_hwVp8SameParamsAsSwVp8_oneVp8() {
-    VideoCodecInfo hwVp8Encoder = new VideoCodecInfo("VP8", new HashMap<>());
+    VideoCodecInfo hwVp8Encoder = new VideoCodecInfo("VP8", new HashMap<>(), new ArrayList<>());
     VideoEncoderFactory hwFactory = new CustomHardwareVideoEncoderFactory(hwVp8Encoder);
     DefaultVideoEncoderFactory defFactory = new DefaultVideoEncoderFactory(hwFactory);
     VideoCodecInfo[] supportedCodecs = defFactory.getSupportedCodecs();
@@ -62,7 +62,7 @@ public void getSupportedCodecs_hwVp8SameParamsAsSwVp8_oneVp8() {
   public void getSupportedCodecs_hwVp8WithDifferentParams_twoVp8() {
     VideoCodecInfo hwVp8Encoder = new VideoCodecInfo("VP8", new HashMap<String, String>() {
       { put("param", "value"); }
-    });
+    }, new ArrayList<>());
     VideoEncoderFactory hwFactory = new CustomHardwareVideoEncoderFactory(hwVp8Encoder);
     DefaultVideoEncoderFactory defFactory = new DefaultVideoEncoderFactory(hwFactory);
     VideoCodecInfo[] supportedCodecs = defFactory.getSupportedCodecs();
diff --git a/sdk/android/instrumentationtests/src/org/webrtc/SoftwareVideoDecoderFactoryTest.java b/sdk/android/instrumentationtests/src/org/webrtc/SoftwareVideoDecoderFactoryTest.java
@@ -14,6 +14,7 @@
 
 import androidx.annotation.Nullable;
 import androidx.test.filters.SmallTest;
+import java.util.ArrayList;
 import java.util.HashMap;
 import org.junit.Before;
 import org.junit.Test;
@@ -55,7 +56,7 @@ public void createDecoder_supportedCodec_returnsNotNull() {
   @Test
   public void createDecoder_unsupportedCodec_returnsNull() {
     VideoDecoderFactory factory = new SoftwareVideoDecoderFactory();
-    VideoCodecInfo codec = new VideoCodecInfo("unsupported", new HashMap<String, String>());
+    VideoCodecInfo codec = new VideoCodecInfo("unsupported", new HashMap<String, String>(), new ArrayList<>());
     VideoDecoder decoder = factory.createDecoder(codec);
     assertThat(decoder).isNull();
   }
diff --git a/sdk/android/instrumentationtests/src/org/webrtc/SoftwareVideoEncoderFactoryTest.java b/sdk/android/instrumentationtests/src/org/webrtc/SoftwareVideoEncoderFactoryTest.java
@@ -14,6 +14,7 @@
 
 import androidx.annotation.Nullable;
 import androidx.test.filters.SmallTest;
+import java.util.ArrayList;
 import java.util.HashMap;
 import org.junit.Before;
 import org.junit.Test;
@@ -52,7 +53,7 @@ public void createEncoder_supportedCodec_returnsNotNull() {
   @Test
   public void createEncoder_unsupportedCodec_returnsNull() {
     VideoEncoderFactory factory = new SoftwareVideoEncoderFactory();
-    VideoCodecInfo codec = new VideoCodecInfo("unsupported", new HashMap<String, String>());
+    VideoCodecInfo codec = new VideoCodecInfo("unsupported", new HashMap<String, String>(), new ArrayList<>());
     VideoEncoder encoder = factory.createEncoder(codec);
     assertThat(encoder).isNull();
   }
diff --git a/sdk/android/native_unittests/org/webrtc/CodecsWrapperTestHelper.java b/sdk/android/native_unittests/org/webrtc/CodecsWrapperTestHelper.java
@@ -12,6 +12,7 @@
 
 import java.util.HashMap;
 import java.util.Map;
+import java.util.ArrayList;
 
 public class CodecsWrapperTestHelper {
   @CalledByNative
@@ -20,7 +21,7 @@ public static VideoCodecInfo createTestVideoCodecInfo() {
     params.put(
         VideoCodecInfo.H264_FMTP_PROFILE_LEVEL_ID, VideoCodecInfo.H264_CONSTRAINED_BASELINE_3_1);
 
-    VideoCodecInfo codec_info = new VideoCodecInfo("H264", params);
+    VideoCodecInfo codec_info = new VideoCodecInfo("H264", params, new ArrayList<>());
     return codec_info;
   }
 
diff --git a/sdk/android/src/java/org/webrtc/H264Utils.java b/sdk/android/src/java/org/webrtc/H264Utils.java
@@ -12,6 +12,7 @@
 
 import java.util.Map;
 import java.util.HashMap;
+import java.util.ArrayList;
 
 /** Container for static helper functions related to dealing with H264 codecs. */
 class H264Utils {
@@ -38,9 +39,9 @@ public static Map<String, String> getDefaultH264Params(boolean isHighProfile) {
   }
 
   public static VideoCodecInfo DEFAULT_H264_BASELINE_PROFILE_CODEC =
-      new VideoCodecInfo("H264", getDefaultH264Params(/* isHighProfile= */ false));
+      new VideoCodecInfo("H264", getDefaultH264Params(/* isHighProfile= */ false), new ArrayList<>());
   public static VideoCodecInfo DEFAULT_H264_HIGH_PROFILE_CODEC =
-      new VideoCodecInfo("H264", getDefaultH264Params(/* isHighProfile= */ true));
+      new VideoCodecInfo("H264", getDefaultH264Params(/* isHighProfile= */ true), new ArrayList<>());
 
   public static boolean isSameH264Profile(
       Map<String, String> params1, Map<String, String> params2) {
diff --git a/sdk/android/src/java/org/webrtc/MediaCodecVideoDecoderFactory.java b/sdk/android/src/java/org/webrtc/MediaCodecVideoDecoderFactory.java
@@ -72,11 +72,11 @@ public VideoCodecInfo[] getSupportedCodecs() {
         String name = type.name();
         if (type == VideoCodecMimeType.H264 && isH264HighProfileSupported(codec)) {
           supportedCodecInfos.add(new VideoCodecInfo(
-              name, MediaCodecUtils.getCodecProperties(type, /* highProfile= */ true)));
+              name, MediaCodecUtils.getCodecProperties(type, /* highProfile= */ true), new ArrayList<>()));
         }
 
         supportedCodecInfos.add(new VideoCodecInfo(
-            name, MediaCodecUtils.getCodecProperties(type, /* highProfile= */ false)));
+            name, MediaCodecUtils.getCodecProperties(type, /* highProfile= */ false), new ArrayList<>()));
       }
     }
 
diff --git a/sdk/android/src/jni/libaom_av1_encoder.cc b/sdk/android/src/jni/libaom_av1_encoder.cc
@@ -15,12 +15,24 @@
 #include "sdk/android/generated_libaom_av1_encoder_jni/LibaomAv1Encoder_jni.h"
 #include "sdk/android/src/jni/jni_helpers.h"
 
+#include<vector>
+#include<string>
+
 namespace webrtc {
 namespace jni {
 
 static jlong JNI_LibaomAv1Encoder_CreateEncoder(JNIEnv* jni) {
   return jlongFromPointer(webrtc::CreateLibaomAv1Encoder().release());
 }
 
+static  webrtc::ScopedJavaLocalRef<jobject> JNI_LibaomAv1Encoder_GetSupportedScalabilityModes(JNIEnv* jni) {
+  std::vector<std::string> modes;
+   for (const auto scalability_mode : webrtc::kAllScalabilityModes) {
+      if (webrtc::ScalabilityStructureConfig(scalability_mode).has_value()) {
+       modes.push_back(std::string(webrtc::ScalabilityModeToString(scalability_mode)));
+      }
+    }
+  return NativeToJavaStringArray(jni, modes);
+}
 }  // namespace jni
 }  // namespace webrtc
diff --git a/sdk/android/src/jni/pc/rtp_parameters.cc b/sdk/android/src/jni/pc/rtp_parameters.cc
@@ -53,6 +53,7 @@ ScopedJavaLocalRef<jobject> NativeToJavaRtpEncodingParameter(
       NativeToJavaInteger(env, encoding.max_framerate),
       NativeToJavaInteger(env, encoding.num_temporal_layers),
       NativeToJavaDouble(env, encoding.scale_resolution_down_by),
+      NativeToJavaString(env, encoding.scalability_mode),
       encoding.ssrc ? NativeToJavaLong(env, *encoding.ssrc) : nullptr,
       encoding.adaptive_ptime);
 }
@@ -116,6 +117,11 @@ RtpEncodingParameters JavaToNativeRtpEncodingParameters(
       Java_Encoding_getScaleResolutionDownBy(jni, j_encoding_parameters);
   encoding.scale_resolution_down_by =
       JavaToNativeOptionalDouble(jni, j_scale_resolution_down_by);
+  ScopedJavaLocalRef<jstring> j_scalability_mode =
+      Java_Encoding_getScalabilityMode(jni, j_encoding_parameters);
+  if (!IsNull(jni, j_scalability_mode)) {
+    encoding.scalability_mode = JavaToNativeString(jni,j_scalability_mode);
+  }
   encoding.adaptive_ptime =
       Java_Encoding_getAdaptivePTime(jni, j_encoding_parameters);
   ScopedJavaLocalRef<jobject> j_ssrc =
diff --git a/sdk/android/src/jni/video_codec_info.cc b/sdk/android/src/jni/video_codec_info.cc
@@ -13,24 +13,46 @@
 #include "sdk/android/generated_video_jni/VideoCodecInfo_jni.h"
 #include "sdk/android/native_api/jni/java_types.h"
 #include "sdk/android/src/jni/jni_helpers.h"
+#include "api/video_codecs/scalability_mode.h"
+#include "modules/video_coding/svc/scalability_mode_util.h"
 
 namespace webrtc {
 namespace jni {
 
 SdpVideoFormat VideoCodecInfoToSdpVideoFormat(JNIEnv* jni,
                                               const JavaRef<jobject>& j_info) {
+  std::vector<std::string> params =
+      JavaToStdVectorStrings(jni, Java_VideoCodecInfo_getScalabilityModes(jni, j_info));
+  absl::InlinedVector<ScalabilityMode, kScalabilityModeCount>
+    scalability_modes;
+  for (auto mode : params) {
+    auto scalability_mode = ScalabilityModeFromString(mode);
+    if (scalability_mode != absl::nullopt) {
+      scalability_modes.push_back(*scalability_mode);
+    }
+  }
   return SdpVideoFormat(
       JavaToNativeString(jni, Java_VideoCodecInfo_getName(jni, j_info)),
-      JavaToNativeStringMap(jni, Java_VideoCodecInfo_getParams(jni, j_info)));
+      JavaToNativeStringMap(jni, Java_VideoCodecInfo_getParams(jni, j_info)),
+      scalability_modes);
 }
 
 ScopedJavaLocalRef<jobject> SdpVideoFormatToVideoCodecInfo(
     JNIEnv* jni,
     const SdpVideoFormat& format) {
   ScopedJavaLocalRef<jobject> j_params =
       NativeToJavaStringMap(jni, format.parameters);
+  webrtc::ScopedJavaLocalRef<jobject> j_scalability_modes;
+  if (!format.scalability_modes.empty()) {
+    JavaListBuilder builder(jni);
+    for (auto mode : format.scalability_modes) {
+      std::string scalability_mode(ScalabilityModeToString(mode));
+      builder.add(NativeToJavaString(jni, scalability_mode));
+    }
+    j_scalability_modes = builder.java_list();
+  }
   return Java_VideoCodecInfo_Constructor(
-      jni, NativeToJavaString(jni, format.name), j_params);
+      jni, NativeToJavaString(jni, format.name), j_params, j_scalability_modes);
 }
 
 }  // namespace jni
diff --git a/sdk/android/src/jni/vp9_codec.cc b/sdk/android/src/jni/vp9_codec.cc
@@ -10,11 +10,18 @@
 
 #include <jni.h>
 
+#include "absl/container/inlined_vector.h"
+#include "api/video_codecs/sdp_video_format.h"
 #include "modules/video_coding/codecs/vp9/include/vp9.h"
+#include "modules/video_coding/svc/create_scalability_structure.h"
+
 #include "sdk/android/generated_libvpx_vp9_jni/LibvpxVp9Decoder_jni.h"
 #include "sdk/android/generated_libvpx_vp9_jni/LibvpxVp9Encoder_jni.h"
 #include "sdk/android/src/jni/jni_helpers.h"
 
+#include<vector>
+#include<string>
+
 namespace webrtc {
 namespace jni {
 
@@ -34,5 +41,14 @@ static jboolean JNI_LibvpxVp9Decoder_IsSupported(JNIEnv* jni) {
   return !SupportedVP9Codecs().empty();
 }
 
+static  webrtc::ScopedJavaLocalRef<jobject> JNI_LibvpxVp9Encoder_GetSupportedScalabilityModes(JNIEnv* jni) {
+  std::vector<std::string> modes;
+   for (const auto scalability_mode : webrtc::kAllScalabilityModes) {
+      if (webrtc::ScalabilityStructureConfig(scalability_mode).has_value()) {
+       modes.push_back(std::string(webrtc::ScalabilityModeToString(scalability_mode)));
+      }
+    }
+  return NativeToJavaStringArray(jni, modes);
+}
 }  // namespace jni
 }  // namespace webrtc
diff --git a/sdk/objc/api/peerconnection/RTCVideoCodecInfo+Private.mm b/sdk/objc/api/peerconnection/RTCVideoCodecInfo+Private.mm
diff --git a/sdk/objc/api/video_codec/RTCVideoEncoderAV1.h b/sdk/objc/api/video_codec/RTCVideoEncoderAV1.h
diff --git a/sdk/objc/api/video_codec/RTCVideoEncoderAV1.mm b/sdk/objc/api/video_codec/RTCVideoEncoderAV1.mm
diff --git a/sdk/objc/api/video_codec/RTCVideoEncoderVP9.h b/sdk/objc/api/video_codec/RTCVideoEncoderVP9.h
diff --git a/sdk/objc/api/video_codec/RTCVideoEncoderVP9.mm b/sdk/objc/api/video_codec/RTCVideoEncoderVP9.mm
diff --git a/sdk/objc/base/RTCVideoCodecInfo.h b/sdk/objc/base/RTCVideoCodecInfo.h
diff --git a/sdk/objc/base/RTCVideoCodecInfo.m b/sdk/objc/base/RTCVideoCodecInfo.m
diff --git a/sdk/objc/components/video_codec/RTCDefaultVideoEncoderFactory.m b/sdk/objc/components/video_codec/RTCDefaultVideoEncoderFactory.m

Original file line number	Diff line number	Diff line change
`@@ -143,11 +143,11 @@ public VideoCodecInfo[] getSupportedCodecs() {`
`143`	`143`	`// supported by the decoder.`
`144`	`144`	`if (type == VideoCodecMimeType.H264 && isH264HighProfileSupported(codec)) {`
`145`	`145`	`supportedCodecInfos.add(new VideoCodecInfo(`
`146`		`- name, MediaCodecUtils.getCodecProperties(type, /* highProfile= */ true)));`
	`146`	`+ name, MediaCodecUtils.getCodecProperties(type, /* highProfile= */ true), new ArrayList<>()));`
`147`	`147`	`}`
`148`	`148`
`149`	`149`	`supportedCodecInfos.add(new VideoCodecInfo(`
`150`		`- name, MediaCodecUtils.getCodecProperties(type, /* highProfile= */ false)));`
	`150`	`+ name, MediaCodecUtils.getCodecProperties(type, /* highProfile= */ false), new ArrayList<>()));`
`151`	`151`	`}`
`152`	`152`	`}`
`153`	`153`
Original file line number	Diff line number	Diff line change
`@@ -48,7 +48,7 @@ public AndroidVideoDecoderInstrumentationTest(String codecName, boolean useEglCo`
`48`	`48`	`if (codecName.equals("H264")) {`
`49`	`49`	`this.codecType = H264Utils.DEFAULT_H264_BASELINE_PROFILE_CODEC;`
`50`	`50`	`} else {`
`51`		`- this.codecType = new VideoCodecInfo(codecName, new HashMap<>());`
	`51`	`+ this.codecType = new VideoCodecInfo(codecName, new HashMap<>(), new ArrayList<>());`
`52`	`52`	`}`
`53`	`53`	`this.useEglContext = useEglContext;`
`54`	`54`	`}`