feat: support alpha buffer

Author: LichKing-2234

.eslintrc

@@ -90,6 +90,7 @@
     "WebGLTexture": false,
     "WebGLBuffer": false,
     "WebGLProgram": false,
+    "WebGLUniformLocation": false,
     "HTMLCanvasElement": false,
     "ResizeObserver": false,
     "name": false,

example/src/renderer/examples/advanced/VirtualBackground/VirtualBackground.tsx

@@ -79,6 +79,10 @@ export default class VirtualBackground
       true
     );
 
+    this.engine.setParameters(
+      JSON.stringify({ 'rtc.video.send_alpha_data': true })
+    );
+
     // Need to enable video on this case
     // If you only call `enableAudio`, only relay the audio stream to the target channel
     this.engine.enableVideo();

package.json

@@ -136,7 +136,7 @@
     "yuv-canvas": "1.2.6"
   },
   "agora_electron": {
-    "iris_sdk_win": "https://download.agora.io/sdk/release/iris_4.2.6-build.3_DCG_Windows_Video_20231116_0243.zip",
-    "iris_sdk_mac": "https://download.agora.io/sdk/release/iris_4.2.6-build.3_DCG_Mac_Video_20231116_0243.zip"
+    "iris_sdk_win": "https://download.agora.io/sdk/release/iris_4.2.6.1-dev.2_DCG_Windows_Video_20231204_0548.zip",
+    "iris_sdk_mac": "https://download.agora.io/sdk/release/iris_4.2.6.1-dev.2_DCG_Mac_Video_20231204_0548.zip"
   }
 }

patches/yuv-canvas+1.2.6.patch

@@ -0,0 +1,62 @@
+diff --git a/node_modules/yuv-canvas/src/SoftwareFrameSink.js b/node_modules/yuv-canvas/src/SoftwareFrameSink.js
+index 9627638..cff283c
+--- a/node_modules/yuv-canvas/src/SoftwareFrameSink.js
++++ b/node_modules/yuv-canvas/src/SoftwareFrameSink.js
+@@ -41,11 +41,11 @@ CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ 			imageData = ctx.createImageData(width, height);
+ 
+ 			// Prefill the alpha to opaque
+-			var data = imageData.data,
+-				pixelCount = width * height * 4;
+-			for (var i = 0; i < pixelCount; i += 4) {
+-				data[i + 3] = 255;
+-			}
++			// var data = imageData.data,
++			// 	pixelCount = width * height * 4;
++			// for (var i = 0; i < pixelCount; i += 4) {
++			// 	data[i + 3] = 255;
++			// }
+ 		}
+ 
+ 		function initResampleCanvas(cropWidth, cropHeight) {
+diff --git a/node_modules/yuv-canvas/src/YCbCr.js b/node_modules/yuv-canvas/src/YCbCr.js
+index 1e15c37..e51abe1
+--- a/node_modules/yuv-canvas/src/YCbCr.js
++++ b/node_modules/yuv-canvas/src/YCbCr.js
+@@ -78,24 +78,28 @@ CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ 					output[outPtr0    ] = (multY + multCrR) >> 8;
+ 					output[outPtr0 + 1] = (multY - multCbCrG) >> 8;
+ 					output[outPtr0 + 2] = (multY + multCbB) >> 8;
++					output[outPtr0 + 3] = buffer.a ? buffer.a[outPtr0 / 4] : 255;
+ 					outPtr0 += 4;
+ 
+ 					multY = 298 * bytesY[Y0Ptr++] | 0;
+ 					output[outPtr0    ] = (multY + multCrR) >> 8;
+ 					output[outPtr0 + 1] = (multY - multCbCrG) >> 8;
+ 					output[outPtr0 + 2] = (multY + multCbB) >> 8;
++					output[outPtr0 + 3] = buffer.a ? buffer.a[outPtr0 / 4] : 255;
+ 					outPtr0 += 4;
+ 
+ 					multY = 298 * bytesY[Y1Ptr++] | 0;
+ 					output[outPtr1    ] = (multY + multCrR) >> 8;
+ 					output[outPtr1 + 1] = (multY - multCbCrG) >> 8;
+ 					output[outPtr1 + 2] = (multY + multCbB) >> 8;
++					output[outPtr1 + 3] = buffer.a ? buffer.a[outPtr1 / 4] : 255;
+ 					outPtr1 += 4;
+ 
+ 					multY = 298 * bytesY[Y1Ptr++] | 0;
+ 					output[outPtr1    ] = (multY + multCrR) >> 8;
+ 					output[outPtr1 + 1] = (multY - multCbCrG) >> 8;
+ 					output[outPtr1 + 2] = (multY + multCbB) >> 8;
++					output[outPtr1 + 3] = buffer.a ? buffer.a[outPtr1 / 4] : 255;
+ 					outPtr1 += 4;
+ 				}
+ 				outPtr0 += outStride;
+@@ -127,6 +131,7 @@ CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ 					output[outPtr    ] = (multY + multCrR) >> 8;
+ 					output[outPtr + 1] = (multY - multCbCrG) >> 8;
+ 					output[outPtr + 2] = (multY + multCbB) >> 8;
++					output[outPtr + 3] = buffer.a ? buffer.a[outPtr / 4] : 255;
+ 					outPtr += 4;
+ 				}
+ 			}

source_code/agora_node_ext/agora_electron_bridge.cpp

@@ -387,17 +387,20 @@ napi_value AgoraElectronBridge::GetVideoFrame(napi_env env,
 
   napi_value obj1 = args[1];
   napi_value y_buffer_obj;
-  void *y_buffer;
-  size_t y_length;
+  void *y_buffer = nullptr;
+  size_t y_length = 0;
   napi_value u_buffer_obj;
-  void *u_buffer;
-  size_t u_length;
+  void *u_buffer = nullptr;
+  size_t u_length = 0;
   napi_value v_buffer_obj;
-  void *v_buffer;
-  size_t v_length;
+  void *v_buffer = nullptr;
+  size_t v_length = 0;
   int width;
   int height;
   int yStride;
+  napi_value alpha_buffer_obj;
+  void *alpha_buffer = nullptr;
+  size_t alpha_length = 0;
 
   napi_obj_get_property(env, obj1, "yBuffer", y_buffer_obj);
   napi_get_buffer_info(env, y_buffer_obj, &y_buffer, &y_length);
@@ -412,6 +415,14 @@ napi_value AgoraElectronBridge::GetVideoFrame(napi_env env,
   napi_obj_get_property(env, obj1, "height", height);
   napi_obj_get_property(env, obj1, "yStride", yStride);
 
+  napi_obj_get_property(env, obj1, "alphaBuffer", alpha_buffer_obj);
+  napi_value undefinedValue;
+  napi_get_undefined(env, &undefinedValue);
+  bool isUndefined;
+  if (napi_strict_equals(env, alpha_buffer_obj, undefinedValue, &isUndefined) == napi_ok && !isUndefined) {
+    napi_get_buffer_info(env, alpha_buffer_obj, &alpha_buffer, &alpha_length);
+  }
+
   IrisCVideoFrame videoFrame;
   videoFrame.yBuffer = (uint8_t *) y_buffer;
   videoFrame.uBuffer = (uint8_t *) u_buffer;
@@ -421,7 +432,8 @@ napi_value AgoraElectronBridge::GetVideoFrame(napi_env env,
   videoFrame.yStride = yStride;
   videoFrame.metadata_buffer = nullptr;
   videoFrame.metadata_size = 0;
-  videoFrame.alphaBuffer = nullptr;
+  videoFrame.alphaBuffer = (uint8_t *) alpha_buffer;
+  videoFrame.hasAlphaBuffer = false;
 
   bool isNewFrame = false;
   napi_value retObj;
@@ -451,6 +463,7 @@ napi_value AgoraElectronBridge::GetVideoFrame(napi_env env,
   napi_obj_set_property(env, obj1, "avsync_type", videoFrame.avsync_type);
   napi_obj_set_property(env, obj1, "metadata_size", videoFrame.metadata_size);
   // napi_obj_set_property(env, obj1, "textureId", videoFrame.textureId);
+  napi_obj_set_property(env, obj1, "hasAlphaBuffer", videoFrame.hasAlphaBuffer);
 
   return retObj;
 }

ts/Renderer/RendererCache.ts

@@ -108,6 +108,16 @@ export class RendererCache {
         return;
     }
 
+    if ((this.videoFrame as any).hasAlphaBuffer) {
+      if (this.videoFrame.alphaBuffer === undefined) {
+        this.videoFrame.alphaBuffer = Buffer.alloc(
+          this.videoFrame.yStride! * this.videoFrame.height!
+        );
+      }
+    } else {
+      this.videoFrame.alphaBuffer = undefined;
+    }
+
     if (isNewFrame) {
       this.renderers.forEach((renderer) => {
         renderer.drawFrame(this.videoFrame);