From f02048d006ecdc85d185a984f70076b6f90efdac Mon Sep 17 00:00:00 2001 From: misos1 <30872003+misos1@users.noreply.github.com> Date: Sat, 29 Jul 2023 18:20:14 +0200 Subject: [PATCH 01/14] [camera_avfoundation] ignore audio samples until first video sample arrives --- packages/camera/camera_avfoundation/ios/Classes/FLTCam.m | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/camera/camera_avfoundation/ios/Classes/FLTCam.m b/packages/camera/camera_avfoundation/ios/Classes/FLTCam.m index 31bffc91794..a72c65e1d50 100644 --- a/packages/camera/camera_avfoundation/ios/Classes/FLTCam.m +++ b/packages/camera/camera_avfoundation/ios/Classes/FLTCam.m @@ -500,6 +500,10 @@ - (void)captureOutput:(AVCaptureOutput *)output return; } + if (_videoWriter.status != AVAssetWriterStatusWriting && output != _captureVideoOutput) { + return; + } + CFRetain(sampleBuffer); CMTime currentSampleTime = CMSampleBufferGetPresentationTimeStamp(sampleBuffer); From a3191147f432ee995f24d82f3b934e1c96ae2ef6 Mon Sep 17 00:00:00 2001 From: misos1 <30872003+misos1@users.noreply.github.com> Date: Sat, 29 Jul 2023 18:28:17 +0200 Subject: [PATCH 02/14] [camera_avfoundation] update version --- packages/camera/camera_avfoundation/pubspec.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/camera/camera_avfoundation/pubspec.yaml b/packages/camera/camera_avfoundation/pubspec.yaml index f6137d59f68..19cb3d126ca 100644 --- a/packages/camera/camera_avfoundation/pubspec.yaml +++ b/packages/camera/camera_avfoundation/pubspec.yaml @@ -2,7 +2,7 @@ name: camera_avfoundation description: iOS implementation of the camera plugin. repository: https://github.com/flutter/packages/tree/main/packages/camera/camera_avfoundation issue_tracker: https://github.com/flutter/flutter/issues?q=is%3Aissue+is%3Aopen+label%3A%22p%3A+camera%22 -version: 0.9.13+2 +version: 0.9.13+3 environment: sdk: ">=2.18.0 <4.0.0" From d8be08ce8b9afcaae487b257059fc2ce89b9938e Mon Sep 17 00:00:00 2001 From: misos1 <30872003+misos1@users.noreply.github.com> Date: Sat, 29 Jul 2023 18:30:09 +0200 Subject: [PATCH 03/14] [camera_avfoundation] update changelog --- packages/camera/camera_avfoundation/CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/camera/camera_avfoundation/CHANGELOG.md b/packages/camera/camera_avfoundation/CHANGELOG.md index 2133c7935c0..b62d125f5e2 100644 --- a/packages/camera/camera_avfoundation/CHANGELOG.md +++ b/packages/camera/camera_avfoundation/CHANGELOG.md @@ -1,5 +1,6 @@ -## NEXT +## 0.9.13+3 +* Ignore audio samples until the first video sample arrives. * Fixes unawaited_futures violations. ## 0.9.13+2 From e73255b308be6bf21c114ccf94e708b2c41abc8d Mon Sep 17 00:00:00 2001 From: misos1 <30872003+misos1@users.noreply.github.com> Date: Sun, 30 Jul 2023 18:51:52 +0200 Subject: [PATCH 04/14] [camera_avfoundation] add comment --- packages/camera/camera_avfoundation/ios/Classes/FLTCam.m | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/camera/camera_avfoundation/ios/Classes/FLTCam.m b/packages/camera/camera_avfoundation/ios/Classes/FLTCam.m index a72c65e1d50..8b067b1f76d 100644 --- a/packages/camera/camera_avfoundation/ios/Classes/FLTCam.m +++ b/packages/camera/camera_avfoundation/ios/Classes/FLTCam.m @@ -500,6 +500,7 @@ - (void)captureOutput:(AVCaptureOutput *)output return; } + // ignore audio samples until the first video sample arrives to avoid black frames at video start if (_videoWriter.status != AVAssetWriterStatusWriting && output != _captureVideoOutput) { return; } From 6abcebb0f89ede70328637dce0603b33d2deafcb Mon Sep 17 00:00:00 2001 From: misos1 <30872003+misos1@users.noreply.github.com> Date: Sun, 30 Jul 2023 19:17:31 +0200 Subject: [PATCH 05/14] [camera_avfoundation] update comment --- packages/camera/camera_avfoundation/ios/Classes/FLTCam.m | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/camera/camera_avfoundation/ios/Classes/FLTCam.m b/packages/camera/camera_avfoundation/ios/Classes/FLTCam.m index 8b067b1f76d..9fd90a43acf 100644 --- a/packages/camera/camera_avfoundation/ios/Classes/FLTCam.m +++ b/packages/camera/camera_avfoundation/ios/Classes/FLTCam.m @@ -500,7 +500,7 @@ - (void)captureOutput:(AVCaptureOutput *)output return; } - // ignore audio samples until the first video sample arrives to avoid black frames at video start + // ignore audio samples until the first video sample arrives to avoid black frames if (_videoWriter.status != AVAssetWriterStatusWriting && output != _captureVideoOutput) { return; } From 7163cac95db6169cfe0dbd528695114cd5a1a23c Mon Sep 17 00:00:00 2001 From: misos1 <30872003+misos1@users.noreply.github.com> Date: Thu, 3 Aug 2023 19:25:23 +0200 Subject: [PATCH 06/14] [camera_avfoundation] add test --- .../example/ios/RunnerTests/CameraTestUtils.h | 4 + .../example/ios/RunnerTests/CameraTestUtils.m | 19 +++++ .../ios/RunnerTests/FLTCamSampleBufferTests.m | 83 ++++++++++++++++--- 3 files changed, 95 insertions(+), 11 deletions(-) diff --git a/packages/camera/camera_avfoundation/example/ios/RunnerTests/CameraTestUtils.h b/packages/camera/camera_avfoundation/example/ios/RunnerTests/CameraTestUtils.h index f2d46114a0c..0c7e62f9fbb 100644 --- a/packages/camera/camera_avfoundation/example/ios/RunnerTests/CameraTestUtils.h +++ b/packages/camera/camera_avfoundation/example/ios/RunnerTests/CameraTestUtils.h @@ -15,4 +15,8 @@ extern FLTCam *FLTCreateCamWithCaptureSessionQueue(dispatch_queue_t captureSessi /// @return a test sample buffer. extern CMSampleBufferRef FLTCreateTestSampleBuffer(void); +/// Creates a test audio sample buffer. +/// @return a test audio sample buffer. +extern CMSampleBufferRef FLTCreateTestAudioSampleBuffer(void); + NS_ASSUME_NONNULL_END diff --git a/packages/camera/camera_avfoundation/example/ios/RunnerTests/CameraTestUtils.m b/packages/camera/camera_avfoundation/example/ios/RunnerTests/CameraTestUtils.m index b42aa34e2a1..3d9833a106b 100644 --- a/packages/camera/camera_avfoundation/example/ios/RunnerTests/CameraTestUtils.m +++ b/packages/camera/camera_avfoundation/example/ios/RunnerTests/CameraTestUtils.m @@ -47,3 +47,22 @@ CMSampleBufferRef FLTCreateTestSampleBuffer(void) { CFRelease(formatDescription); return sampleBuffer; } + +CMSampleBufferRef FLTCreateTestAudioSampleBuffer(void) { + CMBlockBufferRef blockBuffer; + CMBlockBufferCreateWithMemoryBlock(kCFAllocatorDefault, NULL, 100, kCFAllocatorDefault, NULL, + 0, 100, kCMBlockBufferAssureMemoryNowFlag, &blockBuffer); + + CMFormatDescriptionRef formatDescription; + AudioStreamBasicDescription basicDescription = {44100, kAudioFormatLinearPCM, 0, 1, 1, 1, 1, 8}; + CMAudioFormatDescriptionCreate(kCFAllocatorDefault, &basicDescription, 0, + NULL, 0, NULL, NULL, &formatDescription); + + CMSampleBufferRef sampleBuffer; + CMAudioSampleBufferCreateReadyWithPacketDescriptions(kCFAllocatorDefault, blockBuffer, formatDescription, + 1, kCMTimeZero, NULL, &sampleBuffer); + + CFRelease(blockBuffer); + CFRelease(formatDescription); + return sampleBuffer; +} diff --git a/packages/camera/camera_avfoundation/example/ios/RunnerTests/FLTCamSampleBufferTests.m b/packages/camera/camera_avfoundation/example/ios/RunnerTests/FLTCamSampleBufferTests.m index 94426ab3aeb..bd4a4c37701 100644 --- a/packages/camera/camera_avfoundation/example/ios/RunnerTests/FLTCamSampleBufferTests.m +++ b/packages/camera/camera_avfoundation/example/ios/RunnerTests/FLTCamSampleBufferTests.m @@ -11,31 +11,92 @@ /// Includes test cases related to sample buffer handling for FLTCam class. @interface FLTCamSampleBufferTests : XCTestCase - +@property(readonly, nonatomic) dispatch_queue_t captureSessionQueue; +@property(readonly, nonatomic) FLTCam *camera; +@property(readonly, nonatomic) CMSampleBufferRef sampleBuffer; @end @implementation FLTCamSampleBufferTests +- (void)setUp { + _captureSessionQueue = dispatch_queue_create("testing", NULL); + _camera = FLTCreateCamWithCaptureSessionQueue(_captureSessionQueue); + _sampleBuffer = FLTCreateTestSampleBuffer(); +} + +- (void)tearDown { + CFRelease(_sampleBuffer); +} + - (void)testSampleBufferCallbackQueueMustBeCaptureSessionQueue { - dispatch_queue_t captureSessionQueue = dispatch_queue_create("testing", NULL); - FLTCam *cam = FLTCreateCamWithCaptureSessionQueue(captureSessionQueue); - XCTAssertEqual(captureSessionQueue, cam.captureVideoOutput.sampleBufferCallbackQueue, + XCTAssertEqual(_captureSessionQueue, _camera.captureVideoOutput.sampleBufferCallbackQueue, @"Sample buffer callback queue must be the capture session queue."); } - (void)testCopyPixelBuffer { - FLTCam *cam = FLTCreateCamWithCaptureSessionQueue(dispatch_queue_create("test", NULL)); - CMSampleBufferRef capturedSampleBuffer = FLTCreateTestSampleBuffer(); - CVPixelBufferRef capturedPixelBuffer = CMSampleBufferGetImageBuffer(capturedSampleBuffer); + CVPixelBufferRef capturedPixelBuffer = CMSampleBufferGetImageBuffer(_sampleBuffer); // Mimic sample buffer callback when captured a new video sample - [cam captureOutput:cam.captureVideoOutput - didOutputSampleBuffer:capturedSampleBuffer + [_camera captureOutput:_camera.captureVideoOutput + didOutputSampleBuffer:_sampleBuffer fromConnection:OCMClassMock([AVCaptureConnection class])]; - CVPixelBufferRef deliveriedPixelBuffer = [cam copyPixelBuffer]; + CVPixelBufferRef deliveriedPixelBuffer = [_camera copyPixelBuffer]; XCTAssertEqual(deliveriedPixelBuffer, capturedPixelBuffer, @"FLTCam must deliver the latest captured pixel buffer to copyPixelBuffer API."); - CFRelease(capturedSampleBuffer); CFRelease(deliveriedPixelBuffer); } +- (void)testFirstAppendedSampleShouldBeVideo { + id connectionMock = OCMClassMock([AVCaptureConnection class]); + + id writerMock = OCMClassMock([AVAssetWriter class]); + OCMStub([writerMock alloc]).andReturn(writerMock); + OCMStub([writerMock initWithURL:OCMOCK_ANY fileType:OCMOCK_ANY error:[OCMArg setTo:nil]]) + .andReturn(writerMock); + __block AVAssetWriterStatus status = AVAssetWriterStatusUnknown; + OCMStub([writerMock startWriting]).andDo(^(NSInvocation *invocation) { + status = AVAssetWriterStatusWriting; + }); + OCMStub([writerMock status]).andDo(^(NSInvocation *invocation) { + [invocation setReturnValue:&status]; + }); + + __block NSString *writtenSamples = @""; + + id videoMock = OCMClassMock([AVAssetWriterInputPixelBufferAdaptor class]); + OCMStub([videoMock + assetWriterInputPixelBufferAdaptorWithAssetWriterInput:OCMOCK_ANY + sourcePixelBufferAttributes:OCMOCK_ANY]).andReturn(videoMock); + OCMStub([videoMock appendPixelBuffer:[OCMArg anyPointer] withPresentationTime:kCMTimeZero]) + .andDo(^(NSInvocation *invocation) { + writtenSamples = [writtenSamples stringByAppendingString:@"v"]; + }); + + id audioMock = OCMClassMock([AVAssetWriterInput class]); + OCMStub([audioMock + assetWriterInputWithMediaType:[OCMArg isEqual:AVMediaTypeAudio] + outputSettings:OCMOCK_ANY]).andReturn(audioMock); + OCMStub([audioMock isReadyForMoreMediaData]).andReturn(YES); + OCMStub([audioMock appendSampleBuffer:[OCMArg anyPointer]]).andDo(^(NSInvocation *invocation) { + writtenSamples = [writtenSamples stringByAppendingString:@"a"]; + }); + + FLTThreadSafeFlutterResult *result = [[FLTThreadSafeFlutterResult alloc] initWithResult:^(id result) {}]; + [_camera startVideoRecordingWithResult:result]; + + char *samples = "aaavava"; + CMSampleBufferRef audioSampleBuffer = FLTCreateTestAudioSampleBuffer(); + for (int i = 0; i < strlen(samples); i++) { + if(samples[i] == 'v') { + [_camera captureOutput:_camera.captureVideoOutput didOutputSampleBuffer:_sampleBuffer + fromConnection:connectionMock]; + } else { + [_camera captureOutput:nil didOutputSampleBuffer:audioSampleBuffer + fromConnection:connectionMock]; + } + } + CFRelease(audioSampleBuffer); + + XCTAssertEqualObjects(writtenSamples, @"vava", @"First appended sample must be video."); +} + @end From 6ef858cb55757d4b4bca9f5c08c0e4d5546260c1 Mon Sep 17 00:00:00 2001 From: misos1 <30872003+misos1@users.noreply.github.com> Date: Thu, 3 Aug 2023 19:50:12 +0200 Subject: [PATCH 07/14] [camera_avfoundation] update whitespaces in test --- .../example/ios/RunnerTests/CameraTestUtils.m | 8 ++-- .../ios/RunnerTests/FLTCamSampleBufferTests.m | 37 +++++++++++-------- 2 files changed, 25 insertions(+), 20 deletions(-) diff --git a/packages/camera/camera_avfoundation/example/ios/RunnerTests/CameraTestUtils.m b/packages/camera/camera_avfoundation/example/ios/RunnerTests/CameraTestUtils.m index 3d9833a106b..e9d4a3656da 100644 --- a/packages/camera/camera_avfoundation/example/ios/RunnerTests/CameraTestUtils.m +++ b/packages/camera/camera_avfoundation/example/ios/RunnerTests/CameraTestUtils.m @@ -50,13 +50,13 @@ CMSampleBufferRef FLTCreateTestSampleBuffer(void) { CMSampleBufferRef FLTCreateTestAudioSampleBuffer(void) { CMBlockBufferRef blockBuffer; - CMBlockBufferCreateWithMemoryBlock(kCFAllocatorDefault, NULL, 100, kCFAllocatorDefault, NULL, - 0, 100, kCMBlockBufferAssureMemoryNowFlag, &blockBuffer); + CMBlockBufferCreateWithMemoryBlock(kCFAllocatorDefault, NULL, 100, kCFAllocatorDefault, NULL, 0, + 100, kCMBlockBufferAssureMemoryNowFlag, &blockBuffer); CMFormatDescriptionRef formatDescription; AudioStreamBasicDescription basicDescription = {44100, kAudioFormatLinearPCM, 0, 1, 1, 1, 1, 8}; - CMAudioFormatDescriptionCreate(kCFAllocatorDefault, &basicDescription, 0, - NULL, 0, NULL, NULL, &formatDescription); + CMAudioFormatDescriptionCreate( + kCFAllocatorDefault, &basicDescription, 0, NULL, 0, NULL, NULL, &formatDescription); CMSampleBufferRef sampleBuffer; CMAudioSampleBufferCreateReadyWithPacketDescriptions(kCFAllocatorDefault, blockBuffer, formatDescription, diff --git a/packages/camera/camera_avfoundation/example/ios/RunnerTests/FLTCamSampleBufferTests.m b/packages/camera/camera_avfoundation/example/ios/RunnerTests/FLTCamSampleBufferTests.m index bd4a4c37701..9f9049fea7e 100644 --- a/packages/camera/camera_avfoundation/example/ios/RunnerTests/FLTCamSampleBufferTests.m +++ b/packages/camera/camera_avfoundation/example/ios/RunnerTests/FLTCamSampleBufferTests.m @@ -51,7 +51,7 @@ - (void)testFirstAppendedSampleShouldBeVideo { id writerMock = OCMClassMock([AVAssetWriter class]); OCMStub([writerMock alloc]).andReturn(writerMock); OCMStub([writerMock initWithURL:OCMOCK_ANY fileType:OCMOCK_ANY error:[OCMArg setTo:nil]]) - .andReturn(writerMock); + .andReturn(writerMock); __block AVAssetWriterStatus status = AVAssetWriterStatusUnknown; OCMStub([writerMock startWriting]).andDo(^(NSInvocation *invocation) { status = AVAssetWriterStatusWriting; @@ -63,35 +63,40 @@ - (void)testFirstAppendedSampleShouldBeVideo { __block NSString *writtenSamples = @""; id videoMock = OCMClassMock([AVAssetWriterInputPixelBufferAdaptor class]); - OCMStub([videoMock - assetWriterInputPixelBufferAdaptorWithAssetWriterInput:OCMOCK_ANY - sourcePixelBufferAttributes:OCMOCK_ANY]).andReturn(videoMock); + OCMStub([videoMock assetWriterInputPixelBufferAdaptorWithAssetWriterInput:OCMOCK_ANY + sourcePixelBufferAttributes:OCMOCK_ANY]) + .andReturn(videoMock); OCMStub([videoMock appendPixelBuffer:[OCMArg anyPointer] withPresentationTime:kCMTimeZero]) - .andDo(^(NSInvocation *invocation) { - writtenSamples = [writtenSamples stringByAppendingString:@"v"]; - }); + .andDo(^(NSInvocation *invocation) { + writtenSamples = [writtenSamples stringByAppendingString:@"v"]; + }); id audioMock = OCMClassMock([AVAssetWriterInput class]); - OCMStub([audioMock - assetWriterInputWithMediaType:[OCMArg isEqual:AVMediaTypeAudio] - outputSettings:OCMOCK_ANY]).andReturn(audioMock); + OCMStub([audioMock assetWriterInputWithMediaType:[OCMArg isEqual:AVMediaTypeAudio] + outputSettings:OCMOCK_ANY]) + .andReturn(audioMock); OCMStub([audioMock isReadyForMoreMediaData]).andReturn(YES); OCMStub([audioMock appendSampleBuffer:[OCMArg anyPointer]]).andDo(^(NSInvocation *invocation) { writtenSamples = [writtenSamples stringByAppendingString:@"a"]; }); - FLTThreadSafeFlutterResult *result = [[FLTThreadSafeFlutterResult alloc] initWithResult:^(id result) {}]; + FLTThreadSafeFlutterResult *result = + [[FLTThreadSafeFlutterResult alloc] initWithResult:^(id result){ + }]; [_camera startVideoRecordingWithResult:result]; char *samples = "aaavava"; + CMSampleBufferRef audioSampleBuffer = FLTCreateTestAudioSampleBuffer(); for (int i = 0; i < strlen(samples); i++) { - if(samples[i] == 'v') { - [_camera captureOutput:_camera.captureVideoOutput didOutputSampleBuffer:_sampleBuffer - fromConnection:connectionMock]; + if (samples[i] == 'v') { + [_camera captureOutput:_camera.captureVideoOutput + didOutputSampleBuffer:_sampleBuffer + fromConnection:connectionMock]; } else { - [_camera captureOutput:nil didOutputSampleBuffer:audioSampleBuffer - fromConnection:connectionMock]; + [_camera captureOutput:nil + didOutputSampleBuffer:audioSampleBuffer + fromConnection:connectionMock]; } } CFRelease(audioSampleBuffer); From 94f332f9077135b8645a75088b7bbd20a9a22d33 Mon Sep 17 00:00:00 2001 From: misos1 <30872003+misos1@users.noreply.github.com> Date: Thu, 3 Aug 2023 20:00:17 +0200 Subject: [PATCH 08/14] [camera_avfoundation] update whitespaces in test --- .../example/ios/RunnerTests/CameraTestUtils.m | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/camera/camera_avfoundation/example/ios/RunnerTests/CameraTestUtils.m b/packages/camera/camera_avfoundation/example/ios/RunnerTests/CameraTestUtils.m index e9d4a3656da..bb98f7cf71e 100644 --- a/packages/camera/camera_avfoundation/example/ios/RunnerTests/CameraTestUtils.m +++ b/packages/camera/camera_avfoundation/example/ios/RunnerTests/CameraTestUtils.m @@ -55,12 +55,12 @@ CMSampleBufferRef FLTCreateTestAudioSampleBuffer(void) { CMFormatDescriptionRef formatDescription; AudioStreamBasicDescription basicDescription = {44100, kAudioFormatLinearPCM, 0, 1, 1, 1, 1, 8}; - CMAudioFormatDescriptionCreate( - kCFAllocatorDefault, &basicDescription, 0, NULL, 0, NULL, NULL, &formatDescription); + CMAudioFormatDescriptionCreate(kCFAllocatorDefault, &basicDescription, 0, NULL, 0, NULL, NULL, + &formatDescription); CMSampleBufferRef sampleBuffer; - CMAudioSampleBufferCreateReadyWithPacketDescriptions(kCFAllocatorDefault, blockBuffer, formatDescription, - 1, kCMTimeZero, NULL, &sampleBuffer); + CMAudioSampleBufferCreateReadyWithPacketDescriptions( + kCFAllocatorDefault, blockBuffer, formatDescription, 1, kCMTimeZero, NULL, &sampleBuffer); CFRelease(blockBuffer); CFRelease(formatDescription); From 4db9a592bddd2e76c45013a203ce91b22ce0767c Mon Sep 17 00:00:00 2001 From: misos1 <30872003+misos1@users.noreply.github.com> Date: Thu, 3 Aug 2023 20:53:18 +0200 Subject: [PATCH 09/14] [camera_avfoundation] ignore non object arguments in videoMock --- .../example/ios/RunnerTests/FLTCamSampleBufferTests.m | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/camera/camera_avfoundation/example/ios/RunnerTests/FLTCamSampleBufferTests.m b/packages/camera/camera_avfoundation/example/ios/RunnerTests/FLTCamSampleBufferTests.m index 9f9049fea7e..0f3a55b2de3 100644 --- a/packages/camera/camera_avfoundation/example/ios/RunnerTests/FLTCamSampleBufferTests.m +++ b/packages/camera/camera_avfoundation/example/ios/RunnerTests/FLTCamSampleBufferTests.m @@ -67,6 +67,7 @@ - (void)testFirstAppendedSampleShouldBeVideo { sourcePixelBufferAttributes:OCMOCK_ANY]) .andReturn(videoMock); OCMStub([videoMock appendPixelBuffer:[OCMArg anyPointer] withPresentationTime:kCMTimeZero]) + .ignoringNonObjectArgs() .andDo(^(NSInvocation *invocation) { writtenSamples = [writtenSamples stringByAppendingString:@"v"]; }); From dcdb882f1f6137d59f74e07e753641610965fc3d Mon Sep 17 00:00:00 2001 From: misos1 <30872003+misos1@users.noreply.github.com> Date: Wed, 16 Aug 2023 21:36:23 +0200 Subject: [PATCH 10/14] [camera_avfoundation] fix changelog entry --- packages/camera/camera_avfoundation/CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/camera/camera_avfoundation/CHANGELOG.md b/packages/camera/camera_avfoundation/CHANGELOG.md index b62d125f5e2..f1fbf8666c2 100644 --- a/packages/camera/camera_avfoundation/CHANGELOG.md +++ b/packages/camera/camera_avfoundation/CHANGELOG.md @@ -1,6 +1,6 @@ ## 0.9.13+3 -* Ignore audio samples until the first video sample arrives. +* Ignores audio samples until the first video sample arrives. * Fixes unawaited_futures violations. ## 0.9.13+2 From 28290098af06e68983e12eadb37e11a8fd66d598 Mon Sep 17 00:00:00 2001 From: misos1 <30872003+misos1@users.noreply.github.com> Date: Wed, 16 Aug 2023 22:08:05 +0200 Subject: [PATCH 11/14] [camera_avfoundation] simplify test --- .../ios/RunnerTests/FLTCamSampleBufferTests.m | 69 ++++++++----------- 1 file changed, 29 insertions(+), 40 deletions(-) diff --git a/packages/camera/camera_avfoundation/example/ios/RunnerTests/FLTCamSampleBufferTests.m b/packages/camera/camera_avfoundation/example/ios/RunnerTests/FLTCamSampleBufferTests.m index 0f3a55b2de3..4d6941b85ff 100644 --- a/packages/camera/camera_avfoundation/example/ios/RunnerTests/FLTCamSampleBufferTests.m +++ b/packages/camera/camera_avfoundation/example/ios/RunnerTests/FLTCamSampleBufferTests.m @@ -11,41 +11,36 @@ /// Includes test cases related to sample buffer handling for FLTCam class. @interface FLTCamSampleBufferTests : XCTestCase -@property(readonly, nonatomic) dispatch_queue_t captureSessionQueue; -@property(readonly, nonatomic) FLTCam *camera; -@property(readonly, nonatomic) CMSampleBufferRef sampleBuffer; + @end @implementation FLTCamSampleBufferTests -- (void)setUp { - _captureSessionQueue = dispatch_queue_create("testing", NULL); - _camera = FLTCreateCamWithCaptureSessionQueue(_captureSessionQueue); - _sampleBuffer = FLTCreateTestSampleBuffer(); -} - -- (void)tearDown { - CFRelease(_sampleBuffer); -} - - (void)testSampleBufferCallbackQueueMustBeCaptureSessionQueue { - XCTAssertEqual(_captureSessionQueue, _camera.captureVideoOutput.sampleBufferCallbackQueue, + dispatch_queue_t captureSessionQueue = dispatch_queue_create("testing", NULL); + FLTCam *cam = FLTCreateCamWithCaptureSessionQueue(captureSessionQueue); + XCTAssertEqual(captureSessionQueue, cam.captureVideoOutput.sampleBufferCallbackQueue, @"Sample buffer callback queue must be the capture session queue."); } - (void)testCopyPixelBuffer { - CVPixelBufferRef capturedPixelBuffer = CMSampleBufferGetImageBuffer(_sampleBuffer); + FLTCam *cam = FLTCreateCamWithCaptureSessionQueue(dispatch_queue_create("test", NULL)); + CMSampleBufferRef capturedSampleBuffer = FLTCreateTestSampleBuffer(); + CVPixelBufferRef capturedPixelBuffer = CMSampleBufferGetImageBuffer(capturedSampleBuffer); // Mimic sample buffer callback when captured a new video sample - [_camera captureOutput:_camera.captureVideoOutput - didOutputSampleBuffer:_sampleBuffer + [cam captureOutput:cam.captureVideoOutput + didOutputSampleBuffer:capturedSampleBuffer fromConnection:OCMClassMock([AVCaptureConnection class])]; - CVPixelBufferRef deliveriedPixelBuffer = [_camera copyPixelBuffer]; + CVPixelBufferRef deliveriedPixelBuffer = [cam copyPixelBuffer]; XCTAssertEqual(deliveriedPixelBuffer, capturedPixelBuffer, @"FLTCam must deliver the latest captured pixel buffer to copyPixelBuffer API."); + CFRelease(capturedSampleBuffer); CFRelease(deliveriedPixelBuffer); } - (void)testFirstAppendedSampleShouldBeVideo { + FLTCam *cam = FLTCreateCamWithCaptureSessionQueue(dispatch_queue_create("testing", NULL)); + id connectionMock = OCMClassMock([AVCaptureConnection class]); id writerMock = OCMClassMock([AVAssetWriter class]); @@ -60,7 +55,7 @@ - (void)testFirstAppendedSampleShouldBeVideo { [invocation setReturnValue:&status]; }); - __block NSString *writtenSamples = @""; + __block NSArray *writtenSamples = @[]; id videoMock = OCMClassMock([AVAssetWriterInputPixelBufferAdaptor class]); OCMStub([videoMock assetWriterInputPixelBufferAdaptorWithAssetWriterInput:OCMOCK_ANY @@ -69,7 +64,7 @@ - (void)testFirstAppendedSampleShouldBeVideo { OCMStub([videoMock appendPixelBuffer:[OCMArg anyPointer] withPresentationTime:kCMTimeZero]) .ignoringNonObjectArgs() .andDo(^(NSInvocation *invocation) { - writtenSamples = [writtenSamples stringByAppendingString:@"v"]; + writtenSamples = [writtenSamples arrayByAddingObject:@"video"]; }); id audioMock = OCMClassMock([AVAssetWriterInput class]); @@ -78,31 +73,25 @@ - (void)testFirstAppendedSampleShouldBeVideo { .andReturn(audioMock); OCMStub([audioMock isReadyForMoreMediaData]).andReturn(YES); OCMStub([audioMock appendSampleBuffer:[OCMArg anyPointer]]).andDo(^(NSInvocation *invocation) { - writtenSamples = [writtenSamples stringByAppendingString:@"a"]; + writtenSamples = [writtenSamples arrayByAddingObject:@"audio"]; }); FLTThreadSafeFlutterResult *result = [[FLTThreadSafeFlutterResult alloc] initWithResult:^(id result){ }]; - [_camera startVideoRecordingWithResult:result]; - - char *samples = "aaavava"; - - CMSampleBufferRef audioSampleBuffer = FLTCreateTestAudioSampleBuffer(); - for (int i = 0; i < strlen(samples); i++) { - if (samples[i] == 'v') { - [_camera captureOutput:_camera.captureVideoOutput - didOutputSampleBuffer:_sampleBuffer - fromConnection:connectionMock]; - } else { - [_camera captureOutput:nil - didOutputSampleBuffer:audioSampleBuffer - fromConnection:connectionMock]; - } - } - CFRelease(audioSampleBuffer); - - XCTAssertEqualObjects(writtenSamples, @"vava", @"First appended sample must be video."); + [cam startVideoRecordingWithResult:result]; + + CMSampleBufferRef videoSample = FLTCreateTestSampleBuffer(); + CMSampleBufferRef audioSample = FLTCreateTestAudioSampleBuffer(); + [cam captureOutput:nil didOutputSampleBuffer:audioSample fromConnection:connectionMock]; + [cam captureOutput:nil didOutputSampleBuffer:audioSample fromConnection:connectionMock]; + [cam captureOutput:cam.captureVideoOutput didOutputSampleBuffer:videoSample fromConnection:connectionMock]; + [cam captureOutput:nil didOutputSampleBuffer:audioSample fromConnection:connectionMock]; + CFRelease(videoSample); + CFRelease(audioSample); + + NSArray *expectedSamples = @[@"video", @"audio"]; + XCTAssertEqualObjects(writtenSamples, expectedSamples, @"First appended sample must be video."); } @end From 6f4e510ede54ad033441ab1ad5d3e1bdc1d398e0 Mon Sep 17 00:00:00 2001 From: misos1 <30872003+misos1@users.noreply.github.com> Date: Wed, 16 Aug 2023 22:25:07 +0200 Subject: [PATCH 12/14] [camera_avfoundation] update formating --- .../ios/RunnerTests/FLTCamSampleBufferTests.m | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/packages/camera/camera_avfoundation/example/ios/RunnerTests/FLTCamSampleBufferTests.m b/packages/camera/camera_avfoundation/example/ios/RunnerTests/FLTCamSampleBufferTests.m index 4d6941b85ff..2a02a9dc273 100644 --- a/packages/camera/camera_avfoundation/example/ios/RunnerTests/FLTCamSampleBufferTests.m +++ b/packages/camera/camera_avfoundation/example/ios/RunnerTests/FLTCamSampleBufferTests.m @@ -40,6 +40,8 @@ - (void)testCopyPixelBuffer { - (void)testFirstAppendedSampleShouldBeVideo { FLTCam *cam = FLTCreateCamWithCaptureSessionQueue(dispatch_queue_create("testing", NULL)); + CMSampleBufferRef videoSample = FLTCreateTestSampleBuffer(); + CMSampleBufferRef audioSample = FLTCreateTestAudioSampleBuffer(); id connectionMock = OCMClassMock([AVCaptureConnection class]); @@ -81,17 +83,18 @@ - (void)testFirstAppendedSampleShouldBeVideo { }]; [cam startVideoRecordingWithResult:result]; - CMSampleBufferRef videoSample = FLTCreateTestSampleBuffer(); - CMSampleBufferRef audioSample = FLTCreateTestAudioSampleBuffer(); [cam captureOutput:nil didOutputSampleBuffer:audioSample fromConnection:connectionMock]; [cam captureOutput:nil didOutputSampleBuffer:audioSample fromConnection:connectionMock]; - [cam captureOutput:cam.captureVideoOutput didOutputSampleBuffer:videoSample fromConnection:connectionMock]; + [cam captureOutput:cam.captureVideoOutput + didOutputSampleBuffer:videoSample + fromConnection:connectionMock]; [cam captureOutput:nil didOutputSampleBuffer:audioSample fromConnection:connectionMock]; - CFRelease(videoSample); - CFRelease(audioSample); - NSArray *expectedSamples = @[@"video", @"audio"]; + NSArray *expectedSamples = @[ @"video", @"audio" ]; XCTAssertEqualObjects(writtenSamples, expectedSamples, @"First appended sample must be video."); + + CFRelease(videoSample); + CFRelease(audioSample); } @end From 4136256ff9c5db9e22563dd1fa4049561a2a900e Mon Sep 17 00:00:00 2001 From: misos1 <30872003+misos1@users.noreply.github.com> Date: Thu, 17 Aug 2023 19:16:32 +0200 Subject: [PATCH 13/14] [camera_avfoundation] update comment and version --- packages/camera/camera_avfoundation/CHANGELOG.md | 5 ++++- .../example/ios/RunnerTests/FLTCamSampleBufferTests.m | 2 +- packages/camera/camera_avfoundation/ios/Classes/FLTCam.m | 1 + packages/camera/camera_avfoundation/pubspec.yaml | 2 +- 4 files changed, 7 insertions(+), 3 deletions(-) diff --git a/packages/camera/camera_avfoundation/CHANGELOG.md b/packages/camera/camera_avfoundation/CHANGELOG.md index f1fbf8666c2..88e0ae355c2 100644 --- a/packages/camera/camera_avfoundation/CHANGELOG.md +++ b/packages/camera/camera_avfoundation/CHANGELOG.md @@ -1,6 +1,9 @@ -## 0.9.13+3 +## 0.9.13+4 * Ignores audio samples until the first video sample arrives. + +## NEXT + * Fixes unawaited_futures violations. ## 0.9.13+2 diff --git a/packages/camera/camera_avfoundation/example/ios/RunnerTests/FLTCamSampleBufferTests.m b/packages/camera/camera_avfoundation/example/ios/RunnerTests/FLTCamSampleBufferTests.m index 2a02a9dc273..6f0a4edab08 100644 --- a/packages/camera/camera_avfoundation/example/ios/RunnerTests/FLTCamSampleBufferTests.m +++ b/packages/camera/camera_avfoundation/example/ios/RunnerTests/FLTCamSampleBufferTests.m @@ -38,7 +38,7 @@ - (void)testCopyPixelBuffer { CFRelease(deliveriedPixelBuffer); } -- (void)testFirstAppendedSampleShouldBeVideo { +- (void)testDidOutputSampleBufferIgnoreAudioSamplesBeforeVideoSamples { FLTCam *cam = FLTCreateCamWithCaptureSessionQueue(dispatch_queue_create("testing", NULL)); CMSampleBufferRef videoSample = FLTCreateTestSampleBuffer(); CMSampleBufferRef audioSample = FLTCreateTestAudioSampleBuffer(); diff --git a/packages/camera/camera_avfoundation/ios/Classes/FLTCam.m b/packages/camera/camera_avfoundation/ios/Classes/FLTCam.m index 9fd90a43acf..e0f03000d45 100644 --- a/packages/camera/camera_avfoundation/ios/Classes/FLTCam.m +++ b/packages/camera/camera_avfoundation/ios/Classes/FLTCam.m @@ -501,6 +501,7 @@ - (void)captureOutput:(AVCaptureOutput *)output } // ignore audio samples until the first video sample arrives to avoid black frames + // https://github.com/flutter/flutter/issues/57831 if (_videoWriter.status != AVAssetWriterStatusWriting && output != _captureVideoOutput) { return; } diff --git a/packages/camera/camera_avfoundation/pubspec.yaml b/packages/camera/camera_avfoundation/pubspec.yaml index 19cb3d126ca..94d50501af4 100644 --- a/packages/camera/camera_avfoundation/pubspec.yaml +++ b/packages/camera/camera_avfoundation/pubspec.yaml @@ -2,7 +2,7 @@ name: camera_avfoundation description: iOS implementation of the camera plugin. repository: https://github.com/flutter/packages/tree/main/packages/camera/camera_avfoundation issue_tracker: https://github.com/flutter/flutter/issues?q=is%3Aissue+is%3Aopen+label%3A%22p%3A+camera%22 -version: 0.9.13+3 +version: 0.9.13+4 environment: sdk: ">=2.18.0 <4.0.0" From e67fb8238edd8051e2210ad6502b55dff0e5a770 Mon Sep 17 00:00:00 2001 From: misos1 <30872003+misos1@users.noreply.github.com> Date: Mon, 11 Sep 2023 19:52:11 +0200 Subject: [PATCH 14/14] Update pubspec.yaml --- packages/camera/camera_avfoundation/pubspec.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/camera/camera_avfoundation/pubspec.yaml b/packages/camera/camera_avfoundation/pubspec.yaml index d986235f8e5..e13e957cad7 100644 --- a/packages/camera/camera_avfoundation/pubspec.yaml +++ b/packages/camera/camera_avfoundation/pubspec.yaml @@ -2,7 +2,7 @@ name: camera_avfoundation description: iOS implementation of the camera plugin. repository: https://github.com/flutter/packages/tree/main/packages/camera/camera_avfoundation issue_tracker: https://github.com/flutter/flutter/issues?q=is%3Aissue+is%3Aopen+label%3A%22p%3A+camera%22 -version: 0.9.13+4 +version: 0.9.13+5 environment: sdk: ">=2.19.0 <4.0.0"