Add mobile voice notes

mobile/ios/Podfile.lock

@@ -3,6 +3,8 @@ PODS:
     - Flutter
   - app_links (6.4.1):
     - Flutter
+  - audio_session (0.0.1):
+    - Flutter
   - camera_avfoundation (0.0.1):
     - Flutter
   - connectivity_plus (0.0.1):
@@ -45,6 +47,9 @@ PODS:
   - GTMSessionFetcher/Core (3.5.0)
   - image_picker_ios (0.0.1):
     - Flutter
+  - just_audio (0.0.1):
+    - Flutter
+    - FlutterMacOS
   - local_auth_darwin (0.0.1):
     - Flutter
     - FlutterMacOS
@@ -86,6 +91,8 @@ PODS:
     - Flutter
     - FlutterMacOS
   - PromisesObjC (2.4.0)
+  - record_ios (1.2.1):
+    - Flutter
   - share_plus (0.0.1):
     - Flutter
   - shared_preferences_foundation (0.0.1):
@@ -100,6 +107,7 @@ PODS:
 DEPENDENCIES:
   - app_badge_plus (from `.symlinks/plugins/app_badge_plus/ios`)
   - app_links (from `.symlinks/plugins/app_links/ios`)
+  - audio_session (from `.symlinks/plugins/audio_session/ios`)
   - camera_avfoundation (from `.symlinks/plugins/camera_avfoundation/ios`)
   - connectivity_plus (from `.symlinks/plugins/connectivity_plus/ios`)
   - file_selector_ios (from `.symlinks/plugins/file_selector_ios/ios`)
@@ -108,11 +116,13 @@ DEPENDENCIES:
   - google_mlkit_commons (from `.symlinks/plugins/google_mlkit_commons/ios`)
   - google_mlkit_selfie_segmentation (from `.symlinks/plugins/google_mlkit_selfie_segmentation/ios`)
   - image_picker_ios (from `.symlinks/plugins/image_picker_ios/ios`)
+  - just_audio (from `.symlinks/plugins/just_audio/darwin`)
   - local_auth_darwin (from `.symlinks/plugins/local_auth_darwin/darwin`)
   - mobile_scanner (from `.symlinks/plugins/mobile_scanner/darwin`)
   - open_filex (from `.symlinks/plugins/open_filex/ios`)
   - package_info_plus (from `.symlinks/plugins/package_info_plus/ios`)
   - photo_manager (from `.symlinks/plugins/photo_manager/darwin`)
+  - record_ios (from `.symlinks/plugins/record_ios/ios`)
   - share_plus (from `.symlinks/plugins/share_plus/ios`)
   - shared_preferences_foundation (from `.symlinks/plugins/shared_preferences_foundation/darwin`)
   - url_launcher_ios (from `.symlinks/plugins/url_launcher_ios/ios`)
@@ -139,6 +149,8 @@ EXTERNAL SOURCES:
     :path: ".symlinks/plugins/app_badge_plus/ios"
   app_links:
     :path: ".symlinks/plugins/app_links/ios"
+  audio_session:
+    :path: ".symlinks/plugins/audio_session/ios"
   camera_avfoundation:
     :path: ".symlinks/plugins/camera_avfoundation/ios"
   connectivity_plus:
@@ -155,6 +167,8 @@ EXTERNAL SOURCES:
     :path: ".symlinks/plugins/google_mlkit_selfie_segmentation/ios"
   image_picker_ios:
     :path: ".symlinks/plugins/image_picker_ios/ios"
+  just_audio:
+    :path: ".symlinks/plugins/just_audio/darwin"
   local_auth_darwin:
     :path: ".symlinks/plugins/local_auth_darwin/darwin"
   mobile_scanner:
@@ -165,6 +179,8 @@ EXTERNAL SOURCES:
     :path: ".symlinks/plugins/package_info_plus/ios"
   photo_manager:
     :path: ".symlinks/plugins/photo_manager/darwin"
+  record_ios:
+    :path: ".symlinks/plugins/record_ios/ios"
   share_plus:
     :path: ".symlinks/plugins/share_plus/ios"
   shared_preferences_foundation:
@@ -177,6 +193,7 @@ EXTERNAL SOURCES:
 SPEC CHECKSUMS:
   app_badge_plus: 09939f19a075cc742cc155d8ed85e6d8601f0104
   app_links: 3dbc685f76b1693c66a6d9dd1e9ab6f73d97dc0a
+  audio_session: 9bb7f6c970f21241b19f5a3658097ae459681ba0
   camera_avfoundation: 968a9a5323c79a99c166ad9d7866bfd2047b5a9b
   connectivity_plus: cb623214f4e1f6ef8fe7403d580fdad517d2f7dd
   file_selector_ios: ec57ec07954363dd730b642e765e58f199bb621a
@@ -190,6 +207,7 @@ SPEC CHECKSUMS:
   GoogleUtilities: 26a3abef001b6533cf678d3eb38fd3f614b7872d
   GTMSessionFetcher: 5aea5ba6bd522a239e236100971f10cb71b96ab6
   image_picker_ios: e0ece4aa2a75771a7de3fa735d26d90817041326
+  just_audio: 4e391f57b79cad2b0674030a00453ca5ce817eed
   local_auth_darwin: c3ee6cce0a8d56be34c8ccb66ba31f7f180aaebb
   MLImage: 0de5c6c2bf9e93b80ef752e2797f0836f03b58c0
   MLKitCommon: 47d47b50a031d00db62f1b0efe5a1d8b09a3b2e6
@@ -203,6 +221,7 @@ SPEC CHECKSUMS:
   package_info_plus: af8e2ca6888548050f16fa2f1938db7b5a5df499
   photo_manager: 6ab48c2ce7ec21aa06d59e6cc049f0b6d9ba7f94
   PromisesObjC: f5707f49cb48b9636751c5b2e7d227e43fba9f47
+  record_ios: 980fd386a97a35987d0fce3dfda4b26a38c90f4c
   share_plus: 50da8cb520a8f0f65671c6c6a99b3617ed10a58a
   shared_preferences_foundation: 7036424c3d8ec98dfe75ff1667cb0cd531ec82bb
   url_launcher_ios: 7a95fa5b60cc718a708b8f2966718e93db0cef1b

mobile/ios/Runner/AppDelegate.swift

@@ -625,6 +625,18 @@ import os.log
         return
       }
       transcodeVideoToMp4(sourcePath: sourcePath, result: result)
+    case "packageVoiceNoteForUpload":
+      guard let sourcePath = call.arguments as? String else {
+        result(
+          FlutterError(
+            code: "invalid_arguments",
+            message: "Expected source file path as String.",
+            details: nil
+          )
+        )
+        return
+      }
+      packageVoiceNoteForUpload(sourcePath: sourcePath, result: result)
     case "generateVideoPoster":
       guard let sourcePath = call.arguments as? String else {
         result(
@@ -668,6 +680,274 @@ import os.log
     return image.pngData()
   }
 
+  private func packageVoiceNoteForUpload(
+    sourcePath: String,
+    result: @escaping FlutterResult
+  ) {
+    let sourceAsset = AVURLAsset(url: URL(fileURLWithPath: sourcePath))
+    guard let sourceAudio = sourceAsset.tracks(withMediaType: .audio).first else {
+      result(
+        FlutterError(
+          code: "transcode_failed",
+          message: "The recording does not contain an audio track.",
+          details: nil
+        )
+      )
+      return
+    }
+
+    let duration = sourceAudio.timeRange.duration
+    guard duration.isValid, duration.isNumeric, CMTimeCompare(duration, .zero) > 0 else {
+      result(
+        FlutterError(
+          code: "transcode_failed",
+          message: "The recording has no playable audio.",
+          details: nil
+        )
+      )
+      return
+    }
+
+    Self.makeVoiceNoteVideoTrack(duration: duration) { trackResult in
+      switch trackResult {
+      case .failure(let error):
+        result(
+          FlutterError(
+            code: "transcode_failed",
+            message: "Unable to prepare voice note for upload.",
+            details: error.localizedDescription
+          )
+        )
+      case .success(let videoURL):
+        self.exportVoiceNoteEnvelope(
+          sourceURL: URL(fileURLWithPath: sourcePath),
+          duration: duration,
+          videoURL: videoURL,
+          result: result
+        )
+      }
+    }
+  }
+
+  private static func makeVoiceNoteVideoTrack(
+    duration: CMTime,
+    completion: @escaping (Result<URL, Error>) -> Void
+  ) {
+    let outputURL = FileManager.default.temporaryDirectory
+      .appendingPathComponent(UUID().uuidString)
+      .appendingPathExtension("mp4")
+
+    do {
+      let writer = try AVAssetWriter(outputURL: outputURL, fileType: .mp4)
+      let input = AVAssetWriterInput(
+        mediaType: .video,
+        outputSettings: [
+          AVVideoCodecKey: AVVideoCodecType.h264,
+          AVVideoWidthKey: 16,
+          AVVideoHeightKey: 16,
+          AVVideoCompressionPropertiesKey: [
+            AVVideoAverageBitRateKey: 8_000,
+            AVVideoExpectedSourceFrameRateKey: 1,
+            AVVideoMaxKeyFrameIntervalKey: 1,
+          ],
+        ]
+      )
+      input.expectsMediaDataInRealTime = false
+      guard writer.canAdd(input) else {
+        throw NSError(
+          domain: "BuzzVoiceNote",
+          code: 1,
+          userInfo: [NSLocalizedDescriptionKey: "Unable to create the video envelope."]
+        )
+      }
+      writer.add(input)
+      let adaptor = AVAssetWriterInputPixelBufferAdaptor(
+        assetWriterInput: input,
+        sourcePixelBufferAttributes: [
+          kCVPixelBufferPixelFormatTypeKey as String: kCVPixelFormatType_32BGRA,
+          kCVPixelBufferWidthKey as String: 16,
+          kCVPixelBufferHeightKey as String: 16,
+        ]
+      )
+      guard writer.startWriting() else {
+        throw writer.error
+          ?? NSError(
+            domain: "BuzzVoiceNote",
+            code: 2,
+            userInfo: [NSLocalizedDescriptionKey: "Unable to start the video envelope."]
+          )
+      }
+      writer.startSession(atSourceTime: .zero)
+
+      let queue = DispatchQueue(label: "xyz.block.buzz.voice-note-envelope")
+      var appendedFrames = false
+      input.requestMediaDataWhenReady(on: queue) {
+        guard !appendedFrames, input.isReadyForMoreMediaData else { return }
+        appendedFrames = true
+        guard
+          let pool = adaptor.pixelBufferPool,
+          let buffer = Self.makeBlackPixelBuffer(pool: pool),
+          adaptor.append(buffer, withPresentationTime: .zero),
+          adaptor.append(buffer, withPresentationTime: duration)
+        else {
+          writer.cancelWriting()
+          try? FileManager.default.removeItem(at: outputURL)
+          completion(
+            .failure(
+              writer.error
+                ?? NSError(
+                  domain: "BuzzVoiceNote",
+                  code: 3,
+                  userInfo: [NSLocalizedDescriptionKey: "Unable to write the video envelope."]
+                )
+            )
+          )
+          return
+        }
+        input.markAsFinished()
+        writer.endSession(atSourceTime: duration)
+        writer.finishWriting {
+          if writer.status == .completed {
+            completion(.success(outputURL))
+          } else {
+            try? FileManager.default.removeItem(at: outputURL)
+            completion(
+              .failure(
+                writer.error
+                  ?? NSError(
+                    domain: "BuzzVoiceNote",
+                    code: 4,
+                    userInfo: [NSLocalizedDescriptionKey: "Unable to finish the video envelope."]
+                  )
+              )
+            )
+          }
+        }
+      }
+    } catch {
+      try? FileManager.default.removeItem(at: outputURL)
+      completion(.failure(error))
+    }
+  }
+
+  private static func makeBlackPixelBuffer(pool: CVPixelBufferPool) -> CVPixelBuffer? {
+    var pixelBuffer: CVPixelBuffer?
+    guard CVPixelBufferPoolCreatePixelBuffer(nil, pool, &pixelBuffer) == kCVReturnSuccess,
+      let pixelBuffer
+    else {
+      return nil
+    }
+    CVPixelBufferLockBaseAddress(pixelBuffer, [])
+    if let baseAddress = CVPixelBufferGetBaseAddress(pixelBuffer) {
+      memset(baseAddress, 0, CVPixelBufferGetDataSize(pixelBuffer))
+    }
+    CVPixelBufferUnlockBaseAddress(pixelBuffer, [])
+    return pixelBuffer
+  }
+
+  private func exportVoiceNoteEnvelope(
+    sourceURL: URL,
+    duration: CMTime,
+    videoURL: URL,
+    result: @escaping FlutterResult
+  ) {
+    // Reload the recording here so its AVAsset stays alive for the entire
+    // composition insert. Keeping only an AVAssetTrack across the asynchronous
+    // video-envelope write can leave the track detached from its source asset
+    // on physical devices.
+    let sourceAsset = AVURLAsset(url: sourceURL)
+    let videoAsset = AVURLAsset(url: videoURL)
+    let composition = AVMutableComposition()
+    do {
+      guard
+        let sourceAudio = sourceAsset.tracks(withMediaType: .audio).first,
+        let sourceVideo = videoAsset.tracks(withMediaType: .video).first,
+        let destinationVideo = composition.addMutableTrack(
+          withMediaType: .video,
+          preferredTrackID: kCMPersistentTrackID_Invalid
+        ),
+        let destinationAudio = composition.addMutableTrack(
+          withMediaType: .audio,
+          preferredTrackID: kCMPersistentTrackID_Invalid
+        )
+      else {
+        throw NSError(
+          domain: "BuzzVoiceNote",
+          code: 5,
+          userInfo: [NSLocalizedDescriptionKey: "Unable to assemble the voice note envelope."]
+        )
+      }
+      let sourceVideoRange = sourceVideo.timeRange
+      try destinationVideo.insertTimeRange(sourceVideoRange, of: sourceVideo, at: .zero)
+      destinationVideo.scaleTimeRange(
+        CMTimeRange(start: .zero, duration: sourceVideoRange.duration),
+        toDuration: duration
+      )
+      try destinationAudio.insertTimeRange(sourceAudio.timeRange, of: sourceAudio, at: .zero)
+    } catch {
+      try? FileManager.default.removeItem(at: videoURL)
+      result(
+        FlutterError(
+          code: "transcode_failed",
+          message: "Unable to assemble voice note for upload.",
+          details: error.localizedDescription
+        )
+      )
+      return
+    }
+
+    guard let exportSession = AVAssetExportSession(
+      asset: composition,
+      presetName: AVAssetExportPresetMediumQuality
+    ) else {
+      try? FileManager.default.removeItem(at: videoURL)
+      result(
+        FlutterError(
+          code: "transcode_failed",
+          message: "Unable to create voice note export session.",
+          details: nil
+        )
+      )
+      return
+    }
+    let outputURL = FileManager.default.temporaryDirectory
+      .appendingPathComponent(UUID().uuidString)
+      .appendingPathExtension("mp4")
+    exportSession.outputURL = outputURL
+    exportSession.outputFileType = .mp4
+    exportSession.shouldOptimizeForNetworkUse = true
+    exportSession.metadata = []
+    exportSession.metadataItemFilter = nil
+    exportSession.exportAsynchronously {
+      try? FileManager.default.removeItem(at: videoURL)
+      switch exportSession.status {
+      case .completed:
+        do {
+          try Self.neutralizeSampleDependencyBoxes(at: outputURL)
+          result(outputURL.path)
+        } catch {
+          try? FileManager.default.removeItem(at: outputURL)
+          result(
+            FlutterError(
+              code: "transcode_failed",
+              message: "Unable to canonicalize voice note.",
+              details: error.localizedDescription
+            )
+          )
+        }
+      default:
+        try? FileManager.default.removeItem(at: outputURL)
+        result(
+          FlutterError(
+            code: "transcode_failed",
+            message: "Voice note packaging failed.",
+            details: exportSession.error?.localizedDescription
+          )
+        )
+      }
+    }
+  }
+
   private func transcodeVideoToMp4(
     sourcePath: String,
     result: @escaping FlutterResult