Fix OpenAI file transcription finalization

TypeWhisper.xcodeproj/project.pbxproj

@@ -4960,7 +4960,7 @@
 				INFOPLIST_KEY_NSPrincipalClass = OpenAIPlugin;
 				INSTALL_PATH = "$(LOCAL_LIBRARY_DIR)/Bundles";
 				MACOSX_DEPLOYMENT_TARGET = 14.0;
-				MARKETING_VERSION = 1.0;
+				MARKETING_VERSION = 1.3.1;
 				PRODUCT_BUNDLE_IDENTIFIER = com.typewhisper.openai;
 				PRODUCT_NAME = OpenAIPlugin;
 				SKIP_INSTALL = YES;
@@ -4983,7 +4983,7 @@
 				INFOPLIST_KEY_NSPrincipalClass = OpenAIPlugin;
 				INSTALL_PATH = "$(LOCAL_LIBRARY_DIR)/Bundles";
 				MACOSX_DEPLOYMENT_TARGET = 14.0;
-				MARKETING_VERSION = 1.0;
+				MARKETING_VERSION = 1.3.1;
 				PRODUCT_BUNDLE_IDENTIFIER = com.typewhisper.openai;
 				PRODUCT_NAME = OpenAIPlugin;
 				SKIP_INSTALL = YES;

TypeWhisperPluginSDK/Plugins/OpenAIPlugin/OpenAIPlugin.swift

@@ -625,8 +625,8 @@ enum OpenAILiveTranscriptionDelay: String, CaseIterable, Sendable {
     }
 }
 
-private struct OpenAITranscriptionModelCapability {
-    enum Transport {
+private struct OpenAITranscriptionModelCapability: Sendable {
+    enum Transport: Sendable {
         case legacyFile(responseFormat: String)
         case contextAwareFile
         case legacyRealtime
@@ -1042,6 +1042,117 @@ private final class OpenAIRealtimeWebSocketDelegate: NSObject, URLSessionWebSock
     }
 }
 
+actor OpenAIFileTranscriptionSession: LiveTranscriptionSession {
+    typealias Transcribe = @Sendable (AudioData) async throws -> PluginTranscriptionResult
+
+    private static let sampleRate = 16_000
+    private static let previewIntervalSampleCount = sampleRate * 3
+    private static let previewWindowSampleCount = sampleRate * 10
+    private static let previewAnalysisFrameSampleCount = sampleRate / 10
+    private static let previewSpeechRMSFloor: Float = 0.004
+    private static let previewSustainedSilenceSampleCount = Int(Double(sampleRate) * 1.4)
+
+    private let transcribe: Transcribe
+    private let onProgress: @Sendable (String) -> Bool
+    private var bufferedSamples: [Float] = []
+    private var lastPreviewSampleCount = 0
+    private var isPreviewInFlight = false
+    private var didFinish = false
+    private var isCancelled = false
+
+    init(
+        transcribe: @escaping Transcribe,
+        onProgress: @Sendable @escaping (String) -> Bool
+    ) {
+        self.transcribe = transcribe
+        self.onProgress = onProgress
+    }
+
+    func appendAudio(samples: [Float]) async throws {
+        guard !samples.isEmpty, !didFinish, !isCancelled else { return }
+
+        bufferedSamples.append(contentsOf: samples)
+        guard !isPreviewInFlight else { return }
+
+        isPreviewInFlight = true
+        defer { isPreviewInFlight = false }
+
+        while !didFinish, !isCancelled, !Task.isCancelled {
+            let currentSampleCount = bufferedSamples.count
+            guard currentSampleCount - lastPreviewSampleCount >= Self.previewIntervalSampleCount else {
+                break
+            }
+            lastPreviewSampleCount = currentSampleCount
+
+            let previewSamples = Array(bufferedSamples.suffix(Self.previewWindowSampleCount))
+            guard Self.shouldRequestPreview(for: previewSamples) else { continue }
+
+            do {
+                let result = try await transcribe(Self.audioData(from: previewSamples))
+                guard !didFinish, !isCancelled, !Task.isCancelled else { break }
+
+                let text = result.text.trimmingCharacters(in: .whitespacesAndNewlines)
+                if !text.isEmpty {
+                    _ = onProgress(text)
+                }
+            } catch {
+                // A best-effort preview must not prevent the full buffered audio from
+                // being finalized when the user stops dictation.
+            }
+        }
+    }
+
+    func finish() async throws -> PluginTranscriptionResult {
+        guard !isCancelled else { throw CancellationError() }
+        guard !didFinish else {
+            throw PluginTranscriptionError.apiError("File transcription session is already finished.")
+        }
+        didFinish = true
+
+        let result = try await transcribe(Self.audioData(from: bufferedSamples))
+        try Task.checkCancellation()
+        guard !isCancelled else { throw CancellationError() }
+        return result
+    }
+
+    func cancel() async {
+        isCancelled = true
+        bufferedSamples.removeAll(keepingCapacity: false)
+    }
+
+    private static func audioData(from samples: [Float]) -> AudioData {
+        AudioData(
+            samples: samples,
+            wavData: PluginWavEncoder.encode(samples, sampleRate: sampleRate),
+            duration: Double(samples.count) / Double(sampleRate)
+        )
+    }
+
+    private static func shouldRequestPreview(for samples: [Float]) -> Bool {
+        guard !samples.isEmpty else { return false }
+
+        var containsSpeech = false
+        var trailingQuietSampleCount = 0
+        var offset = 0
+
+        while offset < samples.count {
+            let end = min(samples.count, offset + previewAnalysisFrameSampleCount)
+            let frame = samples[offset..<end]
+            let rms = sqrt(frame.reduce(Float.zero) { $0 + $1 * $1 } / Float(frame.count))
+
+            if rms >= previewSpeechRMSFloor {
+                containsSpeech = true
+                trailingQuietSampleCount = 0
+            } else {
+                trailingQuietSampleCount += frame.count
+            }
+            offset = end
+        }
+
+        return containsSpeech && trailingQuietSampleCount < previewSustainedSilenceSampleCount
+    }
+}
+
 final class OpenAIRealtimeTranscriptionSession: LiveTranscriptionSession, @unchecked Sendable {
     static let modelId = OpenAITranscriptionModelCapability.legacyRealtimeModelID
     static let sourceSampleRate = 16_000
@@ -2037,12 +2148,25 @@ final class OpenAIPlugin: NSObject,
             throw PluginTranscriptionError.notConfigured
         }
         let capability = try selectedTranscriptionCapability()
+        try validateTranslationRequest(translate, capability: capability)
+
         guard capability.isRealtime else {
-            throw PluginTranscriptionError.apiError(
-                "\(capability.modelInfo.displayName) is a file transcription model."
+            return OpenAIFileTranscriptionSession(
+                transcribe: { [self] audio in
+                    try await performTranscription(
+                        audio: audio,
+                        languageSelection: languageSelection,
+                        translate: translate,
+                        prompt: prompt,
+                        dictionaryTermHints: dictionaryTermHints,
+                        onProgress: { _ in true },
+                        apiKeyOverride: apiKey,
+                        capabilityOverride: capability
+                    )
+                },
+                onProgress: onProgress
             )
         }
-        try validateTranslationRequest(translate, capability: capability)
 
         return try await OpenAIRealtimeTranscriptionSession.connect(
             apiKey: apiKey,
@@ -2062,12 +2186,19 @@ final class OpenAIPlugin: NSObject,
         translate: Bool,
         prompt: String?,
         dictionaryTermHints: [PluginDictionaryTermHint],
-        onProgress: @Sendable @escaping (String) -> Bool
+        onProgress: @Sendable @escaping (String) -> Bool,
+        apiKeyOverride: String? = nil,
+        capabilityOverride: OpenAITranscriptionModelCapability? = nil
     ) async throws -> PluginTranscriptionResult {
-        guard let apiKey = normalizedAPIKey else {
+        guard let apiKey = apiKeyOverride ?? normalizedAPIKey else {
             throw PluginTranscriptionError.notConfigured
         }
-        let capability = try selectedTranscriptionCapability()
+        let capability: OpenAITranscriptionModelCapability
+        if let capabilityOverride {
+            capability = capabilityOverride
+        } else {
+            capability = try selectedTranscriptionCapability()
+        }
         try validateTranslationRequest(translate, capability: capability)
 
         switch capability.transport {
@@ -2564,7 +2695,7 @@ final class OpenAIPlugin: NSObject,
     private static var chatGPTModelsClientVersion: String {
         let bundle = Bundle(for: OpenAIPlugin.self)
         return bundle.object(forInfoDictionaryKey: "CFBundleShortVersionString") as? String
-            ?? "1.3.0"
+            ?? "1.3.1"
     }
 
     fileprivate var ttsInstructions: String { _ttsInstructions }

TypeWhisperPluginSDK/Plugins/OpenAIPlugin/Tests/OpenAIPluginTests.swift

@@ -649,27 +649,88 @@ final class OpenAIPluginTests: XCTestCase {
         XCTAssertTrue(input["turn_detection"] is NSNull)
     }
 
-    func testFileModelRejectsNativeLiveSessionBeforeOpeningSocket() async throws {
+    func testFileModelLiveSessionFinalizesTheCompleteBufferedAudio() async throws {
         let host = try PluginTestHostServices(
             defaults: ["selectedModel": "gpt-transcribe"],
             secrets: ["api-key": "sk-live"]
         )
         let plugin = OpenAIPlugin()
         plugin.activate(host: host)
 
-        do {
-            _ = try await plugin.createLiveTranscriptionSession(
-                language: "de",
-                translate: false,
-                prompt: nil,
-                onProgress: { _ in true }
-            )
-            XCTFail("Expected file model to reject native live transcription")
-        } catch PluginTranscriptionError.apiError(let message) {
-            XCTAssertEqual(message, "GPT Transcribe is a file transcription model.")
-        } catch {
-            XCTFail("Unexpected error: \(error)")
+        let store = PluginHTTPClientSessionStore()
+        PluginHTTPClientTestHarness.configure { _ in
+            store.makeSession(outcomes: [
+                .success(
+                    Data(#"{"text":"The second ending is present.","languages":[{"code":"en"}]}"#.utf8),
+                    Self.httpResponse(
+                        url: "https://api.openai.com/v1/audio/transcriptions",
+                        statusCode: 200
+                    )
+                ),
+            ])
         }
+
+        let session = try await plugin.createLiveTranscriptionSession(
+            language: "en",
+            translate: false,
+            prompt: nil,
+            onProgress: { _ in true }
+        )
+        try await session.appendAudio(samples: [Float](repeating: 0.1, count: 8_000))
+        try await session.appendAudio(samples: [Float](repeating: 0.2, count: 8_000))
+
+        let result = try await session.finish()
+
+        XCTAssertEqual(result.text, "The second ending is present.")
+        XCTAssertEqual(result.detectedLanguage, "en")
+        XCTAssertEqual(store.sessions.first?.requestedPaths, ["/v1/audio/transcriptions"])
+
+        let request = try XCTUnwrap(store.sessions.first?.requestedRequests.first)
+        let body = String(decoding: try XCTUnwrap(request.httpBody), as: UTF8.self)
+        XCTAssertTrue(body.contains("name=\"model\"\r\n\r\ngpt-transcribe"))
+        XCTAssertTrue(body.contains("name=\"languages[]\"\r\n\r\nen"))
+    }
+
+    func testFileModelLiveSessionKeepsPreviewAndFinalTranscriptionSeparate() async throws {
+        let recorder = OpenAIFileTranscriptionRecorder()
+        let previews = OpenAIFileTranscriptionPreviewRecorder()
+        let session = OpenAIFileTranscriptionSession(
+            transcribe: { audio in
+                await recorder.record(sampleCount: audio.samples.count)
+                return PluginTranscriptionResult(text: "\(audio.samples.count) samples")
+            },
+            onProgress: { text in
+                previews.record(text)
+                return true
+            }
+        )
+
+        try await session.appendAudio(samples: [Float](repeating: 0.1, count: 48_000))
+        try await session.appendAudio(samples: [Float](repeating: 0.1, count: 16_000))
+        let result = try await session.finish()
+
+        let recordedSampleCounts = await recorder.sampleCounts
+        XCTAssertEqual(recordedSampleCounts, [48_000, 64_000])
+        XCTAssertEqual(previews.values, ["48000 samples"])
+        XCTAssertEqual(result.text, "64000 samples")
+    }
+
+    func testFileModelLiveSessionSerializesAndCoalescesReentrantPreviews() async throws {
+        let sessionBox = OpenAIFileTranscriptionSessionBox()
+        let recorder = ReentrantOpenAIFileTranscriptionRecorder(sessionBox: sessionBox)
+        let session = OpenAIFileTranscriptionSession(
+            transcribe: { audio in
+                try await recorder.transcribe(audio)
+            },
+            onProgress: { _ in true }
+        )
+        await sessionBox.store(session)
+
+        try await session.appendAudio(samples: [Float](repeating: 0.1, count: 48_000))
+
+        let snapshot = await recorder.snapshot
+        XCTAssertEqual(snapshot.sampleCounts, [48_000, 96_000])
+        XCTAssertEqual(snapshot.maximumConcurrentRequestCount, 1)
     }
 
     func testOpenAIRealtimePCMConversionResamples16kTo24kPCM16() {
@@ -1080,6 +1141,74 @@ private final class FinishCounter: @unchecked Sendable {
     }
 }
 
+private actor OpenAIFileTranscriptionRecorder {
+    private(set) var sampleCounts: [Int] = []
+
+    func record(sampleCount: Int) {
+        sampleCounts.append(sampleCount)
+    }
+}
+
+private actor OpenAIFileTranscriptionSessionBox {
+    private var session: OpenAIFileTranscriptionSession?
+
+    func store(_ session: OpenAIFileTranscriptionSession) {
+        self.session = session
+    }
+
+    func appendAudio(samples: [Float]) async throws {
+        try await session?.appendAudio(samples: samples)
+    }
+}
+
+private actor ReentrantOpenAIFileTranscriptionRecorder {
+    struct Snapshot {
+        let sampleCounts: [Int]
+        let maximumConcurrentRequestCount: Int
+    }
+
+    private let sessionBox: OpenAIFileTranscriptionSessionBox
+    private var sampleCounts: [Int] = []
+    private var activeRequestCount = 0
+    private var maximumConcurrentRequestCount = 0
+
+    init(sessionBox: OpenAIFileTranscriptionSessionBox) {
+        self.sessionBox = sessionBox
+    }
+
+    func transcribe(_ audio: AudioData) async throws -> PluginTranscriptionResult {
+        activeRequestCount += 1
+        defer { activeRequestCount -= 1 }
+        maximumConcurrentRequestCount = max(maximumConcurrentRequestCount, activeRequestCount)
+        sampleCounts.append(audio.samples.count)
+
+        if sampleCounts.count == 1 {
+            try await sessionBox.appendAudio(samples: [Float](repeating: 0.1, count: 48_000))
+        }
+
+        return PluginTranscriptionResult(text: "\(audio.samples.count) samples")
+    }
+
+    var snapshot: Snapshot {
+        Snapshot(
+            sampleCounts: sampleCounts,
+            maximumConcurrentRequestCount: maximumConcurrentRequestCount
+        )
+    }
+}
+
+private final class OpenAIFileTranscriptionPreviewRecorder: @unchecked Sendable {
+    private let lock = OSAllocatedUnfairLock(initialState: [String]())
+
+    var values: [String] {
+        lock.withLock { $0 }
+    }
+
+    func record(_ value: String) {
+        lock.withLock { $0.append(value) }
+    }
+}
+
 private final class MockOpenAITTSAudioPlayback: OpenAITTSAudioPlayback, @unchecked Sendable {
     private let lock = OSAllocatedUnfairLock(initialState: 0)
     var onDrained: (@Sendable () -> Void)?

TypeWhisperPluginSDK/Plugins/OpenAIPlugin/manifest.json

@@ -1,7 +1,7 @@
 {
     "id": "com.typewhisper.openai",
     "name": "OpenAI / ChatGPT",
-    "version": "1.3.0",
+    "version": "1.3.1",
     "minHostVersion": "1.5.0",
     "sdkCompatibilityVersion": "v1",
     "minOSVersion": "14.0",