feat: Use vision for MRZ scanning (SEL-47)

app/ios/CameraView.swift

@@ -0,0 +1,83 @@
+// CameraView.swift
+// SwiftUI camera preview with frame capture callback
+
+import SwiftUI
+import AVFoundation
+
+struct CameraView: UIViewControllerRepresentable {
+    var frameHandler: (UIImage) -> Void
+    var captureInterval: TimeInterval = 0.5 // seconds
+
+    func makeUIViewController(context: Context) -> CameraViewController {
+        let controller = CameraViewController()
+        controller.frameHandler = frameHandler
+        controller.captureInterval = captureInterval
+        return controller
+    }
+
+    func updateUIViewController(_ uiViewController: CameraViewController, context: Context) {}
+}
+
+class CameraViewController: UIViewController, AVCaptureVideoDataOutputSampleBufferDelegate {
+    var frameHandler: ((UIImage) -> Void)?
+    var captureInterval: TimeInterval = 0.5
+    private let session = AVCaptureSession()
+    private let videoOutput = AVCaptureVideoDataOutput()
+    private var lastCaptureTime = Date(timeIntervalSince1970: 0)
+    private var previewLayer: AVCaptureVideoPreviewLayer?
+
+    override func viewDidLoad() {
+        super.viewDidLoad()
+        setupCamera()
+    }
+
+    private func setupCamera() {
+        session.beginConfiguration()
+        guard let device = AVCaptureDevice.default(.builtInWideAngleCamera, for: .video, position: .back),
+              let input = try? AVCaptureDeviceInput(device: device) else { return }
+        if session.canAddInput(input) { session.addInput(input) }
+        videoOutput.setSampleBufferDelegate(self, queue: DispatchQueue(label: "camera.frame.queue"))
+        if session.canAddOutput(videoOutput) { session.addOutput(videoOutput) }
+        session.commitConfiguration()
+        previewLayer = AVCaptureVideoPreviewLayer(session: session)
+        previewLayer?.videoGravity = .resizeAspectFill
+        previewLayer?.frame = view.bounds
+        if let previewLayer = previewLayer {
+            view.layer.addSublayer(previewLayer)
+        }
+        session.startRunning()
+    }
+
+    override func viewDidLayoutSubviews() {
+        super.viewDidLayoutSubviews()
+        previewLayer?.frame = view.bounds
+    }
+
+    func captureOutput(_ output: AVCaptureOutput, didOutput sampleBuffer: CMSampleBuffer, from connection: AVCaptureConnection) {
+        let now = Date()
+        guard now.timeIntervalSince(lastCaptureTime) >= captureInterval else { return }
+        lastCaptureTime = now
+        guard let imageBuffer = CMSampleBufferGetImageBuffer(sampleBuffer) else { return }
+        let ciImage = CIImage(cvPixelBuffer: imageBuffer)
+        let context = CIContext()
+        if let cgImage = context.createCGImage(ciImage, from: ciImage.extent) {
+            let originalImage = UIImage(cgImage: cgImage, scale: UIScreen.main.scale, orientation: .right)
+            // Rotate to .up orientation
+            let uprightImage = originalImage.fixedOrientation()
+            DispatchQueue.main.async { [weak self] in
+                self?.frameHandler?(uprightImage)
+            }
+        }
+    }
+}
+
+extension UIImage {
+    func fixedOrientation() -> UIImage {
+        if imageOrientation == .up { return self }
+        UIGraphicsBeginImageContextWithOptions(size, false, scale)
+        draw(in: CGRect(origin: .zero, size: size))
+        let normalizedImage = UIGraphicsGetImageFromCurrentImageContext() ?? self
+        UIGraphicsEndImageContext()
+        return normalizedImage
+    }
+} 

app/ios/LiveMRZScannerView.swift

@@ -0,0 +1,123 @@
+// LiveMRZScannerView.swift
+
+import SwiftUI
+import QKMRZParser
+
+struct LiveMRZScannerView: View {
+    @State private var recognizedText: String = ""
+    @State private var lastMRZDetection: Date = Date()
+    @State private var parsedMRZ: QKMRZResult? = nil
+    @State private var scanComplete: Bool = false
+    var onScanComplete: ((QKMRZResult) -> Void)? = nil
+    var onScanResultAsDict: (([String: Any]) -> Void)? = nil
+
+    func singleCorrectDocumentNumberInMRZ(result: String, docNumber: String, parser: QKMRZParser) -> QKMRZResult? {
+        let replacements: [Character: [Character]] = [
+            "0": ["O"], "O": ["0"],
+            "1": ["I"], "I": ["1"],
+            "2": ["Z"], "Z": ["2"],
+            "8": ["B"], "B": ["8"]
+        ]
+        let lines = result.components(separatedBy: "\n")
+        guard lines.count >= 2 else { return nil }
+        for (i, char) in docNumber.enumerated() {
+            if let subs = replacements[char] {
+                for sub in subs {
+                    var chars = Array(docNumber)
+                    chars[i] = sub
+                    let candidate = String(chars)
+                    if let range = lines[1].range(of: docNumber) {
+                        var newLine = lines[1]
+                        let start = newLine.distance(from: newLine.startIndex, to: range.lowerBound)
+                        var lineChars = Array(newLine)
+                        let docNumChars = Array(candidate)
+                        for j in 0..<min(docNumber.count, docNumChars.count) {
+                            lineChars[start + j] = docNumChars[j]
+                        }
+                        newLine = String(lineChars)
+                        var newLines = lines
+                        newLines[1] = newLine
+                        let correctedMRZ = newLines.joined(separator: "\n")
+                        print("Trying candidate: \(candidate), correctedMRZ: \(correctedMRZ)")
+                        if let correctedResult = parser.parse(mrzString: correctedMRZ) {
+                          if correctedResult.isDocumentNumberValid {
+                            return correctedResult
+                          }
+                        }
+                    }
+                }
+            }
+        }
+        return nil
+    }
+
+    private func mapVisionResultToDictionary(_ result: QKMRZResult) -> [String: Any] {
+        return [
+            "documentType": result.documentType,
+            "countryCode": result.countryCode,
+            "surnames": result.surnames,
+            "givenNames": result.givenNames,
+            "documentNumber": result.documentNumber,
+            "nationalityCountryCode": result.nationalityCountryCode,
+            "dateOfBirth": result.birthdate?.description ?? "",
+            "sex": result.sex ?? "",
+            "expiryDate": result.expiryDate?.description ?? "",
+            "personalNumber": result.personalNumber,
+            "personalNumber2": result.personalNumber2 ?? "",
+            "isDocumentNumberValid": result.isDocumentNumberValid,
+            "isBirthdateValid": result.isBirthdateValid,
+            "isExpiryDateValid": result.isExpiryDateValid,
+            "isPersonalNumberValid": result.isPersonalNumberValid ?? false,
+            "allCheckDigitsValid": result.allCheckDigitsValid
+        ]
+    }
+
+    var body: some View {
+        ZStack(alignment: .bottom) {
+                CameraView { image in
+                    // print("[LiveMRZScannerView] CameraView frame received. Size: \(image.size), Orientation: \(image.imageOrientation.rawValue)")
+                    if scanComplete { return }
+                    MRZScanner.scan(image: image) { result, boxes in
+                        recognizedText = result
+                        lastMRZDetection = Date()
+                        let parser = QKMRZParser(ocrCorrection: false)
+                        if let mrzResult = parser.parse(mrzString: result) {
+                            let doc = mrzResult;
+                            if doc.allCheckDigitsValid == true && !scanComplete {
+                                parsedMRZ = mrzResult
+                                scanComplete = true
+                                onScanComplete?(mrzResult)
+                                onScanResultAsDict?(mapVisionResultToDictionary(mrzResult))
+                            } else if doc.isDocumentNumberValid == false && !scanComplete {
+                                if let correctedResult = singleCorrectDocumentNumberInMRZ(result: result, docNumber: doc.documentNumber, parser: parser) {
+                                    let correctedDoc = correctedResult
+                                    if correctedDoc.allCheckDigitsValid == true {
+                                        parsedMRZ = correctedResult
+                                        scanComplete = true
+                                        onScanComplete?(correctedResult)
+                                        onScanResultAsDict?(mapVisionResultToDictionary(correctedResult))
+                                    }
+                                }
+                            }
+                        } else {
+                            if !scanComplete {
+                                parsedMRZ = nil
+                            }
+                        }
+                    }
+                }
+
+            VStack {
+                if !scanComplete {
+                    Text("Position the camera 30-40cm away from the passport for best results")
+                        .font(.footnote)
+                        .padding()
+                        .background(Color.black.opacity(0.7))
+                        .foregroundColor(.white)
+                        .cornerRadius(8)
+                        .padding(.bottom, 40)
+                }
+            }
+        }
+    }
+}

app/ios/MRZScanner.swift

@@ -0,0 +1,51 @@
+//
+//  MRZScanner.swift
+
+import Vision
+import UIKit
+
+struct MRZScanner {
+    static func scan(image: UIImage, completion: @escaping (String, [CGRect]) -> Void) {
+        guard let cgImage = image.cgImage else {
+            completion("Image not valid", [])
+            return
+        }
+
+        let request = VNRecognizeTextRequest { (request, error) in
+            guard let observations = request.results as? [VNRecognizedTextObservation] else {
+                completion("No text found", [])
+                return
+            }
+
+            var mrzLines: [String] = []
+            var boxes: [CGRect] = []
+            for obs in observations {
+                //   if let text = obs.topCandidates(1).first?.string, text.contains("<") {
+                    // mrzLines.append(text)
+                if let candidate = obs.topCandidates(1).first, candidate.string.contains("<") {
+                    mrzLines.append(candidate.string)
+                    boxes.append(obs.boundingBox) // Normalized coordinates
+                    // Log confidence for each character
+//                    for (i, char) in candidate.string.enumerated() {
+//                        if let box = try? candidate.boundingBox(for: candidate.string.index(candidate.string.startIndex, offsetBy: i)..<candidate.string.index(candidate.string.startIndex, offsetBy: i+1)) {
+//                            print("Char: \(char), Confidence: \(box.confidence)")
+//                        }
+//                    }
+                }
+            }
+
+            if mrzLines.isEmpty {
+                completion("", [])
+            } else {
+                completion(mrzLines.joined(separator: "\n"), boxes)
+            }
+        }
+        request.recognitionLevel = .accurate
+        request.usesLanguageCorrection = false
+        request.recognitionLanguages = ["en"]
+        let handler = VNImageRequestHandler(cgImage: cgImage, options: [:])
+        DispatchQueue.global(qos: .userInitiated).async {
+            try? handler.perform([request])
+        }
+    }
+}

app/ios/PassportOCRViewManager.swift

@@ -1,5 +1,4 @@
 import Foundation
-import QKMRZScanner
 import React
 import SwiftUI
 import UIKit
@@ -19,8 +18,7 @@ class PassportOCRView: UIView {
     @objc var onPassportRead: RCTDirectEventBlock?
     @objc var onError: RCTDirectEventBlock?
 
-    private var scannerView: QKMRZScannerViewRepresentable?
-    private var hostingController: UIHostingController<QKMRZScannerViewRepresentable>?
+    private var hostingController: UIHostingController<LiveMRZScannerView>?
 
     override init(frame: CGRect) {
         super.init(frame: frame)
@@ -33,22 +31,20 @@ class PassportOCRView: UIView {
     }
 
     private func initializeScanner() {
-        var scannerView = QKMRZScannerViewRepresentable()
-        scannerView.onScanResult = { [weak self] scanResult in
-            let resultDict: [String: Any] = [
-                "documentNumber": scanResult.documentNumber,
-                "expiryDate": scanResult.expiryDate?.description ?? "",
-                "birthDate": scanResult.birthdate?.description ?? "",
-            ]
-            self?.onPassportRead?(["data": resultDict])
-        }
-
+        let scannerView = LiveMRZScannerView(
+            onScanResultAsDict: { [weak self] resultDict in
+              self?.onPassportRead?([
+                "data": [
+                  "documentNumber": resultDict["documentNumber"] as? String ?? "",
+                  "expiryDate": resultDict["expiryDate"] as? String ?? "",
+                  "birthDate": resultDict["dateOfBirth"] as? String ?? ""
+                ]])
+            }
+        )
         let hostingController = UIHostingController(rootView: scannerView)
         hostingController.view.backgroundColor = .clear
         hostingController.view.translatesAutoresizingMaskIntoConstraints = false
         addSubview(hostingController.view)
-
-        self.scannerView = scannerView
         self.hostingController = hostingController
     }
 

app/ios/Self.xcodeproj/project.pbxproj

@@ -38,6 +38,9 @@
 		A109328F471241A5A931D524 /* Inter-Light.otf in Resources */ = {isa = PBXBuildFile; fileRef = 070CF9E82E3E45DAB6BBA375 /* Inter-Light.otf */; };
 		AE6147EC2DC95A8D00445C0F /* GoogleService-Info.plist in Resources */ = {isa = PBXBuildFile; fileRef = AE6147EB2DC95A8D00445C0F /* GoogleService-Info.plist */; };
 		B0885FC3EE2A41A1AA97EEC0 /* Inter-LightItalic.otf in Resources */ = {isa = PBXBuildFile; fileRef = F4AE10DA498844DF8BF01948 /* Inter-LightItalic.otf */; };
+		BF1044812DD53540009B3688 /* LiveMRZScannerView.swift in Sources */ = {isa = PBXBuildFile; fileRef = BF1044802DD53540009B3688 /* LiveMRZScannerView.swift */; };
+		BF1044832DD5354F009B3688 /* CameraView.swift in Sources */ = {isa = PBXBuildFile; fileRef = BF1044822DD5354F009B3688 /* CameraView.swift */; };
+		BF1044852DD53570009B3688 /* MRZScanner.swift in Sources */ = {isa = PBXBuildFile; fileRef = BF1044842DD53570009B3688 /* MRZScanner.swift */; };
 		C9F1B4F4F38F49EF8723594E /* Inter-MediumItalic.otf in Resources */ = {isa = PBXBuildFile; fileRef = 06064B357139453EB2788C18 /* Inter-MediumItalic.otf */; };
 		CB116B311D63491FA54CCEE1 /* Inter-Thin.otf in Resources */ = {isa = PBXBuildFile; fileRef = DD642F4F3A114B43A22296D7 /* Inter-Thin.otf */; };
 		CC02892C62AE4BB5B7F769EA /* Inter-Medium.otf in Resources */ = {isa = PBXBuildFile; fileRef = 15C30A8C4A6C42558DC9D78B /* Inter-Medium.otf */; };
@@ -115,6 +118,9 @@
 		9BF744D9A73A4BAC96EC569A /* DINOT-Medium.otf */ = {isa = PBXFileReference; explicitFileType = undefined; fileEncoding = 9; includeInIndex = 0; lastKnownFileType = unknown; name = "DINOT-Medium.otf"; path = "../src/assets/fonts/DINOT-Medium.otf"; sourceTree = "<group>"; };
 		AE6147EB2DC95A8D00445C0F /* GoogleService-Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist; path = "GoogleService-Info.plist"; sourceTree = "<group>"; };
 		B4E7218406B64A95BCE0DFE4 /* slkscrb.ttf */ = {isa = PBXFileReference; explicitFileType = undefined; fileEncoding = 9; includeInIndex = 0; lastKnownFileType = unknown; name = slkscrb.ttf; path = "../node_modules/@tamagui/font-silkscreen/files/slkscrb.ttf"; sourceTree = "<group>"; };
+		BF1044802DD53540009B3688 /* LiveMRZScannerView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = LiveMRZScannerView.swift; sourceTree = "<group>"; };
+		BF1044822DD5354F009B3688 /* CameraView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CameraView.swift; sourceTree = "<group>"; };
+		BF1044842DD53570009B3688 /* MRZScanner.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MRZScanner.swift; sourceTree = "<group>"; };
 		C56F122245594D6DA9B7570A /* slkscr.woff */ = {isa = PBXFileReference; explicitFileType = undefined; fileEncoding = 9; includeInIndex = 0; lastKnownFileType = unknown; name = slkscr.woff; path = "../node_modules/@tamagui/font-silkscreen/files/slkscr.woff"; sourceTree = "<group>"; };
 		CA67A75B161A05334E3E9402 /* Pods-OpenPassport.debug.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-OpenPassport.debug.xcconfig"; path = "Target Support Files/Pods-OpenPassport/Pods-OpenPassport.debug.xcconfig"; sourceTree = "<group>"; };
 		CBF96649C103ADB7297A6A7F /* Pods_Self.framework */ = {isa = PBXFileReference; explicitFileType = wrapper.framework; includeInIndex = 0; path = Pods_Self.framework; sourceTree = BUILT_PRODUCTS_DIR; };
@@ -146,6 +152,9 @@
 		13B07FAE1A68108700A75B9A /* OpenPassport */ = {
 			isa = PBXGroup;
 			children = (
+				BF1044842DD53570009B3688 /* MRZScanner.swift */,
+				BF1044822DD5354F009B3688 /* CameraView.swift */,
+				BF1044802DD53540009B3688 /* LiveMRZScannerView.swift */,
 				AE6147EB2DC95A8D00445C0F /* GoogleService-Info.plist */,
 				E9F9A99A2D57FE2900E1362E /* PassportOCRViewManager.m */,
 				E9F9A99B2D57FE2900E1362E /* PassportOCRViewManager.swift */,
@@ -470,10 +479,12 @@
 				13B07FBC1A68108700A75B9A /* AppDelegate.mm in Sources */,
 				1686F0DE2C500F4F00841CDE /* QRScannerViewController.swift in Sources */,
 				1686F0E02C500FBD00841CDE /* QRScannerBridge.m in Sources */,
+				BF1044832DD5354F009B3688 /* CameraView.swift in Sources */,
 				1686F0DC2C500F3800841CDE /* QRScannerBridge.swift in Sources */,
 				13B07FC11A68108700A75B9A /* main.m in Sources */,
 				905B70072A72774000AFA232 /* PassportReader.m in Sources */,
 				16E6646E2B8D292500FDD6A0 /* QKMRZScannerViewRepresentable.swift in Sources */,
+				BF1044852DD53570009B3688 /* MRZScanner.swift in Sources */,
 				165E76BF2B8DC53A0000FA90 /* MRZScannerModule.m in Sources */,
 				905B70052A72767900AFA232 /* PassportReader.swift in Sources */,
 				165E76C32B8DC8370000FA90 /* ScannerHostingController.swift in Sources */,
@@ -482,6 +493,7 @@
 				1648EB782CC9564D003BEA7D /* LottieView.swift in Sources */,
 				164FD9672D569A640067E63B /* QRCodeScannerViewManager.swift in Sources */,
 				165E76BD2B8DC4A00000FA90 /* MRZScannerModule.swift in Sources */,
+				BF1044812DD53540009B3688 /* LiveMRZScannerView.swift in Sources */,
 				164FD9692D569C1F0067E63B /* QRCodeScannerViewManager.m in Sources */,
 			);
 			runOnlyForDeploymentPostprocessing = 0;