diff --git a/app/ios/CameraView.swift b/app/ios/CameraView.swift new file mode 100644 index 000000000..96052de22 --- /dev/null +++ b/app/ios/CameraView.swift @@ -0,0 +1,83 @@ +// CameraView.swift +// SwiftUI camera preview with frame capture callback + +import SwiftUI +import AVFoundation + +struct CameraView: UIViewControllerRepresentable { + var frameHandler: (UIImage) -> Void + var captureInterval: TimeInterval = 0.5 // seconds + + func makeUIViewController(context: Context) -> CameraViewController { + let controller = CameraViewController() + controller.frameHandler = frameHandler + controller.captureInterval = captureInterval + return controller + } + + func updateUIViewController(_ uiViewController: CameraViewController, context: Context) {} +} + +class CameraViewController: UIViewController, AVCaptureVideoDataOutputSampleBufferDelegate { + var frameHandler: ((UIImage) -> Void)? + var captureInterval: TimeInterval = 0.5 + private let session = AVCaptureSession() + private let videoOutput = AVCaptureVideoDataOutput() + private var lastCaptureTime = Date(timeIntervalSince1970: 0) + private var previewLayer: AVCaptureVideoPreviewLayer? + + override func viewDidLoad() { + super.viewDidLoad() + setupCamera() + } + + private func setupCamera() { + session.beginConfiguration() + guard let device = AVCaptureDevice.default(.builtInWideAngleCamera, for: .video, position: .back), + let input = try? AVCaptureDeviceInput(device: device) else { return } + if session.canAddInput(input) { session.addInput(input) } + videoOutput.setSampleBufferDelegate(self, queue: DispatchQueue(label: "camera.frame.queue")) + if session.canAddOutput(videoOutput) { session.addOutput(videoOutput) } + session.commitConfiguration() + previewLayer = AVCaptureVideoPreviewLayer(session: session) + previewLayer?.videoGravity = .resizeAspectFill + previewLayer?.frame = view.bounds + if let previewLayer = previewLayer { + view.layer.addSublayer(previewLayer) + } + session.startRunning() + } + + override func viewDidLayoutSubviews() { + super.viewDidLayoutSubviews() + previewLayer?.frame = view.bounds + } + + func captureOutput(_ output: AVCaptureOutput, didOutput sampleBuffer: CMSampleBuffer, from connection: AVCaptureConnection) { + let now = Date() + guard now.timeIntervalSince(lastCaptureTime) >= captureInterval else { return } + lastCaptureTime = now + guard let imageBuffer = CMSampleBufferGetImageBuffer(sampleBuffer) else { return } + let ciImage = CIImage(cvPixelBuffer: imageBuffer) + let context = CIContext() + if let cgImage = context.createCGImage(ciImage, from: ciImage.extent) { + let originalImage = UIImage(cgImage: cgImage, scale: UIScreen.main.scale, orientation: .right) + // Rotate to .up orientation + let uprightImage = originalImage.fixedOrientation() + DispatchQueue.main.async { [weak self] in + self?.frameHandler?(uprightImage) + } + } + } +} + +extension UIImage { + func fixedOrientation() -> UIImage { + if imageOrientation == .up { return self } + UIGraphicsBeginImageContextWithOptions(size, false, scale) + draw(in: CGRect(origin: .zero, size: size)) + let normalizedImage = UIGraphicsGetImageFromCurrentImageContext() ?? self + UIGraphicsEndImageContext() + return normalizedImage + } +} \ No newline at end of file diff --git a/app/ios/LiveMRZScannerView.swift b/app/ios/LiveMRZScannerView.swift new file mode 100644 index 000000000..034cf4044 --- /dev/null +++ b/app/ios/LiveMRZScannerView.swift @@ -0,0 +1,123 @@ +// LiveMRZScannerView.swift + +import SwiftUI +import QKMRZParser + +struct LiveMRZScannerView: View { + @State private var recognizedText: String = "" + @State private var lastMRZDetection: Date = Date() + @State private var parsedMRZ: QKMRZResult? = nil + @State private var scanComplete: Bool = false + var onScanComplete: ((QKMRZResult) -> Void)? = nil + var onScanResultAsDict: (([String: Any]) -> Void)? = nil + + func singleCorrectDocumentNumberInMRZ(result: String, docNumber: String, parser: QKMRZParser) -> QKMRZResult? { + let replacements: [Character: [Character]] = [ + "0": ["O"], "O": ["0"], + "1": ["I"], "I": ["1"], + "2": ["Z"], "Z": ["2"], + "8": ["B"], "B": ["8"] + ] + let lines = result.components(separatedBy: "\n") + guard lines.count >= 2 else { return nil } + for (i, char) in docNumber.enumerated() { + if let subs = replacements[char] { + for sub in subs { + var chars = Array(docNumber) + chars[i] = sub + let candidate = String(chars) + if let range = lines[1].range(of: docNumber) { + var newLine = lines[1] + let start = newLine.distance(from: newLine.startIndex, to: range.lowerBound) + var lineChars = Array(newLine) + let docNumChars = Array(candidate) + for j in 0.. [String: Any] { + return [ + "documentType": result.documentType, + "countryCode": result.countryCode, + "surnames": result.surnames, + "givenNames": result.givenNames, + "documentNumber": result.documentNumber, + "nationalityCountryCode": result.nationalityCountryCode, + "dateOfBirth": result.birthdate?.description ?? "", + "sex": result.sex ?? "", + "expiryDate": result.expiryDate?.description ?? "", + "personalNumber": result.personalNumber, + "personalNumber2": result.personalNumber2 ?? "", + "isDocumentNumberValid": result.isDocumentNumberValid, + "isBirthdateValid": result.isBirthdateValid, + "isExpiryDateValid": result.isExpiryDateValid, + "isPersonalNumberValid": result.isPersonalNumberValid ?? false, + "allCheckDigitsValid": result.allCheckDigitsValid + ] + } + + var body: some View { + ZStack(alignment: .bottom) { + CameraView { image in + // print("[LiveMRZScannerView] CameraView frame received. Size: \(image.size), Orientation: \(image.imageOrientation.rawValue)") + if scanComplete { return } + MRZScanner.scan(image: image) { result, boxes in + recognizedText = result + lastMRZDetection = Date() + let parser = QKMRZParser(ocrCorrection: false) + if let mrzResult = parser.parse(mrzString: result) { + let doc = mrzResult; + if doc.allCheckDigitsValid == true && !scanComplete { + parsedMRZ = mrzResult + scanComplete = true + onScanComplete?(mrzResult) + onScanResultAsDict?(mapVisionResultToDictionary(mrzResult)) + } else if doc.isDocumentNumberValid == false && !scanComplete { + if let correctedResult = singleCorrectDocumentNumberInMRZ(result: result, docNumber: doc.documentNumber, parser: parser) { + let correctedDoc = correctedResult + if correctedDoc.allCheckDigitsValid == true { + parsedMRZ = correctedResult + scanComplete = true + onScanComplete?(correctedResult) + onScanResultAsDict?(mapVisionResultToDictionary(correctedResult)) + } + } + } + } else { + if !scanComplete { + parsedMRZ = nil + } + } + } + } + + VStack { + if !scanComplete { + Text("Position the camera 30-40cm away from the passport for best results") + .font(.footnote) + .padding() + .background(Color.black.opacity(0.7)) + .foregroundColor(.white) + .cornerRadius(8) + .padding(.bottom, 40) + } + } + } + } +} diff --git a/app/ios/MRZScanner.swift b/app/ios/MRZScanner.swift new file mode 100644 index 000000000..f493ff99a --- /dev/null +++ b/app/ios/MRZScanner.swift @@ -0,0 +1,51 @@ +// +// MRZScanner.swift + +import Vision +import UIKit + +struct MRZScanner { + static func scan(image: UIImage, completion: @escaping (String, [CGRect]) -> Void) { + guard let cgImage = image.cgImage else { + completion("Image not valid", []) + return + } + + let request = VNRecognizeTextRequest { (request, error) in + guard let observations = request.results as? [VNRecognizedTextObservation] else { + completion("No text found", []) + return + } + + var mrzLines: [String] = [] + var boxes: [CGRect] = [] + for obs in observations { + // if let text = obs.topCandidates(1).first?.string, text.contains("<") { + // mrzLines.append(text) + if let candidate = obs.topCandidates(1).first, candidate.string.contains("<") { + mrzLines.append(candidate.string) + boxes.append(obs.boundingBox) // Normalized coordinates + // Log confidence for each character +// for (i, char) in candidate.string.enumerated() { +// if let box = try? candidate.boundingBox(for: candidate.string.index(candidate.string.startIndex, offsetBy: i)..? + private var hostingController: UIHostingController? override init(frame: CGRect) { super.init(frame: frame) @@ -33,22 +31,20 @@ class PassportOCRView: UIView { } private func initializeScanner() { - var scannerView = QKMRZScannerViewRepresentable() - scannerView.onScanResult = { [weak self] scanResult in - let resultDict: [String: Any] = [ - "documentNumber": scanResult.documentNumber, - "expiryDate": scanResult.expiryDate?.description ?? "", - "birthDate": scanResult.birthdate?.description ?? "", - ] - self?.onPassportRead?(["data": resultDict]) - } - + let scannerView = LiveMRZScannerView( + onScanResultAsDict: { [weak self] resultDict in + self?.onPassportRead?([ + "data": [ + "documentNumber": resultDict["documentNumber"] as? String ?? "", + "expiryDate": resultDict["expiryDate"] as? String ?? "", + "birthDate": resultDict["dateOfBirth"] as? String ?? "" + ]]) + } + ) let hostingController = UIHostingController(rootView: scannerView) hostingController.view.backgroundColor = .clear hostingController.view.translatesAutoresizingMaskIntoConstraints = false addSubview(hostingController.view) - - self.scannerView = scannerView self.hostingController = hostingController } diff --git a/app/ios/Self.xcodeproj/project.pbxproj b/app/ios/Self.xcodeproj/project.pbxproj index 9d6a0318b..71323e0b1 100644 --- a/app/ios/Self.xcodeproj/project.pbxproj +++ b/app/ios/Self.xcodeproj/project.pbxproj @@ -38,6 +38,9 @@ A109328F471241A5A931D524 /* Inter-Light.otf in Resources */ = {isa = PBXBuildFile; fileRef = 070CF9E82E3E45DAB6BBA375 /* Inter-Light.otf */; }; AE6147EC2DC95A8D00445C0F /* GoogleService-Info.plist in Resources */ = {isa = PBXBuildFile; fileRef = AE6147EB2DC95A8D00445C0F /* GoogleService-Info.plist */; }; B0885FC3EE2A41A1AA97EEC0 /* Inter-LightItalic.otf in Resources */ = {isa = PBXBuildFile; fileRef = F4AE10DA498844DF8BF01948 /* Inter-LightItalic.otf */; }; + BF1044812DD53540009B3688 /* LiveMRZScannerView.swift in Sources */ = {isa = PBXBuildFile; fileRef = BF1044802DD53540009B3688 /* LiveMRZScannerView.swift */; }; + BF1044832DD5354F009B3688 /* CameraView.swift in Sources */ = {isa = PBXBuildFile; fileRef = BF1044822DD5354F009B3688 /* CameraView.swift */; }; + BF1044852DD53570009B3688 /* MRZScanner.swift in Sources */ = {isa = PBXBuildFile; fileRef = BF1044842DD53570009B3688 /* MRZScanner.swift */; }; C9F1B4F4F38F49EF8723594E /* Inter-MediumItalic.otf in Resources */ = {isa = PBXBuildFile; fileRef = 06064B357139453EB2788C18 /* Inter-MediumItalic.otf */; }; CB116B311D63491FA54CCEE1 /* Inter-Thin.otf in Resources */ = {isa = PBXBuildFile; fileRef = DD642F4F3A114B43A22296D7 /* Inter-Thin.otf */; }; CC02892C62AE4BB5B7F769EA /* Inter-Medium.otf in Resources */ = {isa = PBXBuildFile; fileRef = 15C30A8C4A6C42558DC9D78B /* Inter-Medium.otf */; }; @@ -115,6 +118,9 @@ 9BF744D9A73A4BAC96EC569A /* DINOT-Medium.otf */ = {isa = PBXFileReference; explicitFileType = undefined; fileEncoding = 9; includeInIndex = 0; lastKnownFileType = unknown; name = "DINOT-Medium.otf"; path = "../src/assets/fonts/DINOT-Medium.otf"; sourceTree = ""; }; AE6147EB2DC95A8D00445C0F /* GoogleService-Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist; path = "GoogleService-Info.plist"; sourceTree = ""; }; B4E7218406B64A95BCE0DFE4 /* slkscrb.ttf */ = {isa = PBXFileReference; explicitFileType = undefined; fileEncoding = 9; includeInIndex = 0; lastKnownFileType = unknown; name = slkscrb.ttf; path = "../node_modules/@tamagui/font-silkscreen/files/slkscrb.ttf"; sourceTree = ""; }; + BF1044802DD53540009B3688 /* LiveMRZScannerView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = LiveMRZScannerView.swift; sourceTree = ""; }; + BF1044822DD5354F009B3688 /* CameraView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CameraView.swift; sourceTree = ""; }; + BF1044842DD53570009B3688 /* MRZScanner.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MRZScanner.swift; sourceTree = ""; }; C56F122245594D6DA9B7570A /* slkscr.woff */ = {isa = PBXFileReference; explicitFileType = undefined; fileEncoding = 9; includeInIndex = 0; lastKnownFileType = unknown; name = slkscr.woff; path = "../node_modules/@tamagui/font-silkscreen/files/slkscr.woff"; sourceTree = ""; }; CA67A75B161A05334E3E9402 /* Pods-OpenPassport.debug.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-OpenPassport.debug.xcconfig"; path = "Target Support Files/Pods-OpenPassport/Pods-OpenPassport.debug.xcconfig"; sourceTree = ""; }; CBF96649C103ADB7297A6A7F /* Pods_Self.framework */ = {isa = PBXFileReference; explicitFileType = wrapper.framework; includeInIndex = 0; path = Pods_Self.framework; sourceTree = BUILT_PRODUCTS_DIR; }; @@ -146,6 +152,9 @@ 13B07FAE1A68108700A75B9A /* OpenPassport */ = { isa = PBXGroup; children = ( + BF1044842DD53570009B3688 /* MRZScanner.swift */, + BF1044822DD5354F009B3688 /* CameraView.swift */, + BF1044802DD53540009B3688 /* LiveMRZScannerView.swift */, AE6147EB2DC95A8D00445C0F /* GoogleService-Info.plist */, E9F9A99A2D57FE2900E1362E /* PassportOCRViewManager.m */, E9F9A99B2D57FE2900E1362E /* PassportOCRViewManager.swift */, @@ -470,10 +479,12 @@ 13B07FBC1A68108700A75B9A /* AppDelegate.mm in Sources */, 1686F0DE2C500F4F00841CDE /* QRScannerViewController.swift in Sources */, 1686F0E02C500FBD00841CDE /* QRScannerBridge.m in Sources */, + BF1044832DD5354F009B3688 /* CameraView.swift in Sources */, 1686F0DC2C500F3800841CDE /* QRScannerBridge.swift in Sources */, 13B07FC11A68108700A75B9A /* main.m in Sources */, 905B70072A72774000AFA232 /* PassportReader.m in Sources */, 16E6646E2B8D292500FDD6A0 /* QKMRZScannerViewRepresentable.swift in Sources */, + BF1044852DD53570009B3688 /* MRZScanner.swift in Sources */, 165E76BF2B8DC53A0000FA90 /* MRZScannerModule.m in Sources */, 905B70052A72767900AFA232 /* PassportReader.swift in Sources */, 165E76C32B8DC8370000FA90 /* ScannerHostingController.swift in Sources */, @@ -482,6 +493,7 @@ 1648EB782CC9564D003BEA7D /* LottieView.swift in Sources */, 164FD9672D569A640067E63B /* QRCodeScannerViewManager.swift in Sources */, 165E76BD2B8DC4A00000FA90 /* MRZScannerModule.swift in Sources */, + BF1044812DD53540009B3688 /* LiveMRZScannerView.swift in Sources */, 164FD9692D569C1F0067E63B /* QRCodeScannerViewManager.m in Sources */, ); runOnlyForDeploymentPostprocessing = 0;