From 918eeed13170e87e81e8e1694dd44c372abb2939 Mon Sep 17 00:00:00 2001 From: seshanthS Date: Thu, 15 May 2025 01:17:29 +0530 Subject: [PATCH 1/2] feat: Use vision for MRZ scanning --- app/ios/CameraView.swift | 83 +++++++++++++++++ app/ios/LiveMRZScannerView.swift | 122 +++++++++++++++++++++++++ app/ios/MRZScanner.swift | 51 +++++++++++ app/ios/PassportOCRViewManager.swift | 26 +++--- app/ios/Self.xcodeproj/project.pbxproj | 12 +++ 5 files changed, 279 insertions(+), 15 deletions(-) create mode 100644 app/ios/CameraView.swift create mode 100644 app/ios/LiveMRZScannerView.swift create mode 100644 app/ios/MRZScanner.swift diff --git a/app/ios/CameraView.swift b/app/ios/CameraView.swift new file mode 100644 index 000000000..96052de22 --- /dev/null +++ b/app/ios/CameraView.swift @@ -0,0 +1,83 @@ +// CameraView.swift +// SwiftUI camera preview with frame capture callback + +import SwiftUI +import AVFoundation + +struct CameraView: UIViewControllerRepresentable { + var frameHandler: (UIImage) -> Void + var captureInterval: TimeInterval = 0.5 // seconds + + func makeUIViewController(context: Context) -> CameraViewController { + let controller = CameraViewController() + controller.frameHandler = frameHandler + controller.captureInterval = captureInterval + return controller + } + + func updateUIViewController(_ uiViewController: CameraViewController, context: Context) {} +} + +class CameraViewController: UIViewController, AVCaptureVideoDataOutputSampleBufferDelegate { + var frameHandler: ((UIImage) -> Void)? + var captureInterval: TimeInterval = 0.5 + private let session = AVCaptureSession() + private let videoOutput = AVCaptureVideoDataOutput() + private var lastCaptureTime = Date(timeIntervalSince1970: 0) + private var previewLayer: AVCaptureVideoPreviewLayer? + + override func viewDidLoad() { + super.viewDidLoad() + setupCamera() + } + + private func setupCamera() { + session.beginConfiguration() + guard let device = AVCaptureDevice.default(.builtInWideAngleCamera, for: .video, position: .back), + let input = try? AVCaptureDeviceInput(device: device) else { return } + if session.canAddInput(input) { session.addInput(input) } + videoOutput.setSampleBufferDelegate(self, queue: DispatchQueue(label: "camera.frame.queue")) + if session.canAddOutput(videoOutput) { session.addOutput(videoOutput) } + session.commitConfiguration() + previewLayer = AVCaptureVideoPreviewLayer(session: session) + previewLayer?.videoGravity = .resizeAspectFill + previewLayer?.frame = view.bounds + if let previewLayer = previewLayer { + view.layer.addSublayer(previewLayer) + } + session.startRunning() + } + + override func viewDidLayoutSubviews() { + super.viewDidLayoutSubviews() + previewLayer?.frame = view.bounds + } + + func captureOutput(_ output: AVCaptureOutput, didOutput sampleBuffer: CMSampleBuffer, from connection: AVCaptureConnection) { + let now = Date() + guard now.timeIntervalSince(lastCaptureTime) >= captureInterval else { return } + lastCaptureTime = now + guard let imageBuffer = CMSampleBufferGetImageBuffer(sampleBuffer) else { return } + let ciImage = CIImage(cvPixelBuffer: imageBuffer) + let context = CIContext() + if let cgImage = context.createCGImage(ciImage, from: ciImage.extent) { + let originalImage = UIImage(cgImage: cgImage, scale: UIScreen.main.scale, orientation: .right) + // Rotate to .up orientation + let uprightImage = originalImage.fixedOrientation() + DispatchQueue.main.async { [weak self] in + self?.frameHandler?(uprightImage) + } + } + } +} + +extension UIImage { + func fixedOrientation() -> UIImage { + if imageOrientation == .up { return self } + UIGraphicsBeginImageContextWithOptions(size, false, scale) + draw(in: CGRect(origin: .zero, size: size)) + let normalizedImage = UIGraphicsGetImageFromCurrentImageContext() ?? self + UIGraphicsEndImageContext() + return normalizedImage + } +} \ No newline at end of file diff --git a/app/ios/LiveMRZScannerView.swift b/app/ios/LiveMRZScannerView.swift new file mode 100644 index 000000000..ffdff6aae --- /dev/null +++ b/app/ios/LiveMRZScannerView.swift @@ -0,0 +1,122 @@ +// LiveMRZScannerView.swift + +import SwiftUI +import QKMRZParser + +struct LiveMRZScannerView: View { + @State private var recognizedText: String = "" + @State private var lastMRZDetection: Date = Date() + @State private var parsedMRZ: QKMRZResult? = nil + @State private var scanComplete: Bool = false + var onScanComplete: ((QKMRZResult) -> Void)? = nil + var onScanResultAsDict: (([String: Any]) -> Void)? = nil + + func singleCorrectDocumentNumberInMRZ(result: String, docNumber: String, parser: QKMRZParser) -> QKMRZResult? { + let replacements: [Character: [Character]] = [ + "0": ["O"], "O": ["0"], + "1": ["I"], "I": ["1"], + "2": ["Z"], "Z": ["2"], + "8": ["B"], "B": ["8"] + ] + let lines = result.components(separatedBy: "\n") + guard lines.count >= 2 else { return nil } + for (i, char) in docNumber.enumerated() { + if let subs = replacements[char] { + for sub in subs { + var chars = Array(docNumber) + chars[i] = sub + let candidate = String(chars) + if let range = lines[1].range(of: docNumber) { + var newLine = lines[1] + let start = newLine.distance(from: newLine.startIndex, to: range.lowerBound) + var lineChars = Array(newLine) + let docNumChars = Array(candidate) + for j in 0.. [String: Any] { + return [ + "documentType": result.documentType, + "countryCode": result.countryCode, + "surnames": result.surnames, + "givenNames": result.givenNames, + "documentNumber": result.documentNumber, + "nationalityCountryCode": result.nationalityCountryCode, + "dateOfBirth": result.birthdate?.description ?? "", + "sex": result.sex ?? "", + "expiryDate": result.expiryDate?.description ?? "", + "personalNumber": result.personalNumber, + "personalNumber2": result.personalNumber2 ?? "", + "isDocumentNumberValid": result.isDocumentNumberValid, + "isBirthdateValid": result.isBirthdateValid, + "isExpiryDateValid": result.isExpiryDateValid, + "isPersonalNumberValid": result.isPersonalNumberValid ?? false, + "allCheckDigitsValid": result.allCheckDigitsValid + ] + } + + var body: some View { + ZStack(alignment: .bottom) { + CameraView { image in + // print("[LiveMRZScannerView] CameraView frame received. Size: \(image.size), Orientation: \(image.imageOrientation.rawValue)") + if scanComplete { return } + MRZScanner.scan(image: image) { result, boxes in + recognizedText = result + lastMRZDetection = Date() + let parser = QKMRZParser(ocrCorrection: false) + if let mrzResult = parser.parse(mrzString: result) { + let doc = mrzResult; + if doc.allCheckDigitsValid == true && !scanComplete { + parsedMRZ = mrzResult + scanComplete = true + onScanComplete?(mrzResult) + onScanResultAsDict?(mapVisionResultToDictionary(mrzResult)) + } else if doc.isDocumentNumberValid == false && !scanComplete { + if let correctedResult = singleCorrectDocumentNumberInMRZ(result: result, docNumber: doc.documentNumber, parser: parser) { + let correctedDoc = correctedResult + if correctedDoc.allCheckDigitsValid == true { + parsedMRZ = correctedResult + scanComplete = true + onScanComplete?(correctedResult) + onScanResultAsDict?(mapVisionResultToDictionary(correctedResult)) + } + } + } + } else { + if !scanComplete { + parsedMRZ = nil + } + } + } + } + + VStack { + if !scanComplete { + Text("Scanning... Hold steady for a valid scan.") + .padding() + .background(Color.black.opacity(0.7)) + .foregroundColor(.white) + .cornerRadius(8) + .padding(.bottom, 40) + } + } + } + } +} diff --git a/app/ios/MRZScanner.swift b/app/ios/MRZScanner.swift new file mode 100644 index 000000000..f493ff99a --- /dev/null +++ b/app/ios/MRZScanner.swift @@ -0,0 +1,51 @@ +// +// MRZScanner.swift + +import Vision +import UIKit + +struct MRZScanner { + static func scan(image: UIImage, completion: @escaping (String, [CGRect]) -> Void) { + guard let cgImage = image.cgImage else { + completion("Image not valid", []) + return + } + + let request = VNRecognizeTextRequest { (request, error) in + guard let observations = request.results as? [VNRecognizedTextObservation] else { + completion("No text found", []) + return + } + + var mrzLines: [String] = [] + var boxes: [CGRect] = [] + for obs in observations { + // if let text = obs.topCandidates(1).first?.string, text.contains("<") { + // mrzLines.append(text) + if let candidate = obs.topCandidates(1).first, candidate.string.contains("<") { + mrzLines.append(candidate.string) + boxes.append(obs.boundingBox) // Normalized coordinates + // Log confidence for each character +// for (i, char) in candidate.string.enumerated() { +// if let box = try? candidate.boundingBox(for: candidate.string.index(candidate.string.startIndex, offsetBy: i)..? + private var hostingController: UIHostingController? override init(frame: CGRect) { super.init(frame: frame) @@ -33,22 +31,20 @@ class PassportOCRView: UIView { } private func initializeScanner() { - var scannerView = QKMRZScannerViewRepresentable() - scannerView.onScanResult = { [weak self] scanResult in - let resultDict: [String: Any] = [ - "documentNumber": scanResult.documentNumber, - "expiryDate": scanResult.expiryDate?.description ?? "", - "birthDate": scanResult.birthdate?.description ?? "", - ] - self?.onPassportRead?(["data": resultDict]) - } - + let scannerView = LiveMRZScannerView( + onScanResultAsDict: { [weak self] resultDict in + self?.onPassportRead?([ + "data": [ + "documentNumber": resultDict["documentNumber"] as? String ?? "", + "expiryDate": resultDict["expiryDate"] as? String ?? "", + "birthDate": resultDict["dateOfBirth"] as? String ?? "" + ]]) + } + ) let hostingController = UIHostingController(rootView: scannerView) hostingController.view.backgroundColor = .clear hostingController.view.translatesAutoresizingMaskIntoConstraints = false addSubview(hostingController.view) - - self.scannerView = scannerView self.hostingController = hostingController } diff --git a/app/ios/Self.xcodeproj/project.pbxproj b/app/ios/Self.xcodeproj/project.pbxproj index ee74136f7..83e5c358c 100644 --- a/app/ios/Self.xcodeproj/project.pbxproj +++ b/app/ios/Self.xcodeproj/project.pbxproj @@ -37,6 +37,9 @@ 9C694E1EF05C49DF85423A6A /* Inter-Bold.otf in Resources */ = {isa = PBXBuildFile; fileRef = 3BF771EA645241D9A28A3AE9 /* Inter-Bold.otf */; }; A109328F471241A5A931D524 /* Inter-Light.otf in Resources */ = {isa = PBXBuildFile; fileRef = 070CF9E82E3E45DAB6BBA375 /* Inter-Light.otf */; }; B0885FC3EE2A41A1AA97EEC0 /* Inter-LightItalic.otf in Resources */ = {isa = PBXBuildFile; fileRef = F4AE10DA498844DF8BF01948 /* Inter-LightItalic.otf */; }; + BF1044812DD53540009B3688 /* LiveMRZScannerView.swift in Sources */ = {isa = PBXBuildFile; fileRef = BF1044802DD53540009B3688 /* LiveMRZScannerView.swift */; }; + BF1044832DD5354F009B3688 /* CameraView.swift in Sources */ = {isa = PBXBuildFile; fileRef = BF1044822DD5354F009B3688 /* CameraView.swift */; }; + BF1044852DD53570009B3688 /* MRZScanner.swift in Sources */ = {isa = PBXBuildFile; fileRef = BF1044842DD53570009B3688 /* MRZScanner.swift */; }; C9F1B4F4F38F49EF8723594E /* Inter-MediumItalic.otf in Resources */ = {isa = PBXBuildFile; fileRef = 06064B357139453EB2788C18 /* Inter-MediumItalic.otf */; }; CB116B311D63491FA54CCEE1 /* Inter-Thin.otf in Resources */ = {isa = PBXBuildFile; fileRef = DD642F4F3A114B43A22296D7 /* Inter-Thin.otf */; }; CC02892C62AE4BB5B7F769EA /* Inter-Medium.otf in Resources */ = {isa = PBXBuildFile; fileRef = 15C30A8C4A6C42558DC9D78B /* Inter-Medium.otf */; }; @@ -113,6 +116,9 @@ 905B70082A729CD400AFA232 /* OpenPassport.entitlements */ = {isa = PBXFileReference; lastKnownFileType = text.plist.entitlements; name = OpenPassport.entitlements; path = OpenPassport/OpenPassport.entitlements; sourceTree = ""; }; 9BF744D9A73A4BAC96EC569A /* DINOT-Medium.otf */ = {isa = PBXFileReference; explicitFileType = undefined; fileEncoding = 9; includeInIndex = 0; lastKnownFileType = unknown; name = "DINOT-Medium.otf"; path = "../src/assets/fonts/DINOT-Medium.otf"; sourceTree = ""; }; B4E7218406B64A95BCE0DFE4 /* slkscrb.ttf */ = {isa = PBXFileReference; explicitFileType = undefined; fileEncoding = 9; includeInIndex = 0; lastKnownFileType = unknown; name = slkscrb.ttf; path = "../node_modules/@tamagui/font-silkscreen/files/slkscrb.ttf"; sourceTree = ""; }; + BF1044802DD53540009B3688 /* LiveMRZScannerView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = LiveMRZScannerView.swift; sourceTree = ""; }; + BF1044822DD5354F009B3688 /* CameraView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CameraView.swift; sourceTree = ""; }; + BF1044842DD53570009B3688 /* MRZScanner.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MRZScanner.swift; sourceTree = ""; }; C56F122245594D6DA9B7570A /* slkscr.woff */ = {isa = PBXFileReference; explicitFileType = undefined; fileEncoding = 9; includeInIndex = 0; lastKnownFileType = unknown; name = slkscr.woff; path = "../node_modules/@tamagui/font-silkscreen/files/slkscr.woff"; sourceTree = ""; }; CA67A75B161A05334E3E9402 /* Pods-OpenPassport.debug.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-OpenPassport.debug.xcconfig"; path = "Target Support Files/Pods-OpenPassport/Pods-OpenPassport.debug.xcconfig"; sourceTree = ""; }; CBF96649C103ADB7297A6A7F /* Pods_Self.framework */ = {isa = PBXFileReference; explicitFileType = wrapper.framework; includeInIndex = 0; path = Pods_Self.framework; sourceTree = BUILT_PRODUCTS_DIR; }; @@ -144,6 +150,9 @@ 13B07FAE1A68108700A75B9A /* OpenPassport */ = { isa = PBXGroup; children = ( + BF1044842DD53570009B3688 /* MRZScanner.swift */, + BF1044822DD5354F009B3688 /* CameraView.swift */, + BF1044802DD53540009B3688 /* LiveMRZScannerView.swift */, E9F9A99A2D57FE2900E1362E /* PassportOCRViewManager.m */, E9F9A99B2D57FE2900E1362E /* PassportOCRViewManager.swift */, 169349842CC694DA00166F21 /* OpenPassportDebug.entitlements */, @@ -452,10 +461,12 @@ 13B07FBC1A68108700A75B9A /* AppDelegate.mm in Sources */, 1686F0DE2C500F4F00841CDE /* QRScannerViewController.swift in Sources */, 1686F0E02C500FBD00841CDE /* QRScannerBridge.m in Sources */, + BF1044832DD5354F009B3688 /* CameraView.swift in Sources */, 1686F0DC2C500F3800841CDE /* QRScannerBridge.swift in Sources */, 13B07FC11A68108700A75B9A /* main.m in Sources */, 905B70072A72774000AFA232 /* PassportReader.m in Sources */, 16E6646E2B8D292500FDD6A0 /* QKMRZScannerViewRepresentable.swift in Sources */, + BF1044852DD53570009B3688 /* MRZScanner.swift in Sources */, 165E76BF2B8DC53A0000FA90 /* MRZScannerModule.m in Sources */, 905B70052A72767900AFA232 /* PassportReader.swift in Sources */, 165E76C32B8DC8370000FA90 /* ScannerHostingController.swift in Sources */, @@ -464,6 +475,7 @@ 1648EB782CC9564D003BEA7D /* LottieView.swift in Sources */, 164FD9672D569A640067E63B /* QRCodeScannerViewManager.swift in Sources */, 165E76BD2B8DC4A00000FA90 /* MRZScannerModule.swift in Sources */, + BF1044812DD53540009B3688 /* LiveMRZScannerView.swift in Sources */, 164FD9692D569C1F0067E63B /* QRCodeScannerViewManager.m in Sources */, ); runOnlyForDeploymentPostprocessing = 0; From 734a876ef4a97273781bd1e9be4a19d3a8de1374 Mon Sep 17 00:00:00 2001 From: turnoffthiscomputer Date: Fri, 16 May 2025 16:01:45 +0200 Subject: [PATCH 2/2] modify label to position the smartphone during the OCR scan --- app/ios/LiveMRZScannerView.swift | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/app/ios/LiveMRZScannerView.swift b/app/ios/LiveMRZScannerView.swift index ffdff6aae..034cf4044 100644 --- a/app/ios/LiveMRZScannerView.swift +++ b/app/ios/LiveMRZScannerView.swift @@ -109,7 +109,8 @@ struct LiveMRZScannerView: View { VStack { if !scanComplete { - Text("Scanning... Hold steady for a valid scan.") + Text("Position the camera 30-40cm away from the passport for best results") + .font(.footnote) .padding() .background(Color.black.opacity(0.7)) .foregroundColor(.white) @@ -119,4 +120,4 @@ struct LiveMRZScannerView: View { } } } -} +}