feat[notask]: automated OCR performance and quality reporting across all platforms

.github/workflows/integration-test-ocr-onnx.yml

@@ -166,3 +166,33 @@ jobs:
         run: npm run test:integration
         env:
           QASE_API_TOKEN: ${{ secrets.QASE_API_TOKEN }}
+
+      - name: Generate HTML performance report
+        if: always()
+        working-directory: ${{ inputs.workdir || env.PKG_DIR }}
+        shell: bash
+        run: |
+          echo "Checking for performance report..."
+          ls -la test/results/ 2>/dev/null || echo "test/results/ directory not found"
+          if [ -f test/results/performance-report.json ]; then
+            echo "Found performance-report.json, generating HTML..."
+            node ../../scripts/perf-report/aggregate.js \
+              --dir test/results \
+              --output-html test/results/performance-report.html \
+              --output-json test/results/performance-summary.json \
+              --output test/results/performance-report.md
+          else
+            echo "performance-report.json not found — skipping HTML generation"
+          fi
+
+      - name: Upload performance report
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: perf-report-ocr-${{ matrix.os }}-${{ matrix.arch }}-${{ github.run_number }}
+          path: |
+            ${{ inputs.workdir || env.PKG_DIR }}/test/results/performance-report.json
+            ${{ inputs.workdir || env.PKG_DIR }}/test/results/performance-report.html
+            ${{ inputs.workdir || env.PKG_DIR }}/test/results/performance-summary.json
+          retention-days: 90
+          if-no-files-found: ignore

.github/workflows/integration-test-qvac-lib-infer-onnx-tts.yml

@@ -174,3 +174,12 @@ jobs:
         env:
           CHATTERBOX_VARIANT: ${{ matrix.variant }}
           QASE_API_TOKEN: ${{ secrets.QASE_API_TOKEN }}
+
+      - name: Upload performance report
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: perf-report-tts-${{ matrix.platform }}-${{ matrix.arch }}-${{ matrix.variant }}-${{ github.run_number }}
+          path: ${{ inputs.workdir }}/test/results/performance-report.json
+          retention-days: 90
+          if-no-files-found: ignore

.github/workflows/perf-report.yml

@@ -0,0 +1,137 @@
+name: Performance Report
+
+on:
+  schedule:
+    - cron: "0 9 * * 1" # Every Monday 9am UTC
+  workflow_dispatch:
+    inputs:
+      addon:
+        description: "Addon to generate report for"
+        type: choice
+        required: true
+        options:
+          - ocr-onnx
+          - nmtcpp
+          - llamacpp-llm
+          - onnx-tts
+      workflow_name:
+        description: "Integration test workflow name to query"
+        type: choice
+        required: true
+        options:
+          - "Integration Tests (OCR)"
+          - "Mobile Integration Tests (OCR)"
+          - "Integration Tests (NMTCPP)"
+          - "Integration Tests (LLM)"
+          - "Integration Tests (TTS)"
+      runs:
+        description: "Number of recent runs to aggregate"
+        type: number
+        required: false
+        default: 6
+
+jobs:
+  generate-report:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      actions: read
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v6
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v6
+        with:
+          node-version: lts/*
+
+      - name: Generate performance report (manual)
+        if: ${{ github.event_name == 'workflow_dispatch' }}
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          node scripts/perf-report/aggregate.js \
+            --addon "${{ inputs.addon }}" \
+            --workflow "${{ inputs.workflow_name }}" \
+            --runs ${{ inputs.runs }} \
+            --output "reports/${{ inputs.addon }}-performance.md" \
+            --output-json "reports/${{ inputs.addon }}-performance.json" \
+            --output-html "reports/${{ inputs.addon }}-performance.html"
+
+      - name: Generate performance reports (scheduled - all addons)
+        if: ${{ github.event_name == 'schedule' }}
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          mkdir -p reports
+
+          echo "=== OCR (Desktop) ==="
+          node scripts/perf-report/aggregate.js \
+            --addon ocr-onnx \
+            --workflow "Integration Tests (OCR)" \
+            --runs 6 \
+            --output reports/ocr-onnx-performance.md \
+            --output-json reports/ocr-onnx-performance.json \
+            --output-html reports/ocr-onnx-performance.html || true
+
+          echo "=== OCR (Mobile) ==="
+          node scripts/perf-report/aggregate.js \
+            --addon ocr-onnx \
+            --workflow "Mobile Integration Tests (OCR)" \
+            --runs 6 \
+            --output reports/ocr-onnx-mobile-performance.md \
+            --output-json reports/ocr-onnx-mobile-performance.json \
+            --output-html reports/ocr-onnx-mobile-performance.html || true
+
+          echo "=== Translation ==="
+          node scripts/perf-report/aggregate.js \
+            --addon nmtcpp \
+            --workflow "Integration Tests (NMTCPP)" \
+            --runs 6 \
+            --output reports/nmtcpp-performance.md \
+            --output-json reports/nmtcpp-performance.json \
+            --output-html reports/nmtcpp-performance.html || true
+
+          echo "=== Vision/LLM ==="
+          node scripts/perf-report/aggregate.js \
+            --addon llamacpp-llm \
+            --workflow "Integration Tests (LLM)" \
+            --runs 6 \
+            --output reports/llamacpp-llm-performance.md \
+            --output-json reports/llamacpp-llm-performance.json \
+            --output-html reports/llamacpp-llm-performance.html || true
+
+          echo "=== TTS ==="
+          node scripts/perf-report/aggregate.js \
+            --addon onnx-tts \
+            --workflow "Integration Tests (TTS)" \
+            --runs 6 \
+            --output reports/onnx-tts-performance.md \
+            --output-json reports/onnx-tts-performance.json \
+            --output-html reports/onnx-tts-performance.html || true
+
+      - name: Write GitHub Step Summary
+        if: always()
+        run: |
+          echo "# Performance Reports" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo "Generated: $(date -u '+%Y-%m-%d %H:%M UTC')" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          for f in reports/*.md; do
+            if [ -f "$f" ]; then
+              cat "$f" >> $GITHUB_STEP_SUMMARY
+              echo "" >> $GITHUB_STEP_SUMMARY
+              echo "---" >> $GITHUB_STEP_SUMMARY
+              echo "" >> $GITHUB_STEP_SUMMARY
+            fi
+          done
+
+      - name: Upload reports
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: performance-reports-${{ github.run_number }}
+          path: reports/
+          retention-days: 90
+          if-no-files-found: ignore

packages/ocr-onnx/scripts/generate-ocr-presigned-urls.sh

@@ -79,6 +79,14 @@ for name in "${RECOGNIZERS[@]}"; do
     url_escaped="${url//\"/\\\"}"
     printf '  "%s": "%s",\n' "$key" "$url_escaped" >> "$JSON_FILE"
 done
+# DocTR models — direct GitHub release URLs (no S3 presigning needed, these are public).
+# On mobile, _resolveDoctrUrl() in utils.js reads these from ocr-model-urls.json first,
+# falling back to hardcoded GitHub URLs if missing. Kept here as override point in case
+# GitHub releases are rate-limited on Device Farm.
+printf '  "doctr_db_resnet50_url": "%s",\n' "https://github.com/felixdittrich92/OnnxTR/releases/download/v0.0.1/db_resnet50-69ba0015.onnx" >> "$JSON_FILE"
+printf '  "doctr_parseq_url": "%s",\n' "https://github.com/felixdittrich92/OnnxTR/releases/download/v0.0.1/parseq-00b40714.onnx" >> "$JSON_FILE"
+printf '  "doctr_db_mobilenet_v3_large_url": "%s",\n' "https://github.com/felixdittrich92/OnnxTR/releases/download/v0.2.0/db_mobilenet_v3_large-4987e7bd.onnx" >> "$JSON_FILE"
+printf '  "doctr_crnn_mobilenet_v3_small_url": "%s",\n' "https://github.com/felixdittrich92/OnnxTR/releases/download/v0.0.1/crnn_mobilenet_v3_small-bded4d49.onnx" >> "$JSON_FILE"
 printf '  "generatedAt": "%s"\n}\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" >> "$JSON_FILE"
 
 echo ""

packages/ocr-onnx/test/integration/doctr-basic.test.js

@@ -56,7 +56,7 @@ test('DocTR basic - BMP image', { timeout: TEST_TIMEOUT }, async function (t) {
   // tilted/vertical vary by platform and DocTR lacks per-crop rotation handling (unlike EasyOCR).
   t.ok(outputTexts.some(w => w.toLowerCase().includes('normal')), 'BMP should detect "normal"')
   t.comment('BMP detected texts: ' + JSON.stringify(outputTexts))
-  t.comment(formatOCRPerformanceMetrics('[DocTR BMP]', stats, outputTexts))
+  t.comment(formatOCRPerformanceMetrics('[DocTR BMP]', stats, outputTexts, { skipReport: true }))
 })
 
 test('DocTR basic - JPEG image', { timeout: TEST_TIMEOUT }, async function (t) {
@@ -74,7 +74,7 @@ test('DocTR basic - JPEG image', { timeout: TEST_TIMEOUT }, async function (t) {
   t.ok(results.length > 0, `JPEG: should detect text regions, got ${results.length}`)
   t.ok(outputTexts.some(w => w.toLowerCase().includes('normal')), 'JPEG should detect "normal"')
   t.comment('JPEG detected texts: ' + JSON.stringify(outputTexts))
-  t.comment(formatOCRPerformanceMetrics('[DocTR JPEG]', stats, outputTexts))
+  t.comment(formatOCRPerformanceMetrics('[DocTR JPEG]', stats, outputTexts, { skipReport: true }))
 })
 
 test('DocTR basic - PNG image', { timeout: TEST_TIMEOUT }, async function (t) {
@@ -92,7 +92,7 @@ test('DocTR basic - PNG image', { timeout: TEST_TIMEOUT }, async function (t) {
   t.ok(results.length > 0, `PNG: should detect text regions, got ${results.length}`)
   t.ok(outputTexts.some(w => w.toLowerCase().includes('normal')), 'PNG should detect "normal"')
   t.comment('PNG detected texts: ' + JSON.stringify(outputTexts))
-  t.comment(formatOCRPerformanceMetrics('[DocTR PNG]', stats, outputTexts))
+  t.comment(formatOCRPerformanceMetrics('[DocTR PNG]', stats, outputTexts, { skipReport: true }))
 })
 
 test('DocTR basic - English image', { timeout: TEST_TIMEOUT }, async function (t) {
@@ -124,5 +124,5 @@ test('DocTR basic - English image', { timeout: TEST_TIMEOUT }, async function (t
   t.ok(coordsInBounds, 'All bbox coordinates within image bounds (905x480)')
 
   t.comment('English detected texts: ' + JSON.stringify(outputTexts))
-  t.comment(formatOCRPerformanceMetrics('[DocTR English]', stats, outputTexts))
+  t.comment(formatOCRPerformanceMetrics('[DocTR English]', stats, outputTexts, { skipReport: true }))
 })

packages/ocr-onnx/test/integration/doctr-clinical-chemistry.test.js

@@ -0,0 +1,72 @@
+'use strict'
+
+const test = require('brittle')
+const { getImagePath, formatOCRPerformanceMetrics, runDoctrOCR, ensureDoctrModels } = require('./utils')
+
+const DOCTR_TEST_TIMEOUT = 180 * 1000
+
+let DB_MOBILENET
+let CRNN_MOBILENET
+let modelsAvailable = false
+
+test('DocTR clinical chemistry - download models', { timeout: DOCTR_TEST_TIMEOUT }, async function (t) {
+  const models = await ensureDoctrModels(['db_mobilenet_v3_large.onnx', 'crnn_mobilenet_v3_small.onnx'])
+  if (!models) {
+    t.comment('DocTR models unavailable (download failed) — remaining tests will be skipped')
+    return
+  }
+  DB_MOBILENET = models.db_mobilenet_v3_large
+  CRNN_MOBILENET = models.crnn_mobilenet_v3_small
+  modelsAvailable = true
+  t.ok(DB_MOBILENET, 'db_mobilenet model available')
+  t.ok(CRNN_MOBILENET, 'crnn_mobilenet model available')
+})
+
+const PERF_RUNS = 3
+
+const EXPECTED_WORDS = [
+  'clinical', 'chemistry', 'alkaline', 'phosphatase',
+  'hemoglobin', 'creatinine', 'cholesterol', 'triglycerides',
+  'bilirubin', 'albumin', 'protein', 'lipid'
+]
+
+function runClinicalChemistryTest (ep, run) {
+  const useGPU = ep === 'gpu'
+  const tag = ep.toUpperCase()
+
+  test(`DocTR clinical chemistry [${tag}] run ${run} - db_mobilenet + crnn_mobilenet`, { timeout: DOCTR_TEST_TIMEOUT }, async function (t) {
+    if (!modelsAvailable) { t.comment('Skipped — models unavailable'); return }
+    const imagePath = getImagePath('/test/images/clinical_chemistry.png')
+
+    t.comment(`Testing DocTR on clinical chemistry lab result image [${tag}] (run ${run}/${PERF_RUNS})`)
+    t.comment('Detector: db_mobilenet_v3_large, Recognizer: crnn_mobilenet_v3_small (CTC)')
+    t.comment('straightenPages: true, useGPU: ' + useGPU)
+
+    const { results, stats } = await runDoctrOCR(t, {
+      pathDetector: DB_MOBILENET,
+      pathRecognizer: CRNN_MOBILENET,
+      decodingMethod: 'ctc',
+      straightenPages: true,
+      useGPU
+    }, imagePath)
+
+    const texts = results.map(r => r.text)
+    t.comment('Detected texts: ' + JSON.stringify(texts))
+    t.comment(formatOCRPerformanceMetrics(`[DocTR clinical_chemistry] [${tag}]`, stats, texts, { imagePath }))
+
+    t.ok(results.length > 0, `should detect text regions, got ${results.length}`)
+
+    const lowerTexts = texts.map(w => w.toLowerCase())
+    for (const word of EXPECTED_WORDS) {
+      t.ok(
+        lowerTexts.some(w => w.includes(word)),
+        `should detect "${word}" in clinical chemistry report`
+      )
+    }
+
+    t.pass(`DocTR clinical chemistry [${tag}] run ${run} completed successfully`)
+  })
+}
+
+for (let i = 1; i <= PERF_RUNS; i++) runClinicalChemistryTest('cpu', i)
+for (let i = 1; i <= PERF_RUNS; i++) runClinicalChemistryTest('gpu', i)

packages/ocr-onnx/test/integration/doctr-ct-scan.test.js

@@ -0,0 +1,74 @@
+'use strict'
+
+const test = require('brittle')
+const { getImagePath, formatOCRPerformanceMetrics, runDoctrOCR, ensureDoctrModels } = require('./utils')
+
+const DOCTR_TEST_TIMEOUT = 180 * 1000
+
+let DB_MOBILENET
+let CRNN_MOBILENET
+let modelsAvailable = false
+
+test('DocTR CT scan - download models', { timeout: DOCTR_TEST_TIMEOUT }, async function (t) {
+  const models = await ensureDoctrModels(['db_mobilenet_v3_large.onnx', 'crnn_mobilenet_v3_small.onnx'])
+  if (!models) {
+    t.comment('DocTR models unavailable (download failed) — remaining tests will be skipped')
+    return
+  }
+  DB_MOBILENET = models.db_mobilenet_v3_large
+  CRNN_MOBILENET = models.crnn_mobilenet_v3_small
+  modelsAvailable = true
+  t.ok(DB_MOBILENET, 'db_mobilenet model available')
+  t.ok(CRNN_MOBILENET, 'crnn_mobilenet model available')
+})
+
+const EXPECTED_WORDS = [
+  'diagnostic', 'imaging', 'computed', 'tomography',
+  'chest', 'abdomen', 'lung', 'liver', 'pancreas',
+  'gallbladder', 'spleen', 'radiologist', 'allied',
+  'medical', 'center', 'patient', 'heart', 'trachea',
+  'vascular', 'normal'
+]
+
+const PERF_RUNS = 3
+
+function runCtScanTest (ep, run) {
+  const useGPU = ep === 'gpu'
+  const tag = ep.toUpperCase()
+
+  test(`DocTR CT scan [${tag}] run ${run} - db_mobilenet + crnn_mobilenet`, { timeout: DOCTR_TEST_TIMEOUT }, async function (t) {
+    if (!modelsAvailable) { t.comment('Skipped — models unavailable'); return }
+    const imagePath = getImagePath('/test/images/ct_scan_report.png')
+
+    t.comment(`Testing DocTR on CT scan diagnostic report image [${tag}] (run ${run}/${PERF_RUNS})`)
+    t.comment('Detector: db_mobilenet_v3_large, Recognizer: crnn_mobilenet_v3_small (CTC)')
+    t.comment('straightenPages: true, useGPU: ' + useGPU)
+
+    const { results, stats } = await runDoctrOCR(t, {
+      pathDetector: DB_MOBILENET,
+      pathRecognizer: CRNN_MOBILENET,
+      decodingMethod: 'ctc',
+      straightenPages: true,
+      useGPU
+    }, imagePath)
+
+    const texts = results.map(r => r.text)
+    t.comment('Detected texts: ' + JSON.stringify(texts))
+    t.comment(formatOCRPerformanceMetrics(`[DocTR ct_scan_report] [${tag}]`, stats, texts, { imagePath }))
+
+    t.ok(results.length > 0, `should detect text regions, got ${results.length}`)
+
+    const lowerTexts = texts.map(w => w.toLowerCase())
+    for (const word of EXPECTED_WORDS) {
+      t.ok(
+        lowerTexts.some(w => w.includes(word)),
+        `should detect "${word}" in CT scan report`
+      )
+    }
+
+    t.pass(`DocTR CT scan report [${tag}] run ${run} completed successfully`)
+  })
+}
+
+for (let i = 1; i <= PERF_RUNS; i++) runCtScanTest('cpu', i)
+for (let i = 1; i <= PERF_RUNS; i++) runCtScanTest('gpu', i)

packages/ocr-onnx/test/integration/doctr-french.test.js

@@ -42,6 +42,6 @@ test('DocTR french test - accented characters', { timeout: TEST_TIMEOUT }, async
   )
   t.comment('Contains accented characters: ' + hasAccent)
 
-  t.comment(formatOCRPerformanceMetrics('[DocTR French]', stats, outputTexts))
+  t.comment(formatOCRPerformanceMetrics('[DocTR French]', stats, outputTexts, { skipReport: true }))
   t.pass('DocTR French test completed successfully')
 })