fix: align scanner semantics with vscode-oniguruma (tie-breaking, CAPTURE_GROUP, UTF-16 offsets)

packages/react-native-shiki-engine/cpp/NativeShikiEngineModule.cpp

@@ -1,13 +1,80 @@
 #include "NativeShikiEngineModule.h"
 
 #include <unordered_map>
+#include <vector>
 
 namespace facebook::react {
 
 // Store scanner contexts with their IDs
 static std::unordered_map<double, OnigContext*> g_scanners;
 static double g_nextScannerId = 1;
 
+// ---- UTF-8 <-> UTF-16 offset conversion ----
+//
+// vscode-textmate passes/expects offsets in UTF-16 code units (JS string
+// indexing), but oniguruma scans the UTF-8 encoded buffer and reports byte
+// offsets. For pure-ASCII text the two coincide; any multi-byte character
+// (CJK, emoji, ...) shifts every following offset and corrupts token scopes.
+// Build a byte->UTF-16 offset table per call and convert in both directions,
+// mirroring what the official vscode-oniguruma binding does.
+
+// Returns table of size (bytes + 1): table[byteOffset] = utf16Offset.
+// Continuation bytes map to the UTF-16 offset of the code point they belong to.
+static std::vector<int> buildByteToUtf16Table(const std::string& utf8) {
+  std::vector<int> table(utf8.size() + 1);
+  int u16 = 0;
+  size_t i = 0;
+  const size_t n = utf8.size();
+  while (i < n) {
+    const unsigned char c = static_cast<unsigned char>(utf8[i]);
+    size_t len = 1;
+    int units = 1;
+    if (c < 0x80) {
+      len = 1;
+    } else if ((c & 0xE0) == 0xC0) {
+      len = 2;
+    } else if ((c & 0xF0) == 0xE0) {
+      len = 3;
+    } else if ((c & 0xF8) == 0xF0) {
+      len = 4;
+      units = 2;  // surrogate pair in UTF-16
+    }
+    for (size_t k = 0; k < len && i + k < n; k++) {
+      table[i + k] = u16;
+    }
+    u16 += units;
+    i += len;
+  }
+  table[n] = u16;
+  return table;
+}
+
+// Converts a UTF-16 code unit offset to the corresponding UTF-8 byte offset.
+static int utf16ToByteOffset(const std::vector<int>& table, int utf16Offset) {
+  if (utf16Offset <= 0) {
+    return 0;
+  }
+  // table is monotonically non-decreasing; find first byte whose utf16 >= target
+  const int n = static_cast<int>(table.size()) - 1;
+  for (int b = 0; b <= n; b++) {
+    if (table[b] >= utf16Offset) {
+      return b;
+    }
+  }
+  return n;
+}
+
+static inline int byteToUtf16Offset(const std::vector<int>& table, int byteOffset) {
+  if (byteOffset < 0) {
+    return byteOffset;
+  }
+  const int n = static_cast<int>(table.size()) - 1;
+  if (byteOffset > n) {
+    return table[n];
+  }
+  return table[byteOffset];
+}
+
 NativeShikiEngineModule::NativeShikiEngineModule(std::shared_ptr<CallInvoker> jsInvoker)
   : NativeShikiEngineCxxSpec<NativeShikiEngineModule>(std::move(jsInvoker)) {}
 
@@ -61,7 +128,13 @@ NativeShikiEngineModule::findNextMatchSync(jsi::Runtime& rt, double scannerId, j
   }
 
   std::string textStr = text.utf8(rt);
-  OnigResult* result = find_next_match(it->second, textStr.c_str(), static_cast<int>(startPosition));
+
+  // JS side (vscode-textmate) speaks UTF-16 offsets; oniguruma speaks UTF-8
+  // byte offsets. Convert startPosition in, and all capture indices out.
+  const std::vector<int> b2u = buildByteToUtf16Table(textStr);
+  const int startByte = utf16ToByteOffset(b2u, static_cast<int>(startPosition));
+
+  OnigResult* result = find_next_match(it->second, textStr.c_str(), startByte);
 
   if (!result) {
     return std::nullopt;
@@ -77,6 +150,14 @@ NativeShikiEngineModule::findNextMatchSync(jsi::Runtime& rt, double scannerId, j
     int start = result->capture_indices[i * 2];
     int end = result->capture_indices[i * 2 + 1];
 
+    // Unmatched optional groups report negative offsets; pass them through.
+    if (start >= 0) {
+      start = byteToUtf16Offset(b2u, start);
+    }
+    if (end >= 0) {
+      end = byteToUtf16Offset(b2u, end);
+    }
+
     capture.setProperty(rt, "start", start);
     capture.setProperty(rt, "end", end);
     capture.setProperty(rt, "length", end - start);

packages/react-native-shiki-engine/cpp/onig_regex.cpp

@@ -124,7 +124,11 @@ OnigContext* create_scanner(const char** patterns, int pattern_count, size_t max
           &regex,
           (const OnigUChar*)patterns[i],
           (const OnigUChar*)(patterns[i] + strlen(patterns[i])),
-          ONIG_OPTION_DEFAULT,
+          // ONIG_OPTION_CAPTURE_GROUP matches vscode-oniguruma: without it,
+          // oniguruma disables numbered captures in any pattern that also
+          // contains named groups, silently breaking TextMate `captures`
+          // scope assignment (tokens lose their colors).
+          ONIG_OPTION_CAPTURE_GROUP,
           ONIG_ENCODING_UTF8,
           ONIG_SYNTAX_DEFAULT,
           &einfo
@@ -151,7 +155,8 @@ OnigContext* create_scanner(const char** patterns, int pattern_count, size_t max
   }
 }
 
-/** Finds leftmost-longest match after start_pos across all patterns. */
+/** Finds the leftmost match after start_pos across all patterns;
+ *  position ties are won by the lowest pattern index (TextMate priority). */
 OnigResult* find_next_match(OnigContext* context, const char* text, int start_pos) {
   if (!context || !text || start_pos < 0) {
     return nullptr;
@@ -167,7 +172,6 @@ OnigResult* find_next_match(OnigContext* context, const char* text, int start_po
 
     int text_length = strlen(text);
     int best_match_pos = -1;
-    int best_match_len = -1;
 
     for (int i = 0; i < context->pattern_count; i++) {
       onig_region_clear(context->region);
@@ -183,10 +187,12 @@ OnigResult* find_next_match(OnigContext* context, const char* text, int start_po
       );
 
       if (match_pos >= 0) {
-        if (best_match_pos < 0 || match_pos < best_match_pos ||
-            (match_pos == best_match_pos && context->region->end[0] - context->region->beg[0] > best_match_len)) {
+        // vscode-oniguruma contract: pick the LEFTMOST match; ties (same
+        // position) are won by the LOWEST pattern index — TextMate rule
+        // order is rule priority. Never tie-break by match length: that
+        // lets later rules steal matches and assigns wrong scopes.
+        if (best_match_pos < 0 || match_pos < best_match_pos) {
           best_match_pos = match_pos;
-          best_match_len = context->region->end[0] - context->region->beg[0];
           result->pattern_index = i;
           result->match_start = context->region->beg[0];
           result->match_end = context->region->end[0];
@@ -200,6 +206,12 @@ OnigResult* find_next_match(OnigContext* context, const char* text, int start_po
             result->capture_indices[j * 2] = context->region->beg[j];
             result->capture_indices[j * 2 + 1] = context->region->end[j];
           }
+
+          // Nothing can match earlier than start_pos; later patterns could
+          // only tie and ties keep the current (earlier) pattern.
+          if (best_match_pos == start_pos) {
+            break;
+          }
         }
       }
     }