tokenization: use BPE instead of chars\4 approximation

eng/skill-validator/src/Services/SkillProfiler.cs

@@ -1,11 +1,13 @@
 using System.Text.RegularExpressions;
+using Microsoft.ML.Tokenizers;
 using SkillValidator.Models;
 
 namespace SkillValidator.Services;
 
 public sealed record SkillProfile(
     string Name,
-    int TokenCount,
+    int Chars4TokenCount,
+    int BpeTokenCount,
     string ComplexityTier, // "compact" | "detailed" | "standard" | "comprehensive"
     int SectionCount,
     int CodeBlockCount,
@@ -25,6 +27,11 @@ public static partial class SkillProfiler
     private const int TokenSweetHigh = 2500;
     private const int TokenWarnHigh = 5000;
     internal const int MaxDescriptionLength = 1024;
+
+    // BPE tokenizer (cl100k_base) used as a model-independent sizing heuristic.
+    // Not tied to the configured eval/judge model — TiktokenTokenizer only supports OpenAI
+    // vocabularies, but BPE counts are close enough across models for complexity classification.
+    private static readonly Lazy<TiktokenTokenizer> s_bpeTokenizer = new(() => TiktokenTokenizer.CreateForModel("gpt-4"));
     internal const int MaxAggregateDescriptionLength = 15_000;
     private const int MaxNameLength = 64;
     private const int MaxCompatibilityLength = 500;
@@ -33,7 +40,8 @@ public static partial class SkillProfiler
     public static SkillProfile AnalyzeSkill(SkillInfo skill)
     {
         var content = skill.SkillMdContent;
-        int tokenCount = (int)Math.Ceiling(content.Length / 4.0);
+        int chars4TokenCount = (int)Math.Ceiling(content.Length / 4.0);
+        int bpeTokenCount = s_bpeTokenizer.Value.CountTokens(content);
 
         bool hasFrontmatter = FrontmatterRegex().IsMatch(content);
 
@@ -48,7 +56,7 @@ public static SkillProfile AnalyzeSkill(SkillInfo skill)
         bool hasWhenToUse = WhenToUseRegex().IsMatch(body);
         bool hasWhenNotToUse = WhenNotToUseRegex().IsMatch(body);
 
-        string complexityTier = tokenCount switch
+        string complexityTier = bpeTokenCount switch
         {
             < 400 => "compact",
             <= 2500 => "detailed",
@@ -134,21 +142,21 @@ public static SkillProfile AnalyzeSkill(SkillInfo skill)
             }
         }
 
-        // --- Token size warnings ---
-        if (tokenCount > TokenWarnHigh)
+        // --- Token size warnings (based on BPE token count) ---
+        if (bpeTokenCount > TokenWarnHigh)
         {
             warnings.Add(
-                $"Skill is {tokenCount:N0} tokens — \"comprehensive\" skills hurt performance by 2.9pp on average. Consider splitting into 2–3 focused skills.");
+                $"Skill is {bpeTokenCount:N0} BPE tokens (chars/4 estimate: {chars4TokenCount:N0}) — \"comprehensive\" skills hurt performance by 2.9pp on average. Consider splitting into 2–3 focused skills.");
         }
-        else if (tokenCount > TokenSweetHigh)
+        else if (bpeTokenCount > TokenSweetHigh)
         {
             warnings.Add(
-                $"Skill is {tokenCount:N0} tokens — approaching \"comprehensive\" range where gains diminish.");
+                $"Skill is {bpeTokenCount:N0} BPE tokens (chars/4 estimate: {chars4TokenCount:N0}) — approaching \"comprehensive\" range where gains diminish.");
         }
-        else if (tokenCount < TokenSweetLow)
+        else if (bpeTokenCount < TokenSweetLow)
         {
             warnings.Add(
-                $"Skill is only {tokenCount} tokens — may be too sparse to provide actionable guidance.");
+                $"Skill is only {bpeTokenCount} BPE tokens (chars/4 estimate: {chars4TokenCount}) — may be too sparse to provide actionable guidance.");
         }
 
         if (sectionCount == 0)
@@ -182,7 +190,8 @@ public static SkillProfile AnalyzeSkill(SkillInfo skill)
 
         return new SkillProfile(
             Name: skill.Name,
-            TokenCount: tokenCount,
+            Chars4TokenCount: chars4TokenCount,
+            BpeTokenCount: bpeTokenCount,
             ComplexityTier: complexityTier,
             SectionCount: sectionCount,
             CodeBlockCount: codeBlockCount,
@@ -246,7 +255,7 @@ public static string FormatProfileLine(SkillProfile profile)
         };
 
         return
-            $"📊 {profile.Name}: {profile.TokenCount:N0} tokens ({profile.ComplexityTier} {tierIndicator}), " +
+            $"{profile.Name}: {profile.BpeTokenCount:N0} BPE tokens [chars/4: {profile.Chars4TokenCount:N0}] ({profile.ComplexityTier} {tierIndicator}), " +
             $"{profile.SectionCount} sections, {profile.CodeBlockCount} code blocks";
     }
 

eng/skill-validator/src/SkillValidator.csproj

@@ -31,6 +31,8 @@
 
   <ItemGroup>
     <PackageReference Include="Microsoft.Extensions.FileSystemGlobbing" Version="10.0.3" />
+    <PackageReference Include="Microsoft.ML.Tokenizers" Version="2.0.0" />
+    <PackageReference Include="Microsoft.ML.Tokenizers.Data.Cl100kBase" Version="2.0.0" />
     <PackageReference Include="System.CommandLine" Version="2.0.3" />
     <!-- external -->
     <PackageReference Include="GitHub.Copilot.SDK" Version="0.1.30" />

eng/skill-validator/tests/SkillProfileTests.cs

@@ -70,8 +70,9 @@ public void ClassifiesCompactSkills()
     [Fact]
     public void ClassifiesComprehensiveSkillsAndWarns()
     {
-        // >5000 tokens = >20000 chars
-        var content = "---\nname: foo\n---\n# Big\n" + new string('x', 25000);
+        // >5000 BPE tokens — use varied text since BPE compresses repeated chars efficiently
+        var content = "---\nname: foo\n---\n# Big\n" + string.Concat(
+            Enumerable.Range(0, 5000).Select(i => $"word{i} "));
         var profile = SkillProfiler.AnalyzeSkill(MakeSkill(content));
         Assert.Equal("comprehensive", profile.ComplexityTier);
         Assert.Contains(profile.Warnings, w => w.Contains("comprehensive"));