Skip to content

tokenization: use BPE instead of chars\4 approximation #309

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
JanKrivanek merged 6 commits into from
Mar 10, 2026
Merged

tokenization: use BPE instead of chars\4 approximation #309

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
d870730
use BPE tokenizer
DeagleGross Mar 7, 2026
4b9980a
address PR comments 1
DeagleGross Mar 10, 2026
c25025d
elaborate on model
DeagleGross Mar 10, 2026
a43448e
Merge branch 'main' into dmkorolev/bpe
DeagleGross Mar 10, 2026
fc95d4e
nit
DeagleGross Mar 10, 2026
1624bcf
Merge branch 'dmkorolev/bpe' of https://github.com/dotnet/skills into…
DeagleGross Mar 10, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 19 additions & 11 deletions eng/skill-validator/src/Services/SkillProfiler.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,11 +1,13 @@
using System.Text.RegularExpressions;
using Microsoft.ML.Tokenizers;
using SkillValidator.Models;

namespace SkillValidator.Services;

public sealed record SkillProfile(
string Name,
int TokenCount,
int BpeTokenCount,
Comment thread
DeagleGross marked this conversation as resolved.
string ComplexityTier, // "compact" | "detailed" | "standard" | "comprehensive"
int SectionCount,
int CodeBlockCount,
Expand All @@ -25,6 +27,10 @@ public static partial class SkillProfiler
private const int TokenSweetHigh = 2500;
private const int TokenWarnHigh = 5000;
internal const int MaxDescriptionLength = 1024;

// Lazy-initialized BPE tokenizer (cl100k_base, same BPE family as GPT-4/Claude)
private static readonly Lazy<TiktokenTokenizer> s_bpeTokenizer = new(
() => TiktokenTokenizer.CreateForModel("gpt-4"));
Comment thread
DeagleGross marked this conversation as resolved.
Outdated
internal const int MaxAggregateDescriptionLength = 15_000;
private const int MaxNameLength = 64;
private const int MaxCompatibilityLength = 500;
Expand All @@ -33,7 +39,8 @@ public static partial class SkillProfiler
public static SkillProfile AnalyzeSkill(SkillInfo skill)
{
var content = skill.SkillMdContent;
int tokenCount = (int)Math.Ceiling(content.Length / 4.0);
int chars4TokenCount = (int)Math.Ceiling(content.Length / 4.0);
int bpeTokenCount = s_bpeTokenizer.Value.CountTokens(content);

bool hasFrontmatter = FrontmatterRegex().IsMatch(content);

Expand All @@ -48,7 +55,7 @@ public static SkillProfile AnalyzeSkill(SkillInfo skill)
bool hasWhenToUse = WhenToUseRegex().IsMatch(body);
bool hasWhenNotToUse = WhenNotToUseRegex().IsMatch(body);

string complexityTier = tokenCount switch
string complexityTier = bpeTokenCount switch
{
< 400 => "compact",
<= 2500 => "detailed",
Expand Down Expand Up @@ -134,21 +141,21 @@ public static SkillProfile AnalyzeSkill(SkillInfo skill)
}
}

// --- Token size warnings ---
if (tokenCount > TokenWarnHigh)
// --- Token size warnings (based on BPE token count) ---
if (bpeTokenCount > TokenWarnHigh)
{
warnings.Add(
$"Skill is {tokenCount:N0} tokens — \"comprehensive\" skills hurt performance by 2.9pp on average. Consider splitting into 2–3 focused skills.");
$"Skill is {bpeTokenCount:N0} BPE tokens (chars/4 estimate: {chars4TokenCount:N0}) — \"comprehensive\" skills hurt performance by 2.9pp on average. Consider splitting into 2–3 focused skills.");
}
else if (tokenCount > TokenSweetHigh)
else if (bpeTokenCount > TokenSweetHigh)
{
warnings.Add(
$"Skill is {tokenCount:N0} tokens — approaching \"comprehensive\" range where gains diminish.");
$"Skill is {bpeTokenCount:N0} BPE tokens (chars/4 estimate: {chars4TokenCount:N0}) — approaching \"comprehensive\" range where gains diminish.");
}
else if (tokenCount < TokenSweetLow)
else if (bpeTokenCount < TokenSweetLow)
{
warnings.Add(
$"Skill is only {tokenCount} tokens — may be too sparse to provide actionable guidance.");
$"Skill is only {bpeTokenCount} BPE tokens (chars/4 estimate: {chars4TokenCount}) — may be too sparse to provide actionable guidance.");
Comment thread
DeagleGross marked this conversation as resolved.
Outdated
}

if (sectionCount == 0)
Expand Down Expand Up @@ -177,7 +184,8 @@ public static SkillProfile AnalyzeSkill(SkillInfo skill)

return new SkillProfile(
Name: skill.Name,
TokenCount: tokenCount,
TokenCount: chars4TokenCount,
BpeTokenCount: bpeTokenCount,
ComplexityTier: complexityTier,
SectionCount: sectionCount,
CodeBlockCount: codeBlockCount,
Expand Down Expand Up @@ -230,7 +238,7 @@ public static string FormatProfileLine(SkillProfile profile)
};

return
$"📊 {profile.Name}: {profile.TokenCount:N0} tokens ({profile.ComplexityTier} {tierIndicator}), " +
$"📊 {profile.Name}: {profile.BpeTokenCount:N0} BPE tokens [chars/4: {profile.TokenCount:N0}] ({profile.ComplexityTier} {tierIndicator}), " +
Comment thread
DeagleGross marked this conversation as resolved.
Outdated
$"{profile.SectionCount} sections, {profile.CodeBlockCount} code blocks";
Comment thread
DeagleGross marked this conversation as resolved.
}

Expand Down
4 changes: 3 additions & 1 deletion eng/skill-validator/src/SkillValidator.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
<PublishAot>true</PublishAot>

<!-- dotnet run args for local invocation -->
<RunArguments>--results-dir &quot;$([MSBuild]::NormalizePath('$(ArtifactsPath)', 'TestResults', '$(AssemblyName)'))&quot; --parallel-skills 3 --parallel-scenarios 3 --parallel-runs 3</RunArguments>
<RunArguments>--results-dir "$([MSBuild]::NormalizePath('$(ArtifactsPath)', 'TestResults', '$(AssemblyName)'))" --parallel-skills 3 --parallel-scenarios 3 --parallel-runs 3</RunArguments>
Comment thread
DeagleGross marked this conversation as resolved.
Outdated
</PropertyGroup>

<ItemGroup>
Expand All @@ -31,6 +31,8 @@

<ItemGroup>
<PackageReference Include="Microsoft.Extensions.FileSystemGlobbing" Version="10.0.3" />
<PackageReference Include="Microsoft.ML.Tokenizers" Version="2.0.0" />
<PackageReference Include="Microsoft.ML.Tokenizers.Data.Cl100kBase" Version="2.0.0" />
<PackageReference Include="System.CommandLine" Version="2.0.3" />
<!-- external -->
<PackageReference Include="GitHub.Copilot.SDK" Version="0.1.30" />
Expand Down
5 changes: 3 additions & 2 deletions eng/skill-validator/tests/SkillProfileTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,8 +70,9 @@ public void ClassifiesCompactSkills()
[Fact]
public void ClassifiesComprehensiveSkillsAndWarns()
{
// >5000 tokens = >20000 chars
var content = "---\nname: foo\n---\n# Big\n" + new string('x', 25000);
// >5000 BPE tokens — use varied text since BPE compresses repeated chars efficiently
var content = "---\nname: foo\n---\n# Big\n" + string.Concat(
Enumerable.Range(0, 5000).Select(i => $"word{i} "));
var profile = SkillProfiler.AnalyzeSkill(MakeSkill(content));
Assert.Equal("comprehensive", profile.ComplexityTier);
Assert.Contains(profile.Warnings, w => w.Contains("comprehensive"));
Expand Down
Loading