Skip to content
Merged
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
245 changes: 193 additions & 52 deletions crates/tokscale-cli/src/commands/report.rs
Original file line number Diff line number Diff line change
Expand Up @@ -483,14 +483,12 @@ fn significant_tokens(title: &str) -> Vec<String> {
.chars()
// Map punctuation/ellipsis to spaces; keep alphanumerics. This also
// strips a trailing "…" or "..." that the on-device model often emits.
.map(|c| {
if c.is_alphanumeric() {
c.to_ascii_lowercase()
} else {
' '
}
})
.map(|c| if c.is_alphanumeric() { c } else { ' ' })
.collect::<String>()
// Full Unicode case folding (NOT to_ascii_lowercase): the tokens
// themselves keep non-ASCII characters, so case-folding must cover them
// too — otherwise "Café"/"café" or "İ"/"i̇" produce distinct tokens.
.to_lowercase()
.split_whitespace()
.map(|t| t.to_string())
.filter(|t| !CLUSTER_STOPWORDS.contains(&t.as_str()))
Expand All @@ -500,21 +498,42 @@ fn significant_tokens(title: &str) -> Vec<String> {
tokens
}

/// Two token sets are considered the same task when they overlap strongly:
/// Jaccard similarity ≥ 0.6, OR they share at least two significant tokens.
/// The two-shared-tokens rule lets a long title ("Enhance API security with JWT
/// auth middleware") merge with a short one ("Enhance API Security") even though
/// the length gap drags Jaccard below 0.6.
/// Normalize a title for exact-equality grouping of token-empty titles:
/// full Unicode lowercase with whitespace collapsed. Used only to keep
/// identical all-stopword titles together while keeping distinct ones apart.
fn normalized_title(title: &str) -> String {
title
.to_lowercase()
.split_whitespace()
.collect::<Vec<_>>()
.join(" ")
}

/// Similarity threshold for treating two titles as the same task. Applied to
/// the overlap coefficient (shared / size-of-smaller-set), which — unlike a raw
/// shared-token count — scales with how much of the smaller title is covered.
const CLUSTER_SIMILARITY_THRESHOLD: f64 = 0.6;

/// Two token sets are considered the same task when they overlap strongly,
/// measured by the OVERLAP COEFFICIENT: `shared / min(|a|, |b|)`.
///
/// This deliberately replaces the old "share ≥ 2 tokens" rule, which fired on
/// any two long-but-unrelated titles that happened to share two common words
/// (e.g. "add" + "service") and — combined with union-growing cluster
/// signatures — let a single cluster transitively swallow everything.
///
/// The overlap coefficient stays high (1.0) when a short title is fully
/// contained in a longer one ("Enhance API Security" ⊂ "Enhance API security
/// with JWT auth middleware"), so genuine near-duplicates still merge, while two
/// large sets that share only a couple of incidental tokens score low and stay
/// apart.
fn tokens_overlap(a: &[String], b: &[String]) -> bool {
if a.is_empty() || b.is_empty() {
return false;
}
let shared = a.iter().filter(|t| b.contains(t)).count();
if shared >= 2 {
return true;
}
let union = a.len() + b.len() - shared;
union > 0 && (shared as f64 / union as f64) >= 0.6
let smaller = a.len().min(b.len());
smaller > 0 && (shared as f64 / smaller as f64) >= CLUSTER_SIMILARITY_THRESHOLD

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Avoid treating singleton token matches as full overlap

With the new overlap coefficient, any title that reduces to a single significant token gets a perfect score against every longer title containing that token, so a generic summary like Fix API/API will cluster with unrelated Add API auth, Update API billing, etc. This regresses the over-merge behavior for apple-fm/non-LLM grouping because the previous Jaccard/shared-token rule would not merge on just one shared token; singleton titles need a stricter rule such as requiring at least two significant shared tokens.

Useful? React with 👍 / 👎.

Copy link
Copy Markdown
Owner Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Fixed in 715c070. tokens_overlap now requires at least two shared significant tokens when the smaller token set has only one token (if smaller <= 1 { return shared >= 2; }), before applying the overlap coefficient. This prevents a singleton title ("API"/"Fix API") from merging with every unrelated longer title containing that token, restoring the stricter behavior for apple-fm/non-LLM grouping while keeping genuine multi-token near-duplicates clustering. Regression tests added (tokens_overlap_singleton_does_not_overcluster, cluster_titles_does_not_overcluster_singletons). cargo test + clippy green.

Comment thread
cubic-dev-ai[bot] marked this conversation as resolved.
Resolved
}

/// Deterministically cluster summarized entries by title similarity and return
Expand All @@ -524,77 +543,96 @@ fn tokens_overlap(a: &[String], b: &[String]) -> bool {
/// human-readable title rather than a synthetic key.
fn cluster_titles(entries: &[&WikiEntry]) -> Vec<(String, String)> {
struct Cluster {
tokens: Vec<String>,
// Token sets of EVERY member, kept separately rather than unioned into a
// single signature. A candidate joins if it overlaps ANY member, which
// preserves transitive grouping of genuine near-duplicates WITHOUT the
// union ballooning into a catch-all that absorbs unrelated titles.
member_tokens: Vec<Vec<String>>,
// Exact normalized title shared by a token-empty cluster (all-stopword
// titles). `None` for tokened clusters.
empty_key: Option<String>,
members: Vec<usize>,
}

// Precompute significant tokens once per entry.
let prepared: Vec<(usize, Vec<String>)> = entries
// Precompute significant tokens + normalized title once per entry.
let prepared: Vec<(usize, Vec<String>, String)> = entries
.iter()
.enumerate()
.map(|(i, e)| (i, significant_tokens(e.title.as_deref().unwrap_or(""))))
.map(|(i, e)| {
let raw = e.title.as_deref().unwrap_or("");
(i, significant_tokens(raw), normalized_title(raw))
})
.collect();

let mut clusters: Vec<Cluster> = Vec::new();
for (idx, tokens) in &prepared {
for (idx, tokens, norm) in &prepared {
// Find the first existing cluster this entry overlaps strongly with.
// An entry with no significant tokens (all stopwords) only matches a
// cluster that is itself token-empty, so generic titles still group.
let mut placed = false;
for cluster in clusters.iter_mut() {
let matches = if tokens.is_empty() {
cluster.tokens.is_empty()
// Token-empty titles (all stopwords) carry no signal to cluster
// on, so they only merge with an identical normalized title.
// This stops every generic/stopword-only title from collapsing
// into one arbitrary blob.
cluster.empty_key.as_deref() == Some(norm.as_str())
} else {
tokens_overlap(tokens, &cluster.tokens)
// Tokened entries never join an empty cluster; they match if they
// overlap ANY existing member of the cluster.
cluster.empty_key.is_none()
&& cluster
.member_tokens
.iter()
.any(|m| tokens_overlap(tokens, m))
};
if matches {
cluster.members.push(*idx);
// Grow the cluster signature with this entry's tokens so later
// entries can match on the union of what's been seen.
for t in tokens {
if !cluster.tokens.contains(t) {
cluster.tokens.push(t.clone());
}
}
cluster.tokens.sort();
cluster.member_tokens.push(tokens.clone());
placed = true;
break;
}
}
if !placed {
clusters.push(Cluster {
tokens: tokens.clone(),
member_tokens: vec![tokens.clone()],
empty_key: if tokens.is_empty() {
Some(norm.clone())
} else {
None
},
members: vec![*idx],
});
}
}

// Consolidation pass: the single greedy pass above is order-dependent — an
// entry compared before a cluster grew its signature can land in its own
// cluster even though it overlaps the grown signature. Repeatedly merge any
// two clusters whose signatures overlap until a fixpoint, making the final
// grouping independent of input order. n is small, so the O(n²)-per-round
// loop is cheap.
// entry compared before its eventual neighbor was seen can land in its own
// cluster even though it overlaps a member of another cluster. Repeatedly
// merge any two clusters that have a pair of overlapping members (or, for
// token-empty clusters, an identical normalized title) until a fixpoint,
// making the final grouping independent of input order. n is small, so the
// O(n²)-per-round loop is cheap. Crucially, merging only happens on a real
// member-to-member overlap, so it cannot chain unrelated clusters together.
loop {
let mut merged_any = false;
'outer: for i in 0..clusters.len() {
for j in (i + 1)..clusters.len() {
let overlap = if clusters[i].tokens.is_empty() || clusters[j].tokens.is_empty() {
// Token-empty clusters (all-stopword titles) only merge with
// each other, never with a tokened cluster.
clusters[i].tokens.is_empty() && clusters[j].tokens.is_empty()
} else {
tokens_overlap(&clusters[i].tokens, &clusters[j].tokens)
let overlap = match (&clusters[i].empty_key, &clusters[j].empty_key) {
// Token-empty clusters merge only with an identical title.
(Some(ki), Some(kj)) => ki == kj,
// A token-empty cluster never merges with a tokened one.
(Some(_), None) | (None, Some(_)) => false,
// Tokened clusters merge if any member pair overlaps.
(None, None) => clusters[i].member_tokens.iter().any(|mi| {
clusters[j]
.member_tokens
.iter()
.any(|mj| tokens_overlap(mi, mj))
}),
};
if overlap {
let other = clusters.remove(j);
clusters[i].members.extend(other.members);
for t in other.tokens {
if !clusters[i].tokens.contains(&t) {
clusters[i].tokens.push(t);
}
}
clusters[i].tokens.sort();
clusters[i].member_tokens.extend(other.member_tokens);
merged_any = true;
break 'outer;
}
Expand Down Expand Up @@ -1388,6 +1426,109 @@ mod tests {
assert_eq!(label, "Add JWT auth middleware");
}

#[test]
fn tokens_overlap_uses_ratio_not_absolute_count() {
// Two long, unrelated titles that incidentally share TWO tokens
// ("add" is a stopword, so the shared pair here is "service" + "api").
// Under the old `shared >= 2` rule these merged; the overlap coefficient
// (2 / 5 = 0.4 < 0.6) correctly keeps them apart.
let a = significant_tokens("Add pricing service api cache layer");
let b = significant_tokens("Add billing service api webhook handler");
let shared: Vec<_> = a.iter().filter(|t| b.contains(t)).collect();
assert_eq!(
shared.len(),
2,
"fixture must share exactly two tokens to exercise the old rule"
);
assert!(
!tokens_overlap(&a, &b),
"two shared tokens out of five must NOT merge under the ratio rule"
);

// A short title fully contained in a long one still merges (coeff 1.0).
let short = significant_tokens("pricing service");
assert!(tokens_overlap(&short, &a));
}

#[test]
fn cluster_titles_does_not_transitively_absorb_unrelated() {
// Chain of titles where each adjacent pair shares two incidental tokens
// but the ends are unrelated. The old union-growing signature plus the
// `shared >= 2` rule made all of these collapse into one blob. With the
// ratio rule and member-based matching they stay apart.
let entries = [
titled_entry("a", "Add pricing service api cache"),
titled_entry("b", "Add billing service api webhook"),
titled_entry("c", "Add billing report export csv"),
];
let refs: Vec<&WikiEntry> = entries.iter().collect();
let assignments = cluster_titles(&refs);
let distinct: std::collections::HashSet<_> =
assignments.iter().map(|(_, l)| l.clone()).collect();
assert_eq!(
distinct.len(),
3,
"incidental two-token overlaps must not chain unrelated titles into one cluster"
);
}

#[test]
fn cluster_titles_separates_distinct_stopword_only_titles() {
// Titles made entirely of stopwords/generic verbs reduce to no
// significant tokens. The old code lumped ALL of them into one arbitrary
// group; now each distinct normalized title is its own singleton while
// identical ones still collapse.
let entries = [
titled_entry("a", "Fix and update"),
titled_entry("b", "Refactor and improve"),
titled_entry("c", "Fix and update"),
];
// Sanity: these really are token-empty so we exercise the empty path.
assert!(significant_tokens("Fix and update").is_empty());
assert!(significant_tokens("Refactor and improve").is_empty());

let refs: Vec<&WikiEntry> = entries.iter().collect();
let assignments = cluster_titles(&refs);
let label_of = |sid: &str| {
assignments
.iter()
.find(|(s, _)| s == sid)
.map(|(_, l)| l.clone())
.unwrap()
};
// Identical stopword-only titles merge; distinct ones do not.
assert_eq!(label_of("a"), label_of("c"));
assert_ne!(label_of("a"), label_of("b"));
let distinct: std::collections::HashSet<_> =
assignments.iter().map(|(_, l)| l.clone()).collect();
assert_eq!(distinct.len(), 2);
}

#[test]
fn significant_tokens_folds_non_ascii_case() {
// Full Unicode lowercasing: "Café" and "café" must produce the same
// token. to_ascii_lowercase left the accented capital untouched, so the
// two titles would have clustered apart.
assert_eq!(
significant_tokens("Café Münchën Stratégie"),
significant_tokens("café münchën stratégie")
);
assert!(significant_tokens("Café").contains(&"café".to_string()));
}

#[test]
fn cluster_titles_merges_non_ascii_case_variants() {
let entries = [
titled_entry("a", "Refactor Café Stratégie module"),
titled_entry("b", "Refactor café stratégie module"),
];
let refs: Vec<&WikiEntry> = entries.iter().collect();
let assignments = cluster_titles(&refs);
let distinct: std::collections::HashSet<_> =
assignments.iter().map(|(_, l)| l.clone()).collect();
assert_eq!(distinct.len(), 1, "case-only differences must merge");
}

#[test]
fn cluster_titles_groups_exact_duplicates() {
// The degenerate case from the report: many identical titles must
Expand Down
Loading