diff --git a/Cargo.lock b/Cargo.lock index fd207810565b..f993a26d885d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1837,7 +1837,7 @@ dependencies = [ "rayon", "safetensors 0.7.0", "thiserror 2.0.18", - "tokenizers", + "tokenizers 0.22.2", "yoke 0.8.2", "zip 7.2.0", ] @@ -2799,6 +2799,12 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "daachorse" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f55d7153ba3b507595872a3874803f07a8a81d1e888abed8e5db7da0597d6e2" + [[package]] name = "darling" version = "0.20.11" @@ -4837,7 +4843,7 @@ dependencies = [ "test-case", "thiserror 2.0.18", "tiktoken-rs", - "tokenizers", + "tokenizers 0.23.1", "tokio", "tokio-cron-scheduler", "tokio-stream", @@ -6685,7 +6691,7 @@ dependencies = [ "serde", "serde_json", "thiserror 2.0.18", - "tokenizers", + "tokenizers 0.22.2", ] [[package]] @@ -6698,7 +6704,7 @@ dependencies = [ "serde", "serde_json", "thiserror 2.0.18", - "tokenizers", + "tokenizers 0.22.2", ] [[package]] @@ -11547,6 +11553,39 @@ dependencies = [ "unicode_categories", ] +[[package]] +name = "tokenizers" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44e5bea67576e04b6ff8564c5d9e09c2ef0cf476502245f2f120e497769d3112" +dependencies = [ + "ahash", + "compact_str 0.9.0", + "daachorse", + "dary_heap", + "derive_builder", + "esaxx-rs", + "getrandom 0.3.4", + "itertools 0.14.0", + "log", + "macro_rules_attribute", + "monostate", + "onig", + "paste", + "rand 0.9.4", + "rayon", + "rayon-cond", + "regex", + "regex-syntax", + "serde", + "serde_json", + "spm_precompiled", + "thiserror 2.0.18", + "unicode-normalization-alignments", + "unicode-segmentation", + "unicode_categories", +] + [[package]] name = "tokio" version = "1.52.3" diff --git a/crates/goose/Cargo.toml b/crates/goose/Cargo.toml index 4cba9d8c3711..353ee588ed97 100644 --- a/crates/goose/Cargo.toml +++ b/crates/goose/Cargo.toml @@ -164,7 +164,7 @@ candle-core = { workspace = true, optional = true } candle-nn = { workspace = true, optional = true } candle-transformers = { version = "0.10", default-features = false, optional = true } byteorder = { version = "1.5", default-features = false, features = ["std"], optional = true } -tokenizers = { version = "0.22", default-features = false, features = ["onig"], optional = true } +tokenizers = { version = "0.23", default-features = false, features = ["onig"], optional = true } symphonia = { version = "0.5", default-features = false, features = ["aac", "adpcm", "alac", "isomp4", "mkv", "mp3", "pcm", "vorbis", "wav"], optional = true } rubato = { version = "0.16", default-features = false, optional = true } zip = { workspace = true }