diff --git a/Cargo.lock b/Cargo.lock index f106012a..60215e53 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -101,6 +101,7 @@ dependencies = [ "const-random", "getrandom 0.3.4", "once_cell", + "serde", "version_check", "zerocopy", ] @@ -1337,6 +1338,21 @@ dependencies = [ "memchr", ] +[[package]] +name = "compact_str" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fdb1325a1cece981e8a296ab8f0f9b63ae357bd0784a9faaf548cc7b480707a" +dependencies = [ + "castaway", + "cfg-if 1.0.4", + "itoa", + "rustversion", + "ryu", + "serde", + "static_assertions", +] + [[package]] name = "concurrent-queue" version = "2.5.0" @@ -1736,6 +1752,15 @@ dependencies = [ "syn 2.0.111", ] +[[package]] +name = "dary_heap" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06d2e3287df1c007e74221c49ca10a95d557349e54b3a75dc2fb14712c751f04" +dependencies = [ + "serde", +] + [[package]] name = "dashmap" version = "5.5.3" @@ -3529,15 +3554,6 @@ dependencies = [ "either", ] -[[package]] -name = "itertools" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" -dependencies = [ - "either", -] - [[package]] name = "itertools" version = "0.13.0" @@ -6039,12 +6055,12 @@ dependencies = [ [[package]] name = "rayon-cond" -version = "0.3.0" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "059f538b55efd2309c9794130bc149c6a553db90e9d99c2030785c82f0bd7df9" +checksum = "2964d0cf57a3e7a06e8183d14a8b527195c706b7983549cd5462d5aa3747438f" dependencies = [ "either", - "itertools 0.11.0", + "itertools 0.14.0", "rayon", ] @@ -7877,22 +7893,24 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokenizers" -version = "0.19.1" +version = "0.22.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e500fad1dd3af3d626327e6a3fe5050e664a6eaa4708b8ca92f1794aaf73e6fd" +checksum = "b238e22d44a15349529690fb07bd645cf58149a1b1e44d6cb5bd1641ff1a6223" dependencies = [ + "ahash 0.8.12", "aho-corasick", + "compact_str", + "dary_heap", "derive_builder", "esaxx-rs", - "getrandom 0.2.16", - "itertools 0.12.1", - "lazy_static", + "getrandom 0.3.4", + "itertools 0.14.0", "log", "macro_rules_attribute", "monostate", "onig", "paste", - "rand 0.8.5", + "rand 0.9.2", "rayon", "rayon-cond", "regex", @@ -7900,7 +7918,7 @@ dependencies = [ "serde", "serde_json", "spm_precompiled", - "thiserror 1.0.69", + "thiserror 2.0.17", "unicode-normalization-alignments", "unicode-segmentation", "unicode_categories", diff --git a/crates/synapse-infra/Cargo.toml b/crates/synapse-infra/Cargo.toml index b6623c13..c9bbcedd 100644 --- a/crates/synapse-infra/Cargo.toml +++ b/crates/synapse-infra/Cargo.toml @@ -37,7 +37,7 @@ futures = "0.3" # AI Inference ort = { workspace = true } -tokenizers = { version = "0.19", default-features = false, features = ["onig"] } +tokenizers = { version = "0.22", default-features = false, features = ["onig"] } ndarray = "0.16" candle-core = { workspace = true } candle-nn = { workspace = true }