Upgrading dependencies. (#1801)

Author: Narsil

bindings/python/Cargo.toml

@@ -14,17 +14,17 @@ serde = { version = "1.0", features = ["rc", "derive"] }
 serde_json = "1.0"
 libc = "0.2"
 env_logger = "0.11"
-pyo3 = { version = "0.24.2", features = ["abi3", "abi3-py39", "py-clone"] }
-numpy = "0.24"
+pyo3 = { version = "0.25", features = ["abi3", "abi3-py39", "py-clone"] }
+numpy = "0.25"
 ndarray = "0.16"
-itertools = "0.12"
+itertools = "0.14"
 
 [dependencies.tokenizers]
 path = "../../tokenizers"
 
 [dev-dependencies]
 tempfile = "3.10"
-pyo3 = { version = "0.24.2", features = ["auto-initialize"] }
+pyo3 = { version = "0.25", features = ["auto-initialize"] }
 
 [features]
 default = ["pyo3/extension-module"]

tokenizers/Cargo.toml

@@ -41,7 +41,7 @@ name = "llama3_benchmark"
 harness = false
 
 [dependencies]
-rand = "0.8"
+rand = "0.9"
 onig = { version = "6.5.1", default-features = false, optional = true }
 regex = "1.10"
 regex-syntax = "0.8"
@@ -63,7 +63,7 @@ paste = "1.0.14"
 macro_rules_attribute = "0.2.0"
 thiserror = "2"
 fancy-regex = { version = "0.14", optional = true}
-getrandom = { version = "0.2.10" }
+getrandom = { version = "0.3" }
 esaxx-rs = { version = "0.1.10", default-features = false, features=[]}
 monostate = "0.1.12"
 
@@ -72,11 +72,11 @@ default = ["progressbar", "onig", "esaxx_fast"]
 esaxx_fast = ["esaxx-rs/cpp"]
 progressbar = ["indicatif"]
 http = ["hf-hub"]
-unstable_wasm = ["fancy-regex", "getrandom/js"]
+unstable_wasm = ["fancy-regex", "getrandom/wasm_js"]
 rustls-tls = ["hf-hub?/rustls-tls"]
 
 [dev-dependencies]
-criterion = "0.5"
+criterion = "0.6"
 tempfile = "3.10"
 assert_approx_eq = "1.1"
 tracing = "0.1"

tokenizers/benches/common/mod.rs

@@ -1,6 +1,6 @@
 use std::time::{Duration, Instant};
 
-use criterion::black_box;
+use std::hint::black_box;
 
 use tokenizers::{
     Decoder, EncodeInput, Model, Normalizer, PostProcessor, PreTokenizer, TokenizerImpl, Trainer,

tokenizers/benches/layout_benchmark.rs

@@ -6,8 +6,8 @@ use std::io::{BufRead, BufReader};
 use std::path::Path;
 use std::time::{Duration, Instant};
 
-use criterion::black_box;
 use criterion::Criterion;
+use std::hint::black_box;
 use tokenizers::processors::template::TemplateProcessing;
 use tokenizers::{EncodeInput, Encoding, PostProcessor, Tokenizer};
 

tokenizers/benches/llama3_benchmark.rs

@@ -5,6 +5,7 @@ mod common;
 
 use common::{iter_bench_encode, iter_bench_encode_batch, iter_bench_train};
 use criterion::{Criterion, Throughput};
+use std::hint::black_box;
 use tokenizers::{
     models::{bpe::BpeTrainerBuilder, TrainerWrapper},
     EncodeInput, Tokenizer,
@@ -32,7 +33,7 @@ pub fn llama3(c: &mut Criterion) {
         let add_special_tokens = false;
         b.iter(|| {
             tokenizer
-                .encode_batch_char_offsets(criterion::black_box(data.clone()), add_special_tokens)
+                .encode_batch_char_offsets(black_box(data.clone()), add_special_tokens)
                 .unwrap()
         })
     });
@@ -42,15 +43,6 @@ pub fn llama3(c: &mut Criterion) {
     group.bench_function("llama3-batch", |b| {
         b.iter_custom(|iters| iter_bench_encode_batch(iters, &tokenizer, &batches))
     });
-    // group.bench_function("llama3-nooffsets", |b| {
-    //     let data: Vec<_> = data.lines().collect();
-    //     let add_special_tokens = false;
-    //     b.iter(|| {
-    //         tokenizer
-    //             .encode_batch(criterion::black_box(data.clone()), add_special_tokens)
-    //             .unwrap()
-    //     })
-    // });
     let mut trainer: TrainerWrapper = BpeTrainerBuilder::default()
         .show_progress(false)
         .build()

tokenizers/src/models/bpe/word.rs

@@ -1,5 +1,5 @@
 use super::Pair;
-use rand::{thread_rng, Rng};
+use rand::{rng, Rng};
 use std::cmp::Ordering;
 use std::collections::{BinaryHeap, HashMap};
 
@@ -177,10 +177,7 @@ impl Word {
         );
 
         while let Some(top) = queue.pop() {
-            if dropout
-                .map(|d| thread_rng().gen::<f32>() < d)
-                .unwrap_or(false)
-            {
+            if dropout.map(|d| rng().random::<f32>() < d).unwrap_or(false) {
                 skip.push(top);
             } else {
                 // Re-insert the skipped elements

tokenizers/src/models/unigram/lattice.rs

@@ -1,5 +1,5 @@
-use rand::distributions::WeightedIndex;
-use rand::prelude::*;
+use rand::distr::weighted::WeightedIndex;
+use rand::{prelude::*, rng};
 use std::cell::RefCell;
 use std::cmp::{min, Ordering};
 use std::collections::BinaryHeap;
@@ -397,7 +397,7 @@ impl<'a> Lattice<'a> {
             }
         }
 
-        let mut rng = thread_rng();
+        let mut rng = rng();
         let mut results: Vec<NodeRef> = vec![];
         let mut probs: Vec<f64> = vec![];
         let mut z = alpha[self.eos_node().borrow().node_id];