Cleanup.

Author: Narsil

bindings/node/src/models.rs

@@ -9,7 +9,7 @@ use std::collections::HashMap;
 use std::path::{Path, PathBuf};
 use std::sync::{Arc, RwLock};
 use tokenizers as tk;
-use tokenizers::models::bpe::{BpeBuilder, Merges, Vocab};
+use tokenizers::models::bpe::{BpeBuilder, Merges};
 use tokenizers::models::wordlevel::WordLevelBuilder;
 use tokenizers::models::wordpiece::WordPieceBuilder;
 

tokenizers/src/models/bpe/trainer.rs

@@ -842,7 +842,7 @@ mod tests {
             .build();
         let mut model = BPE::default();
         trainer.do_train(&long_word_counts, &mut model).unwrap();
-        let trained_vocab: AHashMap<String, u32> = model.get_vocab_ahash();
+        let trained_vocab: AHashMap<String, u32> = model.get_vocab().into_iter().collect();
         let expected_vocab: AHashMap<String, u32> = [
             ("短", 12),
             ("n", 6),

tokenizers/src/tokenizer/added_vocabulary.rs

@@ -622,9 +622,6 @@ mod tests {
         fn get_vocab(&self) -> HashMap<String, u32> {
             self.vocab.clone().into_iter().collect()
         }
-        fn get_vocab_ahash(&self) -> AHashMap<String, u32> {
-            self.vocab.clone()
-        }
         fn get_vocab_size(&self) -> usize {
             self.vocab.len()
         }