🚨 breaking: Fix training with special tokens (#1617)

* fix training

* fmt

* clippy

Author: ArthurZucker

tokenizers/src/tokenizer/mod.rs

@@ -1128,6 +1128,7 @@ where
     }
 }
 
+#[allow(dead_code)]
 impl<M, N, PT, PP, D> TokenizerImpl<M, N, PT, PP, D>
 where
     N: Normalizer,
@@ -1388,7 +1389,9 @@ where
                         }
                     }),
                     |seq| {
-                        let normalized = self.do_normalize(seq.as_ref())?;
+                        let normalized = self
+                            .added_vocabulary
+                            .extract_and_normalize(self.normalizer.as_ref(), seq.as_ref());
                         let pre_tokenized = self.do_pre_tokenize(normalized)?;
                         Ok(pre_tokenized
                             .get_splits(OffsetReferential::Original, OffsetType::Byte)
@@ -1439,7 +1442,9 @@ where
                 }
             }),
             |seq| {
-                let normalized = self.do_normalize(seq.as_ref())?;
+                let normalized = self
+                    .added_vocabulary
+                    .extract_and_normalize(self.normalizer.as_ref(), seq.as_ref());
                 let pre_tokenized = self.do_pre_tokenize(normalized)?;
                 Ok(pre_tokenized
                     .get_splits(OffsetReferential::Original, OffsetType::Byte)