We read every piece of feedback, and take your input very seriously.
To see all available qualifiers, see our documentation.
There was an error while loading. Please reload this page.
1 parent 2658dd2 commit b23b903Copy full SHA for b23b903
bindings/python/py_src/tokenizers/pre_tokenizers/__init__.pyi
@@ -535,7 +535,7 @@ class Whitespace(PreTokenizer):
535
536
pre_tokenizer = Whitespace()
537
text = "Hello, world! Let's try the Whitespace pre-tokenizer."
538
- pre_tokenizer.pretokenize_str(text)
+ pre_tokenizer.pre_tokenize_str(text)
539
[('Hello', (0, 5)),
540
(',', (5, 6)),
541
('world', (7, 12)),
0 commit comments