File tree Expand file tree Collapse file tree 1 file changed +3
-2
lines changed Expand file tree Collapse file tree 1 file changed +3
-2
lines changed Original file line number Diff line number Diff line change @@ -65,19 +65,20 @@ static PATTERN_LEXER: LazyLock<Regex> = LazyLock::new(|| {
6565 #[ cfg( windows) ]
6666 {
6767 // On Windows, treat \r\n as a single newline token
68- Regex :: new ( r"(?P<whitespace> +)|(?P<newline>\r\n|\n+ )|(?P<tab>\t+ )" ) . unwrap ( )
68+ Regex :: new ( r"(?P<whitespace> +)|(?P<newline>\r\n|\n)|(?P<tab>\t)" ) . unwrap ( )
6969 }
7070 #[ cfg( not( windows) ) ]
7171 {
7272 // On other platforms, just check for \n
73- Regex :: new ( r"(?P<whitespace> +)|(?P<newline>\n+ )|(?P<tab>\t+ )" ) . unwrap ( )
73+ Regex :: new ( r"(?P<whitespace> +)|(?P<newline>\n)|(?P<tab>\t)" ) . unwrap ( )
7474 }
7575} ) ;
7676
7777fn whitespace_tokens ( input : & str ) -> VecDeque < Token > {
7878 let mut tokens = VecDeque :: new ( ) ;
7979
8080 for cap in PATTERN_LEXER . captures_iter ( input) {
81+ println ! ( "{:?}" , cap) ;
8182 if let Some ( whitespace) = cap. name ( "whitespace" ) {
8283 tokens. push_back ( Token {
8384 token_type : TokenType :: Whitespace ,
You can’t perform that action at this time.
0 commit comments