Sync latest parser files from rescript-lang/rescript#6073

This fixes highlighting, and other things, in presence of unicode strings.

Author: cristianoc

analysis/vendor/res_syntax/res_core.ml

@@ -2469,6 +2469,7 @@ and parseAttributesAndBinding (p : Parser.t) =
   let err = p.scanner.err in
   let ch = p.scanner.ch in
   let offset = p.scanner.offset in
+  let offset16 = p.scanner.offset16 in
   let lineOffset = p.scanner.lineOffset in
   let lnum = p.scanner.lnum in
   let mode = p.scanner.mode in
@@ -2490,6 +2491,7 @@ and parseAttributesAndBinding (p : Parser.t) =
       p.scanner.err <- err;
       p.scanner.ch <- ch;
       p.scanner.offset <- offset;
+      p.scanner.offset16 <- offset16;
       p.scanner.lineOffset <- lineOffset;
       p.scanner.lnum <- lnum;
       p.scanner.mode <- mode;

analysis/vendor/res_syntax/res_driver.ml

@@ -131,24 +131,31 @@ let printEngine =
         print_string (Res_printer.printInterface ~width signature ~comments));
   }
 
-let parse_implementation sourcefile =
+let parse_implementation ?(ignoreParseErrors = false) sourcefile =
   Location.input_name := sourcefile;
   let parseResult =
     parsingEngine.parseImplementation ~forPrinter:false ~filename:sourcefile
   in
   if parseResult.invalid then (
     Res_diagnostics.printReport parseResult.diagnostics parseResult.source;
-    exit 1);
+    if not ignoreParseErrors then exit 1);
   parseResult.parsetree
   [@@raises exit]
 
-let parse_interface sourcefile =
+let parse_interface ?(ignoreParseErrors = false) sourcefile =
   Location.input_name := sourcefile;
   let parseResult =
     parsingEngine.parseInterface ~forPrinter:false ~filename:sourcefile
   in
   if parseResult.invalid then (
     Res_diagnostics.printReport parseResult.diagnostics parseResult.source;
-    exit 1);
+    if not ignoreParseErrors then exit 1);
   parseResult.parsetree
   [@@raises exit]
+
+(* suppress unused optional arg *)
+let _ =
+ fun s ->
+  ( parse_implementation ~ignoreParseErrors:false s,
+    parse_interface ~ignoreParseErrors:false s )
+ [@@raises exit]

analysis/vendor/res_syntax/res_driver.mli

@@ -53,9 +53,10 @@ val parsingEngine : Res_diagnostics.t list parsingEngine
 val printEngine : printEngine
 
 (* ReScript implementation parsing compatible with ocaml pparse driver. Used by the compiler. *)
-val parse_implementation : string -> Parsetree.structure
+val parse_implementation :
+  ?ignoreParseErrors:bool -> string -> Parsetree.structure
   [@@live] [@@raises Location.Error]
 
 (* ReScript interface parsing compatible with ocaml pparse driver. Used by the compiler *)
-val parse_interface : string -> Parsetree.signature
+val parse_interface : ?ignoreParseErrors:bool -> string -> Parsetree.signature
   [@@live] [@@raises Location.Error]

analysis/vendor/res_syntax/res_multi_printer.ml

@@ -1,27 +1,25 @@
 let defaultPrintWidth = 100
 
 (* print res files to res syntax *)
-let printRes ~isInterface ~filename =
-  if isInterface then
+let printRes ~ignoreParseErrors ~isInterface ~filename =
+  if isInterface then (
     let parseResult =
       Res_driver.parsingEngine.parseInterface ~forPrinter:true ~filename
     in
     if parseResult.invalid then (
       Res_diagnostics.printReport parseResult.diagnostics parseResult.source;
-      exit 1)
-    else
-      Res_printer.printInterface ~width:defaultPrintWidth
-        ~comments:parseResult.comments parseResult.parsetree
+      if not ignoreParseErrors then exit 1);
+    Res_printer.printInterface ~width:defaultPrintWidth
+      ~comments:parseResult.comments parseResult.parsetree)
   else
     let parseResult =
       Res_driver.parsingEngine.parseImplementation ~forPrinter:true ~filename
     in
     if parseResult.invalid then (
       Res_diagnostics.printReport parseResult.diagnostics parseResult.source;
-      exit 1)
-    else
-      Res_printer.printImplementation ~width:defaultPrintWidth
-        ~comments:parseResult.comments parseResult.parsetree
+      if not ignoreParseErrors then exit 1);
+    Res_printer.printImplementation ~width:defaultPrintWidth
+      ~comments:parseResult.comments parseResult.parsetree
   [@@raises exit]
 
 (* print ocaml files to res syntax *)
@@ -42,12 +40,15 @@ let printMl ~isInterface ~filename =
       ~comments:parseResult.comments parseResult.parsetree
 
 (* print the given file named input to from "language" to res, general interface exposed by the compiler *)
-let print language ~input =
+let print ?(ignoreParseErrors = false) language ~input =
   let isInterface =
     let len = String.length input in
     len > 0 && String.unsafe_get input (len - 1) = 'i'
   in
   match language with
-  | `res -> printRes ~isInterface ~filename:input
+  | `res -> printRes ~ignoreParseErrors ~isInterface ~filename:input
   | `ml -> printMl ~isInterface ~filename:input
   [@@raises exit]
+
+(* suppress unused optional arg *)
+let _ = fun s -> print ~ignoreParseErrors:false s [@@raises exit]

analysis/vendor/res_syntax/res_multi_printer.mli

@@ -1,3 +1,3 @@
 (* Interface to print source code from different languages to res.
  * Takes a filename called "input" and returns the corresponding formatted res syntax *)
-val print : [`ml | `res] -> input:string -> string
+val print : ?ignoreParseErrors:bool -> [`ml | `res] -> input:string -> string

analysis/vendor/res_syntax/res_parser.ml

@@ -159,6 +159,7 @@ let lookahead p callback =
   let err = p.scanner.err in
   let ch = p.scanner.ch in
   let offset = p.scanner.offset in
+  let offset16 = p.scanner.offset16 in
   let lineOffset = p.scanner.lineOffset in
   let lnum = p.scanner.lnum in
   let mode = p.scanner.mode in
@@ -177,6 +178,7 @@ let lookahead p callback =
   p.scanner.err <- err;
   p.scanner.ch <- ch;
   p.scanner.offset <- offset;
+  p.scanner.offset16 <- offset16;
   p.scanner.lineOffset <- lineOffset;
   p.scanner.lnum <- lnum;
   p.scanner.mode <- mode;

analysis/vendor/res_syntax/res_printer_state.ml

@@ -0,0 +1,18 @@
+let customLayoutThreshold = 2
+
+type t = {
+  customLayout: int;
+  mutable uncurried_config: Res_uncurried.config;
+  customInfixOperators: (string, string) Hashtbl.t;
+}
+
+let init =
+  {
+    customLayout = 0;
+    uncurried_config = Res_uncurried.init;
+    customInfixOperators = Hashtbl.create 0;
+  }
+
+let nextCustomLayout t = {t with customLayout = t.customLayout + 1}
+
+let shouldBreakCallback t = t.customLayout > customLayoutThreshold

analysis/vendor/res_syntax/res_scanner.ml

@@ -19,7 +19,9 @@ type t = {
     Diagnostics.category ->
     unit;
   mutable ch: charEncoding; (* current character *)
-  mutable offset: int; (* character offset *)
+  mutable offset: int; (* current byte offset *)
+  mutable offset16: int;
+      (* current number of utf16 code units since line start *)
   mutable lineOffset: int; (* current line offset *)
   mutable lnum: int; (* current line number *)
   mutable mode: mode list;
@@ -51,12 +53,11 @@ let position scanner =
       (* line number *)
       pos_lnum = scanner.lnum;
       (* offset of the beginning of the line (number
-         of characters between the beginning of the scanner and the beginning
+         of bytes between the beginning of the scanner and the beginning
          of the line) *)
       pos_bol = scanner.lineOffset;
-      (* [pos_cnum] is the offset of the position (number of
-         characters between the beginning of the scanner and the position). *)
-      pos_cnum = scanner.offset;
+      (* [pos_cnum - pos_bol]  is the number of utf16 code units since line start *)
+      pos_cnum = scanner.lineOffset + scanner.offset16;
     }
 
 (* Small debugging util
@@ -95,19 +96,29 @@ let _printDebug ~startPos ~endPos scanner token =
 
 let next scanner =
   let nextOffset = scanner.offset + 1 in
-  (match scanner.ch with
-  | '\n' ->
-    scanner.lineOffset <- nextOffset;
-    scanner.lnum <- scanner.lnum + 1
+  let utf16len =
+    match Ext_utf8.classify scanner.ch with
+    | Single _ | Invalid -> 1
+    | Leading (n, _) -> ( (((n + 1) / 2) [@doesNotRaise]))
+    | Cont _ -> 0
+  in
+  let newline =
+    scanner.ch = '\n'
     (* What about CRLF (\r + \n) on windows?
-     * \r\n will always be terminated by a \n
-     * -> we can just bump the line count on \n *)
-  | _ -> ());
+       \r\n will always be terminated by a \n
+       -> we can just bump the line count on \n *)
+  in
+  if newline then (
+    scanner.lineOffset <- nextOffset;
+    scanner.offset16 <- 0;
+    scanner.lnum <- scanner.lnum + 1)
+  else scanner.offset16 <- scanner.offset16 + utf16len;
   if nextOffset < String.length scanner.src then (
     scanner.offset <- nextOffset;
-    scanner.ch <- String.unsafe_get scanner.src scanner.offset)
+    scanner.ch <- String.unsafe_get scanner.src nextOffset)
   else (
     scanner.offset <- String.length scanner.src;
+    scanner.offset16 <- scanner.offset - scanner.lineOffset;
     scanner.ch <- hackyEOFChar)
 
 let next2 scanner =
@@ -141,6 +152,7 @@ let make ~filename src =
     err = (fun ~startPos:_ ~endPos:_ _ -> ());
     ch = (if src = "" then hackyEOFChar else String.unsafe_get src 0);
     offset = 0;
+    offset16 = 0;
     lineOffset = 0;
     lnum = 1;
     mode = [];
@@ -847,6 +859,7 @@ let rec scan scanner =
       | ch, _ ->
         next scanner;
         let offset = scanner.offset in
+        let offset16 = scanner.offset16 in
         let codepoint, length =
           Res_utf8.decodeCodePoint scanner.offset scanner.src
             (String.length scanner.src)
@@ -863,6 +876,7 @@ let rec scan scanner =
         else (
           scanner.ch <- ch;
           scanner.offset <- offset;
+          scanner.offset16 <- offset16;
           SingleQuote))
     | '!' -> (
       match (peek scanner, peek2 scanner) with

analysis/vendor/res_syntax/res_scanner.mli

@@ -11,7 +11,9 @@ type t = {
     Res_diagnostics.category ->
     unit;
   mutable ch: charEncoding; (* current character *)
-  mutable offset: int; (* character offset *)
+  mutable offset: int; (* current byte offset *)
+  mutable offset16: int;
+      (* current number of utf16 code units since line start *)
   mutable lineOffset: int; (* current line offset *)
   mutable lnum: int; (* current line number *)
   mutable mode: mode list;