[SwiftParser] Improve statement level recovery (#3156)

Consider the following code:
```swift
}
@ATTR1 func foo() {}
```
Previously, this was parsed as a `FunctionDeclSyntax` with the unexpected
code `} @attr` attached as `unexpectedBeforeFuncKeyword`. This was
problematic because `@attr` was effectively ignored. The
`TokenPrecedence`-based recovery strategy couldn't handle this well, since
 `@` is not a declaration keyword and doesn't by itself signal the start
 of a declaration.

One option would be to check `atStartOfDeclaration()` after each
`skipSingle()` and recognize `@` as the start of a declaration. However,
then the preceding `}` would need to be attached to the first attribute in
the attribute list, which would require threading recovery logic into
`parseAttributeList()` and only applying it to the first `parseAttribute()`.
This would make the code more complex and harder to maintain.

This PR introduces a new syntax node `UnexpectedCodeDeclSyntax`
* It represents unexpected code at the statement/declaration level.
* All statement-level recovery is now handled by `parseUnexpectedCodeDeclaration()`.
This simplifies recovery handling significantly.

---

Also:
* Improve `#if` related recovery:
  * orphan `#endif` et al at top-level is now `UnexpectedCodeDecl` instead
    of making everything after it "unexpected"
    ```swift
    #endif
    func foo() {}
    ```
  * Unbalanced `#endif` et al closes code/member block with missing `}`.
     ```swift
     struct S {
       #if FOO
       func foo() {
       #endif
     }
     ```
  * Simplified `parsePoundDirective()`. It now receives only one closure
    instead of 3.
* Don't attach ';' to compiler directives. For example:
  ```swift
  #sourceLocation(file: "<file>", line: 100)
  ;
  ```
  `;` is now diagnosed as  `;`-only statement, or unexpected `;` separator
  depending on the position.
* Adjust TokenPrecedence for decl/statement keyword so. Decl keyword
  recovery won't go beyond `}` or statements. E.g.
  ```swift
  func foo() {
    @attr
  }
  struct S {}
  ```
  ```swift
  func foo() {
    @attr
    if true {}
    struct S {}
  }
  ```

Author: rintaro

CodeGeneration/Sources/SyntaxSupport/CommonNodes.swift

@@ -177,7 +177,7 @@ public let COMMON_NODES: [Node] = [
     kind: .decl,
     base: .syntax,
     nameForDiagnostics: "declaration",
-    parserFunction: "parseDeclaration"
+    parserFunction: "parseDeclarationOrIfConfig"
   ),
 
   Node(
@@ -401,4 +401,18 @@ public let COMMON_NODES: [Node] = [
     elementChoices: [.syntax]
   ),
 
+  Node(
+    kind: .unexpectedCodeDecl,
+    base: .decl,
+    nameForDiagnostics: nil,
+    documentation: "Unexpected code at declaration position",
+    children: [
+      Child(
+        name: "unexpectedCode",
+        // NOTE: This is not .collection() on purpose. We don't need collection related functions for this.
+        kind: .node(kind: .unexpectedNodes)
+      )
+    ],
+    noInterleaveUnexpected: true
+  ),
 ]

CodeGeneration/Sources/SyntaxSupport/Node.swift

@@ -127,7 +127,8 @@ public class Node: NodeChoiceConvertible {
     parserFunction: TokenSyntax? = nil,
     traits: [String] = [],
     children: [Child] = [],
-    childHistory: Child.History = []
+    childHistory: Child.History = [],
+    noInterleaveUnexpected: Bool = false
   ) {
     precondition(base != .syntaxCollection)
     precondition(base.isBase, "unknown base kind '\(base)' for node '\(kind)'")
@@ -140,7 +141,8 @@ public class Node: NodeChoiceConvertible {
     self.documentation = SwiftSyntax.Trivia.docCommentTrivia(from: documentation)
     self.parserFunction = parserFunction
 
-    let childrenWithUnexpected = kind.isBase ? children : interleaveUnexpectedChildren(children)
+    let childrenWithUnexpected =
+      (kind.isBase || noInterleaveUnexpected) ? children : interleaveUnexpectedChildren(children)
 
     self.data = .layout(children: childrenWithUnexpected, childHistory: childHistory, traits: traits)
   }

CodeGeneration/Sources/SyntaxSupport/SyntaxNodeKind.swift

@@ -301,6 +301,7 @@ public enum SyntaxNodeKind: String, CaseIterable, IdentifierConvertible, TypeCon
   case typeSpecifier
   case lifetimeSpecifierArguments
   case typeSpecifierList
+  case unexpectedCodeDecl
   case unexpectedNodes
   case unresolvedAsExpr
   case unresolvedIsExpr

CodeGeneration/Sources/generate-swift-syntax/templates/swiftparser/LayoutNodesParsableFile.swift

@@ -68,7 +68,7 @@ let layoutNodesParsableFile = SourceFileSyntax(leadingTrivia: copyrightHeader) {
     DeclSyntax(
       """
       mutating func parseNonOptionalCodeBlockItem() -> RawCodeBlockItemSyntax {
-        guard let node = self.parseCodeBlockItem(isAtTopLevel: false, allowInitDecl: true) else {
+        guard let node = self.parseCodeBlockItem(allowInitDecl: true, until: { _ in false }) else {
           // The missing item is not necessary to be a declaration,
           // which is just a placeholder here
           return RawCodeBlockItemSyntax(

CodeGeneration/Tests/ValidateSyntaxNodes/ValidateSyntaxNodes.swift

@@ -641,7 +641,12 @@ class ValidateSyntaxNodes: XCTestCase {
 
     assertFailuresMatchXFails(
       failures,
-      expectedFailures: []
+      expectedFailures: [
+        ValidationFailure(
+          node: .unexpectedCodeDecl,
+          message: "child 'unexpectedCode' is a SyntaxCollection but child is not marked as a collection"
+        )
+      ]
     )
   }
 

Sources/SwiftParser/Attributes.swift

@@ -33,11 +33,7 @@ extension Parser {
       //
       // In such cases, the second `#if` is not `consumeIfConfigOfAttributes()`.
       return .ifConfigDecl(
-        self.parsePoundIfDirective { (parser, _) -> RawAttributeListSyntax.Element in
-          return parser.parseAttributeListElement()
-        } syntax: { parser, attributes in
-          return .attributes(RawAttributeListSyntax(elements: attributes, arena: parser.arena))
-        }
+        self.parsePoundIfDirective({ .attributes($0.parseAttributeList()) })
       )
     } else {
       return .attribute(self.parseAttribute())
@@ -896,7 +892,32 @@ extension Parser {
       return makeMissingProviderArguments(unexpectedBefore: [])
     }
 
-    let decl = parseDeclaration(in: .argumentList)
+    let decl: RawDeclSyntax
+    if self.at(.poundIf) {
+      // '#if' is not accepted in '@abi' attribute, but for recovery, parse it
+      // and wrap the first decl in it with unexpected nodes.
+      let ifConfig = self.parsePoundIfDirective({ parser in
+        let decl = parser.parseDeclaration(in: .argumentList)
+        let member = RawMemberBlockItemSyntax(decl: decl, semicolon: nil, arena: parser.arena)
+        return .decls(RawMemberBlockItemListSyntax(elements: [member], arena: parser.arena))
+      })
+      decl = ifConfig.makeUnexpectedKeepingFirstNode(
+        of: RawDeclSyntax.self,
+        arena: self.arena,
+        where: { !$0.is(RawIfConfigDeclSyntax.self) },
+        makeMissing: {
+          RawDeclSyntax(
+            RawMissingDeclSyntax(
+              attributes: self.emptyCollection(RawAttributeListSyntax.self),
+              modifiers: self.emptyCollection(RawDeclModifierListSyntax.self),
+              arena: self.arena
+            )
+          )
+        }
+      )
+    } else {
+      decl = self.parseDeclaration(in: .argumentList)
+    }
 
     guard let provider = RawABIAttributeArgumentsSyntax.Provider(decl) else {
       return makeMissingProviderArguments(unexpectedBefore: [decl.raw])

Sources/SwiftParser/CollectionNodes+Parsable.swift

@@ -116,7 +116,7 @@ extension CodeBlockItemListSyntax: SyntaxParseable {
 extension MemberBlockItemListSyntax: SyntaxParseable {
   public static func parse(from parser: inout Parser) -> Self {
     return parse(from: &parser) { parser in
-      return parser.parseMemberDeclList()
+      return parser.parseMemberDeclList(until: { _ in false })
     } makeMissing: { remainingTokens, arena in
       let missingDecl = RawMissingDeclSyntax(
         attributes: RawAttributeListSyntax(elements: [], arena: arena),

Sources/SwiftParser/CompilerFiles.swift

@@ -89,7 +89,7 @@ extension Parser {
 
   /// Parse a declaration macro expansions in type contexts.
   mutating func parseMemberBlockItemListFile() -> RawMemberBlockItemListFileSyntax {
-    let members = self.parseMemberDeclList()
+    let members = self.parseMemberDeclList(until: { _ in false })
     let (unexpectedBeforeEndOfFileToken, endOfFile) = self.expectEndOfFile()
 
     return RawMemberBlockItemListFileSyntax(