Rewrite parser WIP [skip ci]

Author: jg-rp

jsonpath/env.py

@@ -92,8 +92,8 @@ class attributes `root_token`, `self_token` and `filter_context_token`.
     ## Class attributes
 
     Attributes:
-        pseudo_root_token (str): The pattern used to select a "fake" root node, one level
-            above the real root node.
+        pseudo_root_token (str): The pattern used to select a "fake" root node, one
+            level above the real root node.
         filter_context_token (str): The pattern used to select extra filter context
             data. Defaults to `"_"`.
         intersection_token (str): The pattern used as the intersection operator.
@@ -180,34 +180,35 @@ def compile(self, path: str) -> Union[JSONPath, CompoundJSONPath]:  # noqa: A003
         """
         tokens = self.lexer.tokenize(path)
         stream = TokenStream(tokens)
-        pseudo_root = stream.current.kind == TOKEN_PSEUDO_ROOT
+        pseudo_root = stream.current().kind == TOKEN_PSEUDO_ROOT
         _path: Union[JSONPath, CompoundJSONPath] = JSONPath(
             env=self, segments=self.parser.parse(stream), pseudo_root=pseudo_root
         )
 
-        if stream.current.kind != TOKEN_EOF:
+        # TODO: better!
+        if stream.current().kind != TOKEN_EOF:
             _path = CompoundJSONPath(env=self, path=_path)
-            while stream.current.kind != TOKEN_EOF:
-                if stream.peek.kind == TOKEN_EOF:
+            while stream.current().kind != TOKEN_EOF:
+                if stream.peek().kind == TOKEN_EOF:
                     # trailing union or intersection
                     raise JSONPathSyntaxError(
-                        f"expected a path after {stream.current.value!r}",
-                        token=stream.current,
+                        f"expected a path after {stream.current().value!r}",
+                        token=stream.current(),
                     )
 
-                if stream.current.kind == TOKEN_UNION:
-                    stream.next_token()
-                    pseudo_root = stream.current.kind == TOKEN_PSEUDO_ROOT
+                if stream.current().kind == TOKEN_UNION:
+                    stream.next()
+                    pseudo_root = stream.current().kind == TOKEN_PSEUDO_ROOT
                     _path = _path.union(
                         JSONPath(
                             env=self,
                             segments=self.parser.parse(stream),
                             pseudo_root=pseudo_root,
                         )
                     )
-                elif stream.current.kind == TOKEN_INTERSECTION:
-                    stream.next_token()
-                    pseudo_root = stream.current.kind == TOKEN_PSEUDO_ROOT
+                elif stream.current().kind == TOKEN_INTERSECTION:
+                    stream.next()
+                    pseudo_root = stream.current().kind == TOKEN_PSEUDO_ROOT
                     _path = _path.intersection(
                         JSONPath(
                             env=self,
@@ -218,8 +219,8 @@ def compile(self, path: str) -> Union[JSONPath, CompoundJSONPath]:  # noqa: A003
                 else:  # pragma: no cover
                     # Parser.parse catches this too
                     raise JSONPathSyntaxError(  # noqa: TRY003
-                        f"unexpected token {stream.current.value!r}",
-                        token=stream.current,
+                        f"unexpected token {stream.current().value!r}",
+                        token=stream.current(),
                     )
 
         return _path

jsonpath/lex.py

@@ -15,13 +15,15 @@
 from .token import TOKEN_CONTAINS
 from .token import TOKEN_DDOT
 from .token import TOKEN_DOT
+from .token import TOKEN_DOT_PROPERTY
 from .token import TOKEN_DOUBLE_QUOTE_STRING
 from .token import TOKEN_EQ
 from .token import TOKEN_ERROR
 from .token import TOKEN_FALSE
 from .token import TOKEN_FILTER
 from .token import TOKEN_FILTER_CONTEXT
 from .token import TOKEN_FLOAT
+from .token import TOKEN_FUNCTION
 from .token import TOKEN_GE
 from .token import TOKEN_GT
 from .token import TOKEN_IN
@@ -82,7 +84,6 @@ class attributes. Then setting `lexer_class` on a `JSONPathEnvironment`.
     """
 
     key_pattern = r"[\u0080-\uFFFFa-zA-Z_][\u0080-\uFFFFa-zA-Z0-9_-]*"
-    name_pattern = key_pattern  # XXX:
 
     # ! or `not`
     logical_not_pattern = r"(?:not\b)|!"
@@ -99,10 +100,15 @@ def __init__(self, *, env: JSONPathEnvironment) -> None:
         self.double_quote_pattern = r'"(?P<G_DQUOTE>(?:(?!(?<!\\)").)*)"'
         self.single_quote_pattern = r"'(?P<G_SQUOTE>(?:(?!(?<!\\)').)*)'"
 
-        # TODO: separate re literal tokens
+        # .thing
+        self.dot_property_pattern = rf"(?P<G_DOT>\.)(?P<G_PROP>{self.key_pattern})"
+
         # /pattern/ or /pattern/flags
         self.re_pattern = r"/(?P<G_RE>.+?)/(?P<G_RE_FLAGS>[aims]*)"
 
+        # func(
+        self.function_pattern = r"(?P<G_FUNC>[a-z][a-z_0-9]+)(?P<G_FUNC_PAREN>\()"
+
         self.rules = self.compile_rules()
 
     def compile_rules(self) -> Pattern[str]:
@@ -122,6 +128,7 @@ def compile_rules(self) -> Pattern[str]:
             (TOKEN_DOUBLE_QUOTE_STRING, self.double_quote_pattern),
             (TOKEN_SINGLE_QUOTE_STRING, self.single_quote_pattern),
             (TOKEN_RE_PATTERN, self.re_pattern),
+            (TOKEN_DOT_PROPERTY, self.dot_property_pattern),
             (TOKEN_FLOAT, r"-?\d+\.\d*(?:[eE][+-]?\d+)?"),
             (TOKEN_INT, r"-?\d+(?P<G_EXP>[eE][+\-]?\d+)?\b"),
             (TOKEN_DDOT, r"\.\."),
@@ -160,6 +167,7 @@ def compile_rules(self) -> Pattern[str]:
             (TOKEN_LT, r"<"),
             (TOKEN_GT, r">"),
             (TOKEN_NOT, self.logical_not_pattern),  # Must go after "!="
+            (TOKEN_FUNCTION, self.function_pattern),
             (TOKEN_NAME, self.key_pattern),  # Must go after reserved words
             (TOKEN_LPAREN, r"\("),
             (TOKEN_RPAREN, r"\)"),
@@ -180,7 +188,18 @@ def tokenize(self, path: str) -> Iterator[Token]:  # noqa PLR0912
             kind = match.lastgroup
             assert kind is not None
 
-            if kind == TOKEN_DOUBLE_QUOTE_STRING:
+            if kind == TOKEN_DOT_PROPERTY:
+                yield _token(
+                    kind=TOKEN_DOT,
+                    value=match.group("G_DOT"),
+                    index=match.start("G_DOT"),
+                )
+                yield _token(
+                    kind=TOKEN_NAME,
+                    value=match.group("G_PROP"),
+                    index=match.start("G_PROP"),
+                )
+            elif kind == TOKEN_DOUBLE_QUOTE_STRING:
                 yield _token(
                     kind=TOKEN_DOUBLE_QUOTE_STRING,
                     value=match.group("G_DQUOTE"),
@@ -222,6 +241,18 @@ def tokenize(self, path: str) -> Iterator[Token]:  # noqa PLR0912
                     value=match.group(),
                     index=match.start(),
                 )
+            elif kind == TOKEN_FUNCTION:
+                yield _token(
+                    kind=TOKEN_FUNCTION,
+                    value=match.group("G_FUNC"),
+                    index=match.start("G_FUNC"),
+                )
+
+                yield _token(
+                    kind=TOKEN_LPAREN,
+                    value=match.group("G_FUNC_PAREN"),
+                    index=match.start("G_FUNC_PAREN"),
+                )
             elif kind == TOKEN_ERROR:
                 raise JSONPathSyntaxError(
                     f"unexpected token {match.group()!r}",