Run tree_sitter.parse with whole text.

Drop 3.9 from supported tree-sitter Python versions.
This allows predicates in SCM files to work.

Author: paul-ollis

.github/workflows/pythonpackage.yml

@@ -35,18 +35,18 @@ jobs:
           cache: "poetry"
       - name: Install dependencies
         run: poetry install --no-interaction --extras syntax
-        if: ${{ matrix.python-version != '3.8' }}
-      - name: Install dependencies for 3.8
+        if: ${{ matrix.python-version != '3.8' && matrix.python-version != '3.9' }}
+      - name: Install dependencies for 3.8 and 3.9
         run: poetry install --no-interaction
-        if: ${{ matrix.python-version == '3.8' }}
+        if: ${{ matrix.python-version == '3.8' || matrix.python-version == '3.9' }}
       - name: Test with pytest (Py39+ - with syntax highlighting)
         run: |
           poetry run pytest tests -v --cov=./src/textual --cov-report=xml:./coverage.xml --cov-report term-missing
-        if: ${{ matrix.python-version != '3.8' }}
+        if: ${{ matrix.python-version != '3.8' && matrix.python-version != '3.9' }}
       - name: Test with pytest (Py38 - without syntax highlighting)
         run: |
           poetry run pytest tests -v --cov=./src/textual --cov-report=xml:./coverage.xml --cov-report term-missing -m 'not syntax'
-        if: ${{ matrix.python-version == '3.8' }}
+        if: ${{ matrix.python-version == '3.8' || matrix.python-version == '3.9' }}
       - name: Upload snapshot report
         if: always()
         uses: actions/upload-artifact@v4

poetry.lock

@@ -9,7 +9,7 @@ description = "Modern Text User Interface framework"
 authors = ["Will McGugan <will@textualize.io>"]
 license = "MIT"
 readme = "README.md"
-classifiers = [    
+classifiers = [
     "Development Status :: 5 - Production/Stable",
     "Environment :: Console",
     "Intended Audience :: Developers",
@@ -52,24 +52,24 @@ typing-extensions = "^4.4.0"
 platformdirs = ">=3.6.0,<5"
 
 # start of [syntax] extras
-# Require tree-sitter >= 0.23.0 and python >= 3.9
+# Require tree-sitter >= 0.24.0 and python >= 3.10
 # Windows, MacOS and Linux binary wheels are available for all of the languages below.
-tree-sitter = { version = ">=0.23.0", optional = true, python = ">=3.9" }
-tree-sitter-python = { version = ">=0.23.0", optional = true, python = ">=3.9" }
-tree-sitter-markdown = { version = ">=0.3.0", optional = true, python = ">=3.9"}
-tree-sitter-json = { version = ">=0.24.0", optional = true, python = ">=3.9" }
-tree-sitter-toml = { version = ">=0.6.0", optional = true, python = ">=3.9" }
-tree-sitter-yaml = { version = ">=0.6.0", optional = true, python = ">=3.9" }
-tree-sitter-html = { version = ">=0.23.0", optional = true, python = ">=3.9" }
-tree-sitter-css = { version = ">=0.23.0", optional = true, python = ">=3.9" }
-tree-sitter-javascript = { version = ">=0.23.0", optional = true, python = ">=3.9" }
-tree-sitter-rust = { version = ">=0.23.0", optional = true, python = ">=3.9" }
-tree-sitter-go = { version = ">=0.23.0", optional = true, python = ">=3.9" }
-tree-sitter-regex = { version = ">=0.24.0", optional = true, python = ">=3.9" }
-tree-sitter-xml = { version = ">=0.7.0", optional = true, python = ">=3.9" }
-tree-sitter-sql = { version = ">=0.3.0,<0.3.8", optional = true, python = ">=3.9" }
-tree-sitter-java = { version = ">=0.23.0", optional = true, python = ">=3.9" }
-tree-sitter-bash = { version = ">=0.23.0", optional = true, python = ">=3.9" }
+tree-sitter = { version = ">=0.24.0", optional = true, python = ">=3.10" }
+tree-sitter-python = { version = ">=0.23.0", optional = true, python = ">=3.10" }
+tree-sitter-markdown = { version = ">=0.3.0", optional = true, python = ">=3.10"}
+tree-sitter-json = { version = ">=0.24.0", optional = true, python = ">=3.10" }
+tree-sitter-toml = { version = ">=0.6.0", optional = true, python = ">=3.10" }
+tree-sitter-yaml = { version = ">=0.6.0", optional = true, python = ">=3.10" }
+tree-sitter-html = { version = ">=0.23.0", optional = true, python = ">=3.10" }
+tree-sitter-css = { version = ">=0.23.0", optional = true, python = ">=3.10" }
+tree-sitter-javascript = { version = ">=0.23.0", optional = true, python = ">=3.10" }
+tree-sitter-rust = { version = ">=0.23.0", optional = true, python = ">=3.10" }
+tree-sitter-go = { version = ">=0.23.0", optional = true, python = ">=3.10" }
+tree-sitter-regex = { version = ">=0.24.0", optional = true, python = ">=3.10" }
+tree-sitter-xml = { version = ">=0.7.0", optional = true, python = ">=3.10" }
+tree-sitter-sql = { version = ">=0.3.0,<0.3.8", optional = true, python = ">=3.10" }
+tree-sitter-java = { version = ">=0.23.0", optional = true, python = ">=3.10" }
+tree-sitter-bash = { version = ">=0.23.0", optional = true, python = ">=3.10" }
 # end of [syntax] extras
 
 [tool.poetry.extras]

pyproject.toml

@@ -2,7 +2,6 @@
 
 import weakref
 from asyncio import CancelledError, Event, Task, create_task, sleep
-from functools import partial
 from typing import Callable, NamedTuple
 
 try:
@@ -61,7 +60,7 @@ def __init__(
         """The tree-sitter Parser or None if tree-sitter is unavailable."""
 
         self._syntax_tree: Tree = self._parser.parse(
-            partial(self._read_callable, lines=self.lines)
+            self.text.encode("utf-8")
         )  # type: ignore
         """The tree-sitter Tree (syntax tree) built from the document."""
 
@@ -165,7 +164,7 @@ def replace_range(self, start: Location, end: Location, text: str) -> EditResult
         )
         return replace_result
 
-    def reparse(self, timeout_us: int, lines: list[str], syntax_tree=None) -> bool:
+    def reparse(self, timeout_us: int, text: bytes) -> bool:
         """Reparse the document.
 
         Args:
@@ -176,13 +175,12 @@ def reparse(self, timeout_us: int, lines: list[str], syntax_tree=None) -> bool:
             True if parsing succeeded and False if a timeout occurred.
         """
         assert timeout_us > 0
-        read_source = partial(self._read_callable, lines=lines)
         tree = self._syntax_tree
         saved_timeout = self._parser.timeout_micros
         try:
             self._parser.timeout_micros = timeout_us
             try:
-                tree = self._parser.parse(read_source, tree)  # type: ignore[arg-type]
+                tree = self._parser.parse(text, tree)  # type: ignore[arg-type]
             except ValueError:
                 # The only known cause is a timeout.
                 return False
@@ -194,7 +192,7 @@ def set_new_tree():
                         self._syntax_tree = tree
 
                     changed_ranges = self._syntax_tree.changed_ranges(tree)
-                    self._syntax_tree_update_callback(self._syntax_tree, len(lines))
+                    self._syntax_tree_update_callback(self._syntax_tree)
                 else:
                     self._syntax_tree = tree
                 return True
@@ -256,50 +254,6 @@ def _location_to_point(self, location: Location) -> tuple[int, int]:
             bytes_on_left = 0
         return row, bytes_on_left
 
-    def _read_callable(
-        self,
-        byte_offset: int,
-        point: tuple[int, int],
-        lines: list[str],
-    ) -> bytes:
-        """A callable which informs tree-sitter about the document content.
-
-        This is passed to tree-sitter which will call it frequently to retrieve
-        the bytes from the document.
-
-        Args:
-            byte_offset: The number of (utf-8) bytes from the start of the document.
-            point: A tuple (row index, column *byte* offset). Note that this differs
-                from our Location tuple which is (row_index, column codepoint offset).
-            lines: The lines of the document being parsed.
-
-        Returns:
-            All the utf-8 bytes between the byte_offset/point and the end of the current
-                line _including_ the line separator character(s). Returns None if the
-                offset/point requested by tree-sitter doesn't correspond to a byte.
-        """
-        row, column = point
-        newline = self.newline
-
-        row_out_of_bounds = row >= len(lines)
-        if row_out_of_bounds:
-            return b""
-        else:
-            row_text = lines[row]
-
-        encoded_row = _utf8_encode(row_text)
-        encoded_row_length = len(encoded_row)
-
-        if column < encoded_row_length:
-            return encoded_row[column:] + _utf8_encode(newline)
-        elif column == encoded_row_length:
-            return _utf8_encode(newline[0])
-        elif column == encoded_row_length + 1:
-            if newline == "\r\n":
-                return b"\n"
-
-        return b""
-
 
 class BackgroundSyntaxParser:
     """A provider of incremental background parsing for syntax highlighting.
@@ -357,15 +311,15 @@ async def _perform_a_single_reparse(self, force_update: bool) -> None:
 
         # In order to allow the user to continue editing without interruption, we reparse
         # a snapshot of the TextArea's document.
-        copy_of_text_for_parsing = document.copy_of_lines()
+        copy_of_text_for_parsing = document.text.encode("utf-8")
 
-        # Use tree-sitter's parser timeout mechanism, when necessary, break the
-        # full reparse into multiple steps. Most of the time, tree-sitter is so
-        # fast that no looping occurs.
+        # Use tree-sitter's parser timeout mechanism to break the full reparse
+        # into multiple steps. Most of the time, tree-sitter is so fast that no
+        # looping occurs.
         parsed_ok = False
         while not parsed_ok:
             parsed_ok = document.reparse(
-                self.PARSE_TIMEOUT_MICROSECONDS, lines=copy_of_text_for_parsing
+                self.PARSE_TIMEOUT_MICROSECONDS, text=copy_of_text_for_parsing
             )
             if not parsed_ok:
                 # Sleeping for zero seconds allows other tasks, I/O, *etc.* to execute,

src/textual/document/_syntax_aware_document.py

@@ -11,9 +11,9 @@
 
 ;; Identifier naming conventions
 ((identifier) @type
- (#lua-match? @type "^[A-Z].*[a-z]"))
+ (#match? @type "^[A-Z].*[a-z]"))
 ((identifier) @constant
- (#lua-match? @constant "^[A-Z][A-Z_0-9]*$"))
+ (#match? @constant "^[A-Z][A-Z_0-9]*$"))
 
 ((attribute
     attribute: (identifier) @field)
@@ -59,12 +59,12 @@
 
 ((call
    function: (identifier) @constructor)
- (#lua-match? @constructor "^[A-Z]"))
+ (#match? @constructor "^[A-Z]"))
 
 ((call
   function: (attribute
               attribute: (identifier) @constructor))
- (#lua-match? @constructor "^[A-Z]"))
+ (#match? @constructor "^[A-Z]"))
 
 ;; Decorators
 

src/textual/tree-sitter/highlights/python.scm

@@ -1208,7 +1208,7 @@ def check_consume_key(self, key: str, character: str | None = None) -> bool:
         # Otherwise we capture all printable keys
         return character is not None and character.isprintable()
 
-    def _handle_syntax_tree_update(self, tree: Tree, line_count: int) -> None:
+    def _handle_syntax_tree_update(self, tree: Tree) -> None:
         """Reflect changes to the syntax tree."""
         self._trigger_repaint()
 
@@ -1693,11 +1693,6 @@ def _prepare_for_repaint(self) -> Collection[Region]:
             return self._do_prepare_for_repaint()
 
     def _do_prepare_for_repaint(self) -> Collection[Region]:
-        # TODO:
-        #   This is being used as a hook to prepare for an imminent screen
-        #   update, which is not the intended use of this method. A proper
-        #   'prepare for screen update' hook.
-
         is_syntax_aware = self.is_syntax_aware
         if is_syntax_aware:
             highlights = self._highlights
@@ -1849,7 +1844,7 @@ def create_select_range() -> range:
                         range_end = min(sel_end, text_length)
                         if not line_text and yy != cursor_y:
                             # Make sure that empty line show up as selected.
-                            line_text = TextReprString("▌")
+                            line_text = TextReprString.create("▌")
                             select_range = range(1, 0, -1)
                         elif sel_start < sel_end:
                             # The selection covers part of this line section.