feat: update core & tests

And fix remaining issues

Author: ObserverOfTime

.github/workflows/ci.yml

@@ -34,16 +34,11 @@ jobs:
         with:
           python-version: ${{matrix.python}}
       - name: Lint
-        run: pipx run ruff check .
+        continue-on-error: true
+        run: pipx run ruff check . --output-format=github
       - name: Build
-        run: pip install -e .
+        run: pip install -v -e .[tests]
         env:
-          CFLAGS: "-O0 -g"
+          CFLAGS: -Wextra -Og -g -fno-omit-frame-pointer
       - name: Test
-        shell: python
-        # run: python -Wignore:::tree_sitter -munittest
-        run: |-
-          try: __import__('tree_sitter').Language(1048576)
-          except RuntimeError as err: print(err)
-        env:
-          PYTHONFAULTHANDLER: 1
+        run: python -munittest -v

.gitmodules

@@ -1,21 +1,3 @@
 [submodule "tree-sitter"]
 	path = tree_sitter/core
 	url = https://github.com/tree-sitter/tree-sitter
-[submodule "tree-sitter-embedded-template"]
-	path = tests/fixtures/tree-sitter-embedded-template
-	url = https://github.com/tree-sitter/tree-sitter-embedded-template
-[submodule "tree-sitter-html"]
-	path = tests/fixtures/tree-sitter-html
-	url = https://github.com/tree-sitter/tree-sitter-html
-[submodule "tree-sitter-javascript"]
-	path = tests/fixtures/tree-sitter-javascript
-	url = https://github.com/tree-sitter/tree-sitter-javascript
-[submodule "tree-sitter-json"]
-	path = tests/fixtures/tree-sitter-json
-	url = https://github.com/tree-sitter/tree-sitter-json
-[submodule "tree-sitter-python"]
-	path = tests/fixtures/tree-sitter-python
-	url = https://github.com/tree-sitter/tree-sitter-python
-[submodule "tree-sitter-rust"]
-	path = tests/fixtures/tree-sitter-rust
-	url = https://github.com/tree-sitter/tree-sitter-rust

docs/conf.py

@@ -54,30 +54,30 @@
 html_favicon = "_static/favicon.png"
 
 
-special_doc = regex('\S*self[^.]+')
+special_doc = regex("\S*self[^.]+")
 
 
 def process_signature(_app, _what, name, _obj, _options, _signature, return_annotation):
-    if name == 'tree_sitter.Language':
-        return '(ptr)', return_annotation
-    if name == 'tree_sitter.Query':
-        return '(language, source)', return_annotation
-    if name == 'tree_sitter.Parser':
-        return '(language, *, included_ranges=None, timeout_micros=None)', return_annotation
-    if name == 'tree_sitter.Range':
-        return '(start_point, end_point, start_byte, end_byte)', return_annotation
+    if name == "tree_sitter.Language":
+        return "(ptr)", return_annotation
+    if name == "tree_sitter.Query":
+        return "(language, source)", return_annotation
+    if name == "tree_sitter.Parser":
+        return "(language, *, included_ranges=None, timeout_micros=None)", return_annotation
+    if name == "tree_sitter.Range":
+        return "(start_point, end_point, start_byte, end_byte)", return_annotation
 
 
 def process_docstring(_app, what, name, _obj, _options, lines):
-    if what == 'data':
+    if what == "data":
         lines.clear()
-    elif what == 'method':
-        if name.endswith('__index__'):
-            lines[0] = 'Converts ``self`` to an integer for use as an index.'
-        elif name.endswith('__') and lines and 'self' in lines[0]:
-            lines[0] = f'Implements ``{special_doc.search(lines[0]).group(0)}``.'
+    elif what == "method":
+        if name.endswith("__index__"):
+            lines[0] = "Converts ``self`` to an integer for use as an index."
+        elif name.endswith("__") and lines and "self" in lines[0]:
+            lines[0] = f"Implements ``{special_doc.search(lines[0]).group(0)}``."
 
 
 def setup(app):
-    app.connect('autodoc-process-signature', process_signature)
-    app.connect('autodoc-process-docstring', process_docstring)
+    app.connect("autodoc-process-signature", process_signature)
+    app.connect("autodoc-process-docstring", process_docstring)

pyproject.toml

@@ -31,6 +31,13 @@ email = "maxbrunsfeld@gmail.com"
 
 [project.optional-dependencies]
 docs = ["sphinx~=7.3", "sphinx-book-theme"]
+tests = [
+    "tree-sitter-html",
+    "tree-sitter-javascript",
+    "tree-sitter-json",
+    "tree-sitter-python",
+    "tree-sitter-rust",
+]
 
 [tool.ruff]
 target-version = "py39"
@@ -39,7 +46,7 @@ indent-width = 4
 extend-exclude = [
     ".github",
     "__pycache__",
-    "tests/fixtures",
+    "setup.py",
     "tree_sitter/core",
 ]
 
@@ -49,6 +56,7 @@ indent-style = "space"
 
 [tool.cibuildwheel]
 build-frontend = "build"
+test-extras = ["tests"]
 test-command = "python -munittest discover -s {project}/tests"
 
 [tool.cibuildwheel.environment]

setup.py

@@ -1,14 +1,12 @@
-"""Py-Tree-sitter"""
-
 from platform import system
 
-from setuptools import Extension, setup
+from setuptools import Extension, setup  # type: ignore
 
 setup(
     packages=["tree_sitter"],
     include_package_data=False,
     package_data={
-        "tree_sitter": ["py.typed", "*.pyi"]
+        "tree_sitter": ["py.typed", "*.pyi"],
     },
     ext_modules=[
         Extension(
@@ -41,8 +39,12 @@
             extra_compile_args=[
                 "-std=c11",
                 "-fvisibility=hidden",
+                "-Wno-cast-function-type",
                 "-Werror=implicit-function-declaration",
-            ] if system() != "Windows" else None
+            ] if system() != "Windows" else [
+                "/std:c11",
+                "/wd4244",
+            ],
         )
-    ]
+    ],
 )

tests/fixtures/tree-sitter-embedded-template

@@ -0,0 +1,86 @@
+from unittest import TestCase
+
+from tree_sitter import Language, Query
+
+import tree_sitter_html
+import tree_sitter_javascript
+import tree_sitter_json
+import tree_sitter_python
+import tree_sitter_rust
+
+
+class TestLanguage(TestCase):
+    def setUp(self):
+        self.html = tree_sitter_html.language()
+        self.javascript = tree_sitter_javascript.language()
+        self.json = tree_sitter_json.language()
+        self.python = tree_sitter_python.language()
+        self.rust = tree_sitter_rust.language()
+
+    def test_init_not_positive(self):
+        self.assertRaises(ValueError, Language, -1)
+
+    def test_init_segv(self):
+        self.assertRaises(RuntimeError, Language, 1024)
+
+    def test_properties(self):
+        lang = Language(self.python)
+        self.assertEqual(lang.version, 14)
+        self.assertEqual(lang.node_kind_count, 274)
+        self.assertEqual(lang.parse_state_count, 2831)
+        self.assertEqual(lang.field_count, 32)
+
+    def test_node_kind_for_id(self):
+        lang = Language(self.json)
+        self.assertEqual(lang.node_kind_for_id(1), "{")
+        self.assertEqual(lang.node_kind_for_id(3), "}")
+
+    def test_id_for_node_kind(self):
+        lang = Language(self.json)
+        self.assertEqual(lang.id_for_node_kind(":", False), 4)
+        self.assertEqual(lang.id_for_node_kind("string", True), 20)
+
+    def test_node_kind_is_named(self):
+        lang = Language(self.json)
+        self.assertFalse(lang.node_kind_is_named(4))
+        self.assertTrue(lang.node_kind_is_named(20))
+
+    def test_node_kind_is_visible(self):
+        lang = Language(self.json)
+        self.assertTrue(lang.node_kind_is_visible(2))
+
+    def test_field_name_for_id(self):
+        lang = Language(self.json)
+        self.assertEqual(lang.field_name_for_id(1), "key")
+        self.assertEqual(lang.field_name_for_id(2), "value")
+
+    def test_field_id_for_name(self):
+        lang = Language(self.json)
+        self.assertEqual(lang.field_id_for_name("key"), 1)
+        self.assertEqual(lang.field_id_for_name("value"), 2)
+
+    def test_next_state(self):
+        lang = Language(self.javascript)
+        self.assertNotEqual(lang.next_state(1, 1), 0)
+
+    def test_lookahead_iterator(self):
+        lang = Language(self.javascript)
+        self.assertIsNotNone(lang.lookahead_iterator(0))
+        self.assertIsNone(lang.lookahead_iterator(9999))
+
+    def test_query(self):
+        lang = Language(self.json)
+        query = lang.query("(string) @string")
+        self.assertIsInstance(query, Query)
+
+    def test_eq(self):
+        self.assertEqual(Language(self.json), Language(self.json))
+        self.assertNotEqual(Language(self.rust), Language(self.html))
+
+    def test_int(self):
+        for name in ["html", "javascript", "json", "python", "rust"]:
+            with self.subTest(language=name):
+                ptr = getattr(self, name)
+                lang = Language(ptr)
+                self.assertEqual(int(lang), ptr)
+                self.assertEqual(hash(lang), ptr)

tests/fixtures/tree-sitter-html

@@ -0,0 +1,43 @@
+from unittest import TestCase
+
+from tree_sitter import Language, Parser
+
+import tree_sitter_rust
+
+
+class TestLookaheadIterator(TestCase):
+    @classmethod
+    def setUpClass(self):
+        self.rust = Language(tree_sitter_rust.language())
+
+    def test_lookahead_iterator(self):
+        parser = Parser(self.rust)
+        cursor = parser.parse(b"struct Stuff{}").walk()
+
+        self.assertEqual(cursor.goto_first_child(), True)  # struct
+        self.assertEqual(cursor.goto_first_child(), True)  # struct keyword
+
+        next_state = cursor.node.next_parse_state
+
+        self.assertNotEqual(next_state, 0)
+        self.assertEqual(
+            next_state, self.rust.next_state(cursor.node.parse_state, cursor.node.grammar_id)
+        )
+        self.assertLess(next_state, self.rust.parse_state_count)
+        self.assertEqual(cursor.goto_next_sibling(), True)  # type_identifier
+        self.assertEqual(next_state, cursor.node.parse_state)
+        self.assertEqual(cursor.node.grammar_name, "identifier")
+        self.assertNotEqual(cursor.node.grammar_id, cursor.node.kind_id)
+
+        expected_symbols = ["//", "/*", "identifier", "line_comment", "block_comment"]
+        lookahead = self.rust.lookahead_iterator(next_state)
+        self.assertEqual(lookahead.language, self.rust)
+        self.assertListEqual(list(lookahead.iter_names()), expected_symbols)
+
+        lookahead.reset_state(next_state)
+        self.assertListEqual(list(lookahead.iter_names()), expected_symbols)
+
+        lookahead.reset_state(next_state, self.rust)
+        self.assertListEqual(
+            list(map(self.rust.node_kind_for_id, list(iter(lookahead)))), expected_symbols
+        )