languages tree-sitter

[jmio23]

I have tried to install tree-sitter-org, but not getting anywhere
This is what I've been trying:

from tree_sitter_languages import get_language
org_language = get_language("org")
org_highlight_query = (Path(__file__).parent "highlights.scm").read_text()

TextArea.register_language(org_language, org_highlight_query)

and
`code_view.language = "org"'

code_view is a TextArea.code_editor, treesitter and syntax are installed, pretty sure the files are where they are meant to be (the error output pointed me right to where I cloned tree-sitter-org too and the file is there (highlights.scm)

I'd tried setting language before using treesitter languages, to "org" but that complained and I cannot find a list of built-in languages anywhere, I don't imagine org is among them but was worth a try

I'm missing something obvious I'm sure, but still too new to python

this is what i get so far, everything else works fine without the code above

Traceback (most recent call last): File "/acs/0.4.1/acs.py", line 17, in <module> org_language = get_language("org") ^^^^^^^^^^^^^^^^^^^ File "tree_sitter_languages/core.pyx", line 14, in tree_sitter_\ languages.core.get_language File "/acs/py//lib/python3.12/site-packages/tree_sitter/__init_\ _.py", line 132, in __init__ language_function: Callable[[], c_void_p] = getattr(self.lib,\ "tree_sitter_%s" % name) ^^^^^^^^^^^^^^^^^\ ^^^^^^^^^^^^^^^^^^^^^^^^^ File "/usr/lib/python3.12/ctypes/__init__.py", line 392, in __g\ etattr__ func = self.__getitem__(name) ^^^^^^^^^^^^^^^^^^^^^^ File "/usr/lib/python3.12/ctypes/__init__.py", line 397, in __g\ etitem__ func = self._FuncPtr((name_or_ordinal, self)) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ AttributeError: /acs/py/lib/python3.12/site-packages/tree_sitter_\ languages/languages.so: undefined symbol: tree_sitter_org

[merriam]

First, you will find that TreeSitter on github appears to be specifically borked, probably on purpose. No one screws up that bad normally.

For Textual, you should use poetry install -E tree-sitter. Then you can verify that TextArea, the only widget using TreeSitter, should work. Note the CodeBrowser example uses Pygments, and does not require tree sitter.

There are three components to tree-sitter:

• The TreeSitter source code
• The *.scm language grammer files. That is, the files that understand how to parse up a language like JavaScript or Python.
• The Python Bindings, meaning a Python interface to the TreeSitter source code. It will define Python bindings like "Parse()".

If you can run a command like tree-sitter parse test.py command from your MacOS command line, then you have the first two parts installed.

You should ignore the current github repository for py-treesitter. It directs you to a non-existent PyPi library and the maintainer just hides all the bug filings of people complaining.

You will find the "query" system truly stupid and un-pythonic. You might like this terrible hack for ideas.

"""
Helper script to help document all widgets.

"""
from __future__ import annotations

from pathlib import Path
from typing import TYPE_CHECKING  # True for static, False at runtime...

import tree_sitter
from tree_sitter import Language, Parser, Tree
from tree_sitter.binding import Query
from tree_sitter_languages import get_language, get_parser
from textual.document import _syntax_aware_document as syn

if TYPE_CHECKING:  # totally not understanding this, imports frozen data class
    from textual.binding import Binding  # which just a frozen dataclass, and only during type checking?

Cursor = tree_sitter.TreeCursor


def grab_text(start_point, end_point):
    # Things unknown.  do start_point columns go by bytes or chars, its unknown.
    start_line, start_col = start_point
    end_line, end_col = end_point

    lines = syn.the_doc._lines
    newline = syn.the_doc.newline

    if start_line >= len(lines):
        return ""
    if start_line == end_line:
        return lines[start_line][start_col:end_col]
    else:
        text = lines[start_line][start_col:] + newline
        for line_number in range(start_line + 1, end_line):
            text += lines[line_number] + newline
        text += lines[end_line][:end_col]
        return text

    # if start_line != end_line:
    #     text = syn.the_doc._read_callable(0, start_point)
    #     for n in range(start_line, end_line):
    #         text += syn.the_doc._read_callable(0, (n, 0))
    #     text += syn.the_doc._read_callable(0, (end_line, 0))[:end_col].strip()
    #     return text
    #     # raise NotImplementedError()
    # s = syn.the_doc._read_callable(0, start_point)[:end_col - start_col].strip()
    # return s.decode("utf8")


def grab_cursor_text(c: Cursor):
    return grab_text(c.node.start_point, c.node.end_point)


def print_cursor(cursor: Cursor, indent=0) -> None:
    i = ' ' * indent
    c = cursor
    n = cursor.node
    print(f"{i} Cursor: {c.depth=} {c.field_id=} {c.field_name=}   ")
    byte_len = n.byte_range[1] - n.byte_range[0]
    start_line, start_col = n.start_point
    end_line, end_col = n.end_point
    line_note = '**MULTILINE**' if start_line != end_line else ''
    print(f"{i} Node:  {n.type=}  line={n.start_point[0]} {line_note}, bytes={n.byte_range[1] - n.start_point[0]}")
    print(f"{i}        {n.id=} {n.text=}")
    print(f"{i}        {n.sexp()[:20]=}")
    print(f"{i}        {n.is_named=} {n.grammar_id=}, {n.grammar_name=} {n.type=}")
    print(f"{i}        {n.byte_range=} {n.end_byte=}")
    print(f"{i}        {n.start_point=} {n.end_point=}  ")
    print(f"{i}        {n.child_count=} {n.descendant_count=} ")
    print(f"{i}        {n.has_error=} {n.is_error=} {n.is_missing=}")

    if byte_len < 8000:
        # read text from start point, to end point, which may be a portion or multiple lines.
        if start_line == end_line:
            part = syn.the_doc._read_callable(0, n.start_point)[:end_col - start_col].strip()
            print(f"{i}        {part} ")
        else:
            pass
    print()
    pass


def recursive_print_cursor(cursor: tree_sitter.TreeCursor, indent=0) -> None:
    print_cursor(cursor, indent)
    is_child = cursor.goto_first_child()
    while is_child:
        recursive_print_cursor(cursor.copy(), indent + 2)
        is_child = cursor.goto_next_sibling()


Printer_On = True


def walk(c: Cursor, match_type=None, match_text=None, max_depth=1, current_depth=0):
    print(f"{current_depth}: walk {c.node.type}, {match_type=} {match_text=}")
    c = c.copy()
    is_child = c.goto_first_child()
    while is_child:
        if Printer_On:
            print(f"   {c.node.type} {c.node.start_point} to {c.node.end_point}  {c.node.child_count}/{grab_cursor_text(c)[:60]}")
            # print_cursor(c)
        ty = match_type is None or c.node.type == match_type
        te = match_text is None or match_text == grab_cursor_text(c)
        if ty and te:
            yield c.copy()
        if c.node.child_count and current_depth + 1 < max_depth:
            yield from walk(c, match_type, match_text, max_depth, current_depth + 1)
        is_child = c.goto_next_sibling()
    print(f"{current_depth} done")


def first(c: Cursor, match_type=None, match_text=None, max_depth=1):
    try:
        return next(walk(c, match_type=match_type, match_text=match_text, max_depth=max_depth))
    except StopIteration:
        return None




def extract_default_css(cursor: tree_sitter.TreeCursor, indent=0) -> None:
    for class_def in walk(cursor, match_type="class_definition", max_depth=2):
        class_name = grab_cursor_text(first(class_def, match_type="identifier", max_depth=3))
        print(f"Found class {class_name}")
        if css := first(class_def, match_type="identifier", match_text="DEFAULT_CSS", max_depth=5):
            css.goto_parent()
            if content_node := first(css, match_type="string_content", max_depth=4):
                content = grab_cursor_text(content_node)
                print(f"Found content {content}")
                return content
                pass


def main() -> None:
    filepath = Path("~/p/textual/src/textual/widgets/_data_table.py").expanduser().resolve()
    text = filepath.read_text(encoding='utf-8')
    # language = get_language("python")
    # parser = Parser()
    # parser.set_language(language)

    doc = syn.SyntaxAwareDocument(text, 'python')
    syn.the_doc = doc
    tree = doc._syntax_tree
    cursor = tree.walk()
    # recursive_print_cursor(cursor)
    contents = extract_default_css(cursor)

    contents = Path("~/p/textual/tools/y.css").expanduser().resolve().read_text(encoding='utf-8')
    print_css(contents)

    pass


def print_css(contents):
    doc = syn.SyntaxAwareDocument(contents, 'css')
    syn.the_doc = doc
    tree = doc._syntax_tree
    cursor = tree.walk()
    list(walk(cursor, max_depth=3))


if __name__ == "__main__":
        main()

[jmio23]

thanks for the reply helps a lot; I installed tree sitter via pip, cloned a tree-sitter-org language from github, have resisted adding poetry so far, virtual environments for libraries etc new to me for programming, I'll give the hack a go

it sounds like i'm going in the wrong direction though, I need to read up on pygments!