Support Astroid v3

.github/workflows/build-and-test.yml

@@ -26,6 +26,22 @@ jobs:
           - pypy-3.6
           - pypy-3.7
           - pypy-3.8
+        astroid-version:
+          - ''
+        include:
+          # Test recent Python on Astroid v2 too (the above natural tests will
+          # pick up Astroid v3 for these Python versions)
+          - python-version: '3.8'
+            astroid-version: '<3'
+          - python-version: '3.9'
+            astroid-version: '<3'
+          - python-version: '3.10'
+            astroid-version: '<3'
+          - python-version: '3.11'
+            astroid-version: '<3'
+          - python-version: '3.12'
+            astroid-version: '<3'
+
     env:
       ASTTOKENS_SLOW_TESTS: 1
       COVERALLS_PARALLEL: true
@@ -53,7 +69,7 @@ jobs:
       - name: Install dependencies
         run: |
           pip install --upgrade coveralls pytest setuptools setuptools_scm pep517
-          pip install .[test]
+          pip install .[test] 'astroid${{ matrix.astroid-version }}'
 
       - name: Mypy testing
         run: |

asttokens/astroid_compat.py

@@ -3,12 +3,16 @@
 
   # astroid_node_classes should be whichever module has the NodeNG class
   from astroid.nodes import NodeNG
+  from astroid.nodes import BaseContainer
 except Exception:
   try:
     from astroid import node_classes as astroid_node_classes
     from astroid.node_classes import NodeNG
+    from astroid.node_classes import _BaseContainer as BaseContainer
   except Exception:  # pragma: no cover
     astroid_node_classes = None
     NodeNG = None
+    BaseContainer = None
 
-__all__ = ["astroid_node_classes", "NodeNG"]
+
+__all__ = ["astroid_node_classes", "NodeNG", "BaseContainer"]

asttokens/mark_tokens.py

@@ -24,7 +24,7 @@
 from . import util
 from .asttokens import ASTTokens
 from .util import AstConstant
-from .astroid_compat import astroid_node_classes as nc
+from .astroid_compat import astroid_node_classes as nc, BaseContainer as AstroidBaseContainer
 
 if TYPE_CHECKING:
   from .util import AstNode
@@ -229,7 +229,7 @@ def handle_def(self, node, first_token, last_token):
     # type: (AstNode, util.Token, util.Token) -> Tuple[util.Token, util.Token]
     # With astroid, nodes that start with a doc-string can have an empty body, in which case we
     # need to adjust the last token to include the doc string.
-    if not node.body and getattr(node, 'doc', None): # type: ignore[union-attr]
+    if not node.body and (getattr(node, 'doc_node', None) or getattr(node, 'doc', None)): # type: ignore[union-attr]
       last_token = self._code.find_token(last_token, token.STRING)
 
     # Include @ from decorator
@@ -312,7 +312,7 @@ def handle_bare_tuple(self, node, first_token, last_token):
     # In Python3.8 parsed tuples include parentheses when present.
     def handle_tuple_nonempty(self, node, first_token, last_token):
       # type: (AstNode, util.Token, util.Token) -> Tuple[util.Token, util.Token]
-      assert isinstance(node, ast.Tuple) or isinstance(node, nc._BaseContainer)
+      assert isinstance(node, ast.Tuple) or isinstance(node, AstroidBaseContainer)
       # It's a bare tuple if the first token belongs to the first child. The first child may
       # include extraneous parentheses (which don't create new nodes), so account for those too.
       child = node.elts[0]
@@ -331,7 +331,7 @@ def handle_tuple_nonempty(self, node, first_token, last_token):
 
   def visit_tuple(self, node, first_token, last_token):
     # type: (AstNode, util.Token, util.Token) -> Tuple[util.Token, util.Token]
-    assert isinstance(node, ast.Tuple) or isinstance(node, nc._BaseContainer)
+    assert isinstance(node, ast.Tuple) or isinstance(node, AstroidBaseContainer)
     if not node.elts:
       # An empty tuple is just "()", and we need no further info.
       return (first_token, last_token)

setup.cfg

@@ -42,10 +42,10 @@ setup_requires = setuptools>=44; setuptools_scm[toml]>=3.4.3
 [options.extras_require]
 astroid =
     astroid >=1, <2; python_version < "3"
-    astroid >=2, <3; python_version >= "3"
+    astroid >=2, <4; python_version >= "3"
 test =
     astroid >=1, <2; python_version < "3"
-    astroid >=2, <3; python_version >= "3"
+    astroid >=2, <4; python_version >= "3"
     pytest
 
 [options.package_data]

tests/test_astroid.py

@@ -11,6 +11,7 @@
 class TestAstroid(test_mark_tokens.TestMarkTokens):
 
   is_astroid_test = True
+  astroid_version = int(astroid.__version__.split('.')[0])
   module = astroid
 
   nodes_classes = astroid_node_classes.NodeNG
@@ -27,9 +28,12 @@ def iter_fields(node):
 
     Similar to ast.iter_fields, but for astroid and ignores context
     """
-    for field in node._astroid_fields + node._other_fields:
+    fields = node._astroid_fields + node._other_fields
+    for field in fields:
       if field == 'ctx':
         continue
+      if field == 'doc' and 'doc_node' in fields:
+        continue
       yield field, getattr(node, field)
 
   @staticmethod

tests/test_mark_tokens.py

@@ -32,6 +32,7 @@ class TestMarkTokens(unittest.TestCase):
   # the `astroid` library. The latter derives TestAstroid class from TestMarkTokens. For checks
   # that differ between them, .is_astroid_test allows to distinguish.
   is_astroid_test = False
+  astroid_version = None  # type: int | None
   module = ast
 
   def create_mark_checker(self, source, verify=True):
@@ -140,9 +141,14 @@ def verify_fixture_file(self, path):
     tested_nodes = m.verify_all_nodes(self)
 
     exp_index = (0 if six.PY2 else 1) + (3 if self.is_astroid_test else 0)
-    # For ast on Python 3.9, slices are expressions, we handle them and test them.
-    if not self.is_astroid_test and issubclass(ast.Slice, ast.expr):
-      exp_index += 1
+    if not self.is_astroid_test:
+      # For ast on Python 3.9, slices are expressions, we handle them and test them.
+      if issubclass(ast.Slice, ast.expr):
+        exp_index += 1
+    else:
+      # Astroid v3 has some changes from v2
+      if self.astroid_version == 3:
+        exp_index += 1
     exp_tested_nodes = self.expect_tested_nodes[path][exp_index]
     self.assertEqual(tested_nodes, exp_tested_nodes)
 
@@ -151,20 +157,20 @@ def verify_fixture_file(self, path):
   # change reduces the count by a lot, it's a red flag that the test is now covering fewer nodes.
   expect_tested_nodes = {
     #                                   AST                  | Astroid
-    #                                   Py2   Py3  Py3+slice | Py2   Py3
-    'astroid/__init__.py':            ( 4,    4,   4,          4,    4,   ),
-    'astroid/absimport.py':           ( 4,    3,   3,          4,    3,   ),
-    'astroid/all.py':                 ( 21,   23,  23,         21,   23,  ),
-    'astroid/clientmodule_test.py':   ( 75,   67,  67,         69,   69,  ),
-    'astroid/descriptor_crash.py':    ( 30,   28,  28,         30,   30,  ),
-    'astroid/email.py':               ( 3,    3,   3,          1,    1,   ),
-    'astroid/format.py':              ( 64,   61,  61,         62,   62,  ),
-    'astroid/module.py':              ( 185,  174, 174,        171,  171, ),
-    'astroid/module2.py':             ( 248,  253, 255,        240,  253, ),
-    'astroid/noendingnewline.py':     ( 57,   59,  59,         57,   63,  ),
-    'astroid/notall.py':              ( 15,   17,  17,         15,   17,  ),
-    'astroid/recursion.py':           ( 6,    6,   6,          4,    4,   ),
-    'astroid/suppliermodule_test.py': ( 20,   17,  17,         18,   18,  ),
+    #                                   Py2   Py3  Py3+slice | Py2   Py3+v2 Py3+v2
+    'astroid/__init__.py':            ( 4,    4,   4,          4,    4,     4,   ),
+    'astroid/absimport.py':           ( 4,    3,   3,          4,    3,     3,   ),
+    'astroid/all.py':                 ( 21,   23,  23,         21,   23,    23,  ),
+    'astroid/clientmodule_test.py':   ( 75,   67,  67,         69,   69,    69,  ),
+    'astroid/descriptor_crash.py':    ( 30,   28,  28,         30,   30,    30,  ),
+    'astroid/email.py':               ( 3,    3,   3,          1,    1,     1,   ),
+    'astroid/format.py':              ( 64,   61,  61,         62,   62,    62,  ),
+    'astroid/module.py':              ( 185,  174, 174,        171,  171,   173, ),
+    'astroid/module2.py':             ( 248,  253, 255,        240,  253,   253, ),
+    'astroid/noendingnewline.py':     ( 57,   59,  59,         57,   63,    63,  ),
+    'astroid/notall.py':              ( 15,   17,  17,         15,   17,    17,  ),
+    'astroid/recursion.py':           ( 6,    6,   6,          4,    4,     4,   ),
+    'astroid/suppliermodule_test.py': ( 20,   17,  17,         18,   18,    18,  ),
   }
 
   # This set of methods runs verifications for the variety of syntax constructs used in the