diff --git a/setup.cfg b/setup.cfg index 6bd1da1..08fe486 100644 --- a/setup.cfg +++ b/setup.cfg @@ -48,6 +48,7 @@ test = astroid >=2, <4; python_version >= "3" pytest pytest-xdist + parameterized [options.package_data] asttokens = py.typed diff --git a/tests/test_mark_tokens.py b/tests/test_mark_tokens.py index 1ba01f2..1f27623 100644 --- a/tests/test_mark_tokens.py +++ b/tests/test_mark_tokens.py @@ -5,13 +5,12 @@ import inspect import io import os +from parameterized import parameterized import re import sys import textwrap import token -from typing import List import unittest -from time import time import astroid import six @@ -630,7 +629,15 @@ def test_complex_slice_and_parens(self): self.create_mark_checker(source) if six.PY3: - def test_sys_modules(self): + def _parameterize_sys_modules(): + modules = list(sys.modules.keys()) + if not os.environ.get('ASTTOKENS_SLOW_TESTS'): + modules = modules[:20] + + return parameterized.expand([(x,) for x in modules]) + + @_parameterize_sys_modules() + def test_sys_modules(self, module_name): """ Verify all nodes on source files obtained from sys.modules. This can take a long time as there are many modules, @@ -639,51 +646,42 @@ def test_sys_modules(self): """ from .test_astroid import AstroidTreeException - modules = list(sys.modules.values()) - if not os.environ.get('ASTTOKENS_SLOW_TESTS'): - modules = modules[:20] + module = sys.modules[module_name] + + try: + filename = inspect.getsourcefile(module) + except Exception: # some modules raise weird errors + self.skipTest("Failed to get source file") + + if not filename: + self.skipTest("No source file for module") + + filename = os.path.abspath(filename) + print(filename) + try: + with io.open(filename) as f: + source = f.read() + except OSError: + self.skipTest("Error reading source file") + + if self.is_astroid_test and ( + # Astroid fails with a syntax error if a type comment is on its own line + re.search(r'^\s*# type: ', source, re.MULTILINE) + ): + self.skipTest("Skipping potential Astroid syntax error") + + try: + self.create_mark_checker(source) + except AstroidTreeException: + # Astroid sometimes fails with errors like: + # AttributeError: 'TreeRebuilder' object has no attribute 'visit_typealias' + # See https://github.com/gristlabs/asttokens/actions/runs/6015907789/job/16318767911?pr=110 + # Should be fixed in the next astroid release: + # https://github.com/pylint-dev/pylint/issues/8782#issuecomment-1669967220 + # Note that this exception is raised before asttokens is even involved, + # it's purely an astroid bug that we can safely ignore. + self.skipTest("Skipping Astroid error") - start = time() - for module in modules: - # Don't let this test (which runs twice) take longer than 13 minutes - # to avoid the travis build time limit of 30 minutes - if time() - start > 13 * 60: - break - - try: - filename = inspect.getsourcefile(module) - except Exception: # some modules raise weird errors - continue - - if not filename: - continue - - filename = os.path.abspath(filename) - print(filename) - try: - with io.open(filename) as f: - source = f.read() - except OSError: - continue - - if self.is_astroid_test and ( - # Astroid fails with a syntax error if a type comment is on its own line - re.search(r'^\s*# type: ', source, re.MULTILINE) - ): - print('Skipping', filename) - continue - - try: - self.create_mark_checker(source) - except AstroidTreeException: - # Astroid sometimes fails with errors like: - # AttributeError: 'TreeRebuilder' object has no attribute 'visit_typealias' - # See https://github.com/gristlabs/asttokens/actions/runs/6015907789/job/16318767911?pr=110 - # Should be fixed in the next astroid release: - # https://github.com/pylint-dev/pylint/issues/8782#issuecomment-1669967220 - # Note that this exception is raised before asttokens is even involved, - # it's purely an astroid bug that we can safely ignore. - continue if six.PY3: def test_dict_merge(self):