From 397715dce6ead75f0a938e6824cfea7894199e52 Mon Sep 17 00:00:00 2001 From: Oleh Prypin Date: Fri, 11 Oct 2024 06:17:46 -0700 Subject: [PATCH] Stop exposing fake builtins `generator`, `asyncgenerator`, `coroutine` Update type annotations for the `throw` method of `Generator` and others. Previously these type annotations happened to be unused, so it was not noticeable that they are incomplete. So, make them more similar to typeshed/typing.pyi. PiperOrigin-RevId: 684814282 --- pytype/abstract/_interpreter_function.py | 4 ++-- pytype/convert.py | 6 ++++-- pytype/pyi/parser.py | 2 +- pytype/pytd/pep484.py | 13 +------------ pytype/pytd/printer.py | 16 ++-------------- pytype/stubs/builtins/builtins.pytd | 19 ------------------- pytype/stubs/builtins/typing.pytd | 17 ++++++++++++++--- pytype/tests/test_coroutine.py | 2 +- pytype/tests/test_generators1.py | 4 ++-- pytype/tests/test_generators2.py | 8 ++++---- pytype/tests/test_stdlib2.py | 2 +- pytype/vm.py | 6 +++--- 12 files changed, 35 insertions(+), 64 deletions(-) diff --git a/pytype/abstract/_interpreter_function.py b/pytype/abstract/_interpreter_function.py index ac0e12767..008757218 100644 --- a/pytype/abstract/_interpreter_function.py +++ b/pytype/abstract/_interpreter_function.py @@ -41,12 +41,12 @@ def _matches_generator_helper(type_obj, allowed_types): def _matches_generator(type_obj): - allowed_types = ("generator", "Iterable", "Iterator") + allowed_types = ("Generator", "Iterable", "Iterator") return _matches_generator_helper(type_obj, allowed_types) def _matches_async_generator(type_obj): - allowed_types = ("asyncgenerator", "AsyncIterable", "AsyncIterator") + allowed_types = ("AsyncGenerator", "AsyncIterable", "AsyncIterator") return _matches_generator_helper(type_obj, allowed_types) diff --git a/pytype/convert.py b/pytype/convert.py index 8c3bc3754..08f198045 100644 --- a/pytype/convert.py +++ b/pytype/convert.py @@ -163,11 +163,13 @@ def _type_to_name(self, t): elif t is IteratorType: return "builtins", "object" elif t is CoroutineType: - return "builtins", "coroutine" + return "typing", "Coroutine" elif t is AwaitableType: return "typing", "Awaitable" elif t is AsyncGeneratorType: - return "builtins", "asyncgenerator" + return "typing", "AsyncGenerator" + elif t is types.GeneratorType: + return "typing", "Generator" else: return "builtins", t.__name__ diff --git a/pytype/pyi/parser.py b/pytype/pyi/parser.py index 1e7e99d19..ebb6291f2 100644 --- a/pytype/pyi/parser.py +++ b/pytype/pyi/parser.py @@ -395,7 +395,7 @@ def _extract_function_properties(self, node): abstract = True elif self.defs.matches_type( d.name, - ("builtins.coroutine", "asyncio.coroutine", "coroutines.coroutine"), + ("typing.Coroutine", "asyncio.coroutine", "coroutines.coroutine"), ): coroutine = True elif self.defs.matches_type(d.name, "typing.final"): diff --git a/pytype/pytd/pep484.py b/pytype/pytd/pep484.py index fb32c18d1..bed33d994 100644 --- a/pytype/pytd/pep484.py +++ b/pytype/pytd/pep484.py @@ -58,17 +58,6 @@ ] -# These lowercase names are used inside pytype as if they're builtins, but they -# are actually not real at all. In fact, pytype accepts literally writing -# `generator[int,...]` in type annotations even though there's no such type. -# TODO(b/372205529): Remove this implementation detail. -PYTYPE_SPECIFIC_FAKE_BUILTINS = { - "generator": "Generator", - "coroutine": "Coroutine", - "asyncgenerator": "AsyncGenerator", -} - - # The PEP 484 definition of built-in types. # E.g. "typing.List" is used to represent the "list" type. BUILTIN_TO_TYPING = { @@ -81,7 +70,7 @@ "FrozenSet", "Type", ] -} | PYTYPE_SPECIFIC_FAKE_BUILTINS +} TYPING_TO_BUILTIN = {v: k for k, v in BUILTIN_TO_TYPING.items()} diff --git a/pytype/pytd/printer.py b/pytype/pytd/printer.py index 21f07f184..3130652bc 100644 --- a/pytype/pytd/printer.py +++ b/pytype/pytd/printer.py @@ -754,13 +754,6 @@ def VisitParamSpecKwargs(self, node): def VisitModule(self, node): return "module" - def MaybeCapitalize(self, name): - """Capitalize a generic type, if necessary.""" - if name in pep484.PYTYPE_SPECIFIC_FAKE_BUILTINS: - return self._FromTyping(pep484.PYTYPE_SPECIFIC_FAKE_BUILTINS[name]) - else: - return name - def VisitGenericType(self, node): """Convert a generic type to a string.""" parameters = node.parameters @@ -776,15 +769,10 @@ def VisitGenericType(self, node): else: assert isinstance(param, (pytd.NothingType, pytd.TypeParameter)), param parameters = ("...",) + parameters[1:] - return ( - self.MaybeCapitalize(node.base_type) - + "[" - + ", ".join(str(p) for p in parameters) - + "]" - ) + return node.base_type + "[" + ", ".join(str(p) for p in parameters) + "]" def VisitCallableType(self, node): - typ = self.MaybeCapitalize(node.base_type) + typ = node.base_type if len(node.args) == 1 and node.args[0] in self._paramspec_names: return f"{typ}[{node.args[0]}, {node.ret}]" elif node.args and "Concatenate" in node.args[0]: diff --git a/pytype/stubs/builtins/builtins.pytd b/pytype/stubs/builtins/builtins.pytd index b1e556cda..8d8e5df63 100644 --- a/pytype/stubs/builtins/builtins.pytd +++ b/pytype/stubs/builtins/builtins.pytd @@ -953,25 +953,6 @@ class complex(SupportsAbs, SupportsInt, SupportsFloat, SupportsComplex): def __truediv__(self, y: Union[int, float, complex]) -> complex: ... def conjugate(self) -> complex: ... -class generator(Generator[_T, _T2, _V]): - __slots__ = [] - def __iter__(self) -> generator[_T, _T2, _V]: ... - def __next__(self) -> _T: ... - def send(self, value: _T2) -> _T: ... - def close(self) -> NoneType: ... - -class coroutine(Coroutine[_T, _T2, _V]): - __slots__ = [] - def close(self) -> NoneType: ... - def send(self, value: _T2) -> _T: ... - -class asyncgenerator(AsyncGenerator[_T, _T2]): - __slots__ = [] - def __aiter__(self) -> asyncgenerator[_T, _T2]: ... - def __anext__(self) -> coroutine[Any, Any, _T]: ... - def asend(self, value: _T2) -> coroutine[Any, Any, _T]: ... - def aclose(self) -> coroutine[Any, Any, None]: ... - class instancemethod(object): __slots__ = [] def __init__(self, function: Union[Callable, instancemethod], instance, cls) -> NoneType: ... diff --git a/pytype/stubs/builtins/typing.pytd b/pytype/stubs/builtins/typing.pytd index c4e821305..d0ca51d71 100644 --- a/pytype/stubs/builtins/typing.pytd +++ b/pytype/stubs/builtins/typing.pytd @@ -356,7 +356,10 @@ class Generator(Iterator[_T], Generic[_T, _T2, _V]): gi_yieldfrom: Optional[Generator[Any, Any, Any]] def __next__(self) -> _T: ... def send(self, value: _T2) -> _T: ... - def throw(self, typ: BaseException, val = ..., tb = ...) -> _T: ... + @overload + def throw(self, typ: type[BaseException], val: BaseException | object = None, tb = None) -> _T: ... + @overload + def throw(self, typ: BaseException, val: None = None, tb = None) -> _T: ... def close(self) -> None: ... @@ -539,8 +542,12 @@ class Coroutine(Awaitable[_V], Generic[_T, _T2, _V]): __slots__ = [] @abstractmethod def send(self, value: _T2) -> _T: ... + @overload + @abstractmethod + def throw(self, typ: type[BaseException], val: BaseException | object = None, tb = None) -> _T: ... + @overload @abstractmethod - def throw(self, typ: Type[BaseException], val = ..., tb = ...) -> _T: ... + def throw(self, typ: BaseException, val: None = None, tb = None) -> _T: ... @abstractmethod def close(self) -> None: ... @@ -559,8 +566,12 @@ class AsyncGenerator(AsyncIterator[_T], Generic[_T, _T2]): def __anext__(self) -> Coroutine[Any, Any, _T]: ... @abstractmethod def asend(self, value: _T2) -> Coroutine[Any, Any, _T]: ... + @overload + @abstractmethod + def athrow(self, typ: type[BaseException], val: BaseException | object = None, tb = None) -> Coroutine[Any, Any, _T]: ... + @overload @abstractmethod - def athrow(self, typ: Type[BaseException], val = ..., tb = ...) -> Coroutine[Any, Any, _T]: ... + def athrow(self, typ: BaseException, val: None = None, tb = None) -> Coroutine[Any, Any, _T]: ... @abstractmethod def aclose(self) -> Coroutine[Any, Any, None]: ... diff --git a/pytype/tests/test_coroutine.py b/pytype/tests/test_coroutine.py index cadf8961e..4143e7279 100644 --- a/pytype/tests/test_coroutine.py +++ b/pytype/tests/test_coroutine.py @@ -577,7 +577,7 @@ async def main(): """ import asyncio from typing import Any, Coroutine - def worker(queue) -> coroutine: ... + def worker(queue) -> Coroutine: ... def main() -> Coroutine[Any, Any, None]: ... """, ) diff --git a/pytype/tests/test_generators1.py b/pytype/tests/test_generators1.py index 485cc956c..c87ff909b 100644 --- a/pytype/tests/test_generators1.py +++ b/pytype/tests/test_generators1.py @@ -64,7 +64,7 @@ def bar(self): for x in __any_object__: return x def __iter__(self): - return generator() + return (i for i in range(5)) """) self.assertTypesMatchPytd( ty, @@ -72,7 +72,7 @@ def __iter__(self): from typing import Any, Generator class Foo: def bar(self) -> Any: ... - def __iter__(self) -> Generator[nothing, nothing, nothing]: ... + def __iter__(self) -> Generator[int, Any, None]: ... """, ) diff --git a/pytype/tests/test_generators2.py b/pytype/tests/test_generators2.py index 2f7deccb4..4ea1c03d1 100644 --- a/pytype/tests/test_generators2.py +++ b/pytype/tests/test_generators2.py @@ -10,7 +10,7 @@ class GeneratorBasicTest(test_base.BaseTest): def test_return_before_yield(self): self.Check(""" from typing import Generator - def f() -> generator: + def f() -> Generator: if __random__: return yield 5 @@ -116,8 +116,8 @@ def func3() -> Generator[int]: # invalid-annotation[e2] self.assertErrorSequences( errors, { - "e1": ["generator[int, int]", "generator[_T, _T2, _V]", "3", "2"], - "e2": ["generator[int]", "generator[_T, _T2, _V]", "3", "1"], + "e1": ["Generator[int, int]", "Generator[_T, _T2, _V]", "3", "2"], + "e2": ["Generator[int]", "Generator[_T, _T2, _V]", "3", "1"], }, ) @@ -125,7 +125,7 @@ def test_hidden_fields(self): self.Check(""" from typing import Generator from types import GeneratorType - a: generator = __any_object__ + a: Generator = __any_object__ a.gi_code a.gi_frame a.gi_running diff --git a/pytype/tests/test_stdlib2.py b/pytype/tests/test_stdlib2.py index f757d83bd..eb1794103 100644 --- a/pytype/tests/test_stdlib2.py +++ b/pytype/tests/test_stdlib2.py @@ -160,7 +160,7 @@ def test_collections_collection(self): def test_collections_generator(self): self._testCollectionsObject( - "Generator", "i for i in range(42)", "42", r"generator.*int" + "Generator", "i for i in range(42)", "42", r"Generator.*int" ) def test_collections_reversible(self): diff --git a/pytype/vm.py b/pytype/vm.py index a204de2c7..baa8d752a 100644 --- a/pytype/vm.py +++ b/pytype/vm.py @@ -3107,7 +3107,7 @@ def byte_GET_YIELD_FROM_ITER(self, state, op): unchanged = self.ctx.program.NewVariable() state, tos = state.pop() for b in tos.bindings: - if b.data.full_name in ("builtins.generator", "builtins.coroutine"): + if b.data.full_name in ("typing.Generator", "typing.Coroutine"): unchanged.PasteBinding(b) else: get_iter.PasteBinding(b) @@ -3250,7 +3250,7 @@ def byte_GET_AWAITABLE(self, state, op): def _get_generator_yield(self, node, generator_var): yield_var = self.frame.yield_variable.AssignToNewVariable(node) for generator in generator_var.data: - if generator.full_name == "builtins.generator": + if generator.full_name == "typing.Generator": yield_value = generator.get_instance_type_parameter(abstract_utils.T) yield_var.PasteVariable(yield_value, node) return yield_var @@ -3272,7 +3272,7 @@ def _get_generator_return(self, node, generator_var): generator.cls, (abstract.ParameterizedClass, abstract.PyTDClass) ) and generator.cls.full_name - in ("typing.Awaitable", "builtins.coroutine", "builtins.generator") + in ("typing.Awaitable", "typing.Coroutine", "typing.Generator") ): if generator.cls.full_name == "typing.Awaitable": ret = generator.get_instance_type_parameter(abstract_utils.T)