Error handling for recursive TypeVar defaults (PEP 696) (#16925)

cdce8p · hamdanal · commit 1df0dd269c7d · 2024-02-20T21:43:14.000+01:00
This PR adds some additional error handling for recursive TypeVar defaults. Open issue for future PRs: - Expanding nested recursive defaults, e.g. `T2 = list[T1 = str]` - Scope binding, especially for TypeAliasTypes Ref: #14851
diff --git a/mypy/messages.py b/mypy/messages.py
@@ -2059,6 +2059,15 @@ def impossible_intersection(
             template.format(formatted_base_class_list, reason), context, code=codes.UNREACHABLE
         )
 
+    def tvar_without_default_type(
+        self, tvar_name: str, last_tvar_name_with_default: str, context: Context
+    ) -> None:
+        self.fail(
+            f'"{tvar_name}" cannot appear after "{last_tvar_name_with_default}" '
+            "in type parameter list because it has no default type",
+            context,
+        )
+
     def report_protocol_problems(
         self,
         subtype: Instance | TupleType | TypedDictType | TypeType | CallableType,
diff --git a/mypy/semanal.py b/mypy/semanal.py
@@ -226,6 +226,7 @@
     SELF_TYPE_NAMES,
     FindTypeVarVisitor,
     TypeAnalyser,
+    TypeVarDefaultTranslator,
     TypeVarLikeList,
     analyze_type_alias,
     check_for_explicit_any,
@@ -252,6 +253,7 @@
     TPDICT_NAMES,
     TYPE_ALIAS_NAMES,
     TYPE_CHECK_ONLY_NAMES,
+    TYPE_VAR_LIKE_NAMES,
     TYPED_NAMEDTUPLE_NAMES,
     AnyType,
     CallableType,
@@ -1953,17 +1955,19 @@ class Foo(Bar, Generic[T]): ...
             defn.removed_base_type_exprs.append(defn.base_type_exprs[i])
             del base_type_exprs[i]
         tvar_defs: list[TypeVarLikeType] = []
+        last_tvar_name_with_default: str | None = None
         for name, tvar_expr in declared_tvars:
-            tvar_expr_default = tvar_expr.default
-            if isinstance(tvar_expr_default, UnboundType):
-                # TODO: - detect out of order and self-referencing TypeVars
-                #       - nested default types, e.g. list[T1]
-                n = self.lookup_qualified(
-                    tvar_expr_default.name, tvar_expr_default, suppress_errors=True
-                )
-                if n is not None and (default := self.tvar_scope.get_binding(n)) is not None:
-                    tvar_expr.default = default
+            tvar_expr.default = tvar_expr.default.accept(
+                TypeVarDefaultTranslator(self, tvar_expr.name, context)
+            )
             tvar_def = self.tvar_scope.bind_new(name, tvar_expr)
+            if last_tvar_name_with_default is not None and not tvar_def.has_default():
+                self.msg.tvar_without_default_type(
+                    tvar_def.name, last_tvar_name_with_default, context
+                )
+                tvar_def.default = AnyType(TypeOfAny.from_error)
+            elif tvar_def.has_default():
+                last_tvar_name_with_default = tvar_def.name
             tvar_defs.append(tvar_def)
         return base_type_exprs, tvar_defs, is_protocol
 
@@ -2857,6 +2861,10 @@ def visit_assignment_stmt(self, s: AssignmentStmt) -> None:
             with self.allow_unbound_tvars_set():
                 s.rvalue.accept(self)
             self.basic_type_applications = old_basic_type_applications
+        elif self.can_possibly_be_typevarlike_declaration(s):
+            # Allow unbound tvars inside TypeVarLike defaults to be evaluated later
+            with self.allow_unbound_tvars_set():
+                s.rvalue.accept(self)
         else:
             s.rvalue.accept(self)
 
@@ -3033,6 +3041,16 @@ def can_possibly_be_type_form(self, s: AssignmentStmt) -> bool:
         # Something that looks like Foo = Bar[Baz, ...]
         return True
 
+    def can_possibly_be_typevarlike_declaration(self, s: AssignmentStmt) -> bool:
+        """Check if r.h.s. can be a TypeVarLike declaration."""
+        if len(s.lvalues) != 1 or not isinstance(s.lvalues[0], NameExpr):
+            return False
+        if not isinstance(s.rvalue, CallExpr) or not isinstance(s.rvalue.callee, NameExpr):
+            return False
+        ref = s.rvalue.callee
+        ref.accept(self)
+        return ref.fullname in TYPE_VAR_LIKE_NAMES
+
     def is_type_ref(self, rv: Expression, bare: bool = False) -> bool:
         """Does this expression refer to a type?
 
@@ -3522,9 +3540,20 @@ def analyze_alias(
         tvar_defs: list[TypeVarLikeType] = []
         namespace = self.qualified_name(name)
         alias_type_vars = found_type_vars if declared_type_vars is None else declared_type_vars
+        last_tvar_name_with_default: str | None = None
         with self.tvar_scope_frame(self.tvar_scope.class_frame(namespace)):
             for name, tvar_expr in alias_type_vars:
+                tvar_expr.default = tvar_expr.default.accept(
+                    TypeVarDefaultTranslator(self, tvar_expr.name, typ)
+                )
                 tvar_def = self.tvar_scope.bind_new(name, tvar_expr)
+                if last_tvar_name_with_default is not None and not tvar_def.has_default():
+                    self.msg.tvar_without_default_type(
+                        tvar_def.name, last_tvar_name_with_default, typ
+                    )
+                    tvar_def.default = AnyType(TypeOfAny.from_error)
+                elif tvar_def.has_default():
+                    last_tvar_name_with_default = tvar_def.name
                 tvar_defs.append(tvar_def)
 
             analyzed, depends_on = analyze_type_alias(
diff --git a/mypy/typeanal.py b/mypy/typeanal.py
@@ -38,7 +38,12 @@
 )
 from mypy.options import Options
 from mypy.plugin import AnalyzeTypeContext, Plugin, TypeAnalyzerPluginInterface
-from mypy.semanal_shared import SemanticAnalyzerCoreInterface, paramspec_args, paramspec_kwargs
+from mypy.semanal_shared import (
+    SemanticAnalyzerCoreInterface,
+    SemanticAnalyzerInterface,
+    paramspec_args,
+    paramspec_kwargs,
+)
 from mypy.state import state
 from mypy.tvar_scope import TypeVarLikeScope
 from mypy.types import (
@@ -2520,3 +2525,32 @@ def process_types(self, types: list[Type] | tuple[Type, ...]) -> None:
         else:
             for t in types:
                 t.accept(self)
+
+
+class TypeVarDefaultTranslator(TrivialSyntheticTypeTranslator):
+    """Type translate visitor that replaces UnboundTypes with in-scope TypeVars."""
+
+    def __init__(
+        self, api: SemanticAnalyzerInterface, tvar_expr_name: str, context: Context
+    ) -> None:
+        self.api = api
+        self.tvar_expr_name = tvar_expr_name
+        self.context = context
+
+    def visit_unbound_type(self, t: UnboundType) -> Type:
+        sym = self.api.lookup_qualified(t.name, t, suppress_errors=True)
+        if sym is not None:
+            if type_var := self.api.tvar_scope.get_binding(sym):
+                return type_var
+            if isinstance(sym.node, TypeVarLikeExpr):
+                self.api.fail(
+                    f'Type parameter "{self.tvar_expr_name}" has a default type '
+                    "that refers to one or more type variables that are out of scope",
+                    self.context,
+                )
+                return AnyType(TypeOfAny.from_error)
+        return super().visit_unbound_type(t)
+
+    def visit_type_alias_type(self, t: TypeAliasType) -> Type:
+        # TypeAliasTypes are analyzed separately already, just return it
+        return t
diff --git a/mypy/types.py b/mypy/types.py
@@ -85,6 +85,15 @@
         TypeVisitor as TypeVisitor,
     )
 
+TYPE_VAR_LIKE_NAMES: Final = (
+    "typing.TypeVar",
+    "typing_extensions.TypeVar",
+    "typing.ParamSpec",
+    "typing_extensions.ParamSpec",
+    "typing.TypeVarTuple",
+    "typing_extensions.TypeVarTuple",
+)
+
 TYPED_NAMEDTUPLE_NAMES: Final = ("typing.NamedTuple", "typing_extensions.NamedTuple")
 
 # Supported names of TypedDict type constructors.
diff --git a/test-data/unit/check-typevar-defaults.test b/test-data/unit/check-typevar-defaults.test
@@ -82,6 +82,74 @@ T3 = TypeVar("T3", int, str, default=bytes)  # E: TypeVar default must be one of
 T4 = TypeVar("T4", int, str, default=Union[int, str])  # E: TypeVar default must be one of the constraint types
 T5 = TypeVar("T5", float, str, default=int)  # E: TypeVar default must be one of the constraint types
 
+[case testTypeVarDefaultsInvalid3]
+from typing import Dict, Generic, TypeVar
+
+T1 = TypeVar("T1")
+T2 = TypeVar("T2", default=T3)  # E: Name "T3" is used before definition
+T3 = TypeVar("T3", default=str)
+T4 = TypeVar("T4", default=T3)
+
+class ClassError1(Generic[T3, T1]): ...  # E: "T1" cannot appear after "T3" in type parameter list because it has no default type
+
+def func_error1(
+    a: ClassError1,
+    b: ClassError1[int],
+    c: ClassError1[int, float],
+) -> None:
+    reveal_type(a)  # N: Revealed type is "__main__.ClassError1[builtins.str, Any]"
+    reveal_type(b)  # N: Revealed type is "__main__.ClassError1[builtins.int, Any]"
+    reveal_type(c)  # N: Revealed type is "__main__.ClassError1[builtins.int, builtins.float]"
+
+    k = ClassError1()
+    reveal_type(k)  # N: Revealed type is "__main__.ClassError1[builtins.str, Any]"
+    l = ClassError1[int]()
+    reveal_type(l)  # N: Revealed type is "__main__.ClassError1[builtins.int, Any]"
+    m = ClassError1[int, float]()
+    reveal_type(m)  # N: Revealed type is "__main__.ClassError1[builtins.int, builtins.float]"
+
+class ClassError2(Generic[T4, T3]): ...  # E: Type parameter "T4" has a default type that refers to one or more type variables that are out of scope
+
+def func_error2(
+    a: ClassError2,
+    b: ClassError2[int],
+    c: ClassError2[int, float],
+) -> None:
+    reveal_type(a)  # N: Revealed type is "__main__.ClassError2[Any, builtins.str]"
+    reveal_type(b)  # N: Revealed type is "__main__.ClassError2[builtins.int, builtins.str]"
+    reveal_type(c)  # N: Revealed type is "__main__.ClassError2[builtins.int, builtins.float]"
+
+    k = ClassError2()
+    reveal_type(k)  # N: Revealed type is "__main__.ClassError2[Any, builtins.str]"
+    l = ClassError2[int]()
+    reveal_type(l)  # N: Revealed type is "__main__.ClassError2[builtins.int, builtins.str]"
+    m = ClassError2[int, float]()
+    reveal_type(m)  # N: Revealed type is "__main__.ClassError2[builtins.int, builtins.float]"
+
+TERR1 = Dict[T3, T1]  # E: "T1" cannot appear after "T3" in type parameter list because it has no default type
+
+def func_error_alias1(
+    a: TERR1,
+    b: TERR1[int],
+    c: TERR1[int, float],
+) -> None:
+    reveal_type(a)  # N: Revealed type is "builtins.dict[builtins.str, Any]"
+    reveal_type(b)  # N: Revealed type is "builtins.dict[builtins.int, Any]"
+    reveal_type(c)  # N: Revealed type is "builtins.dict[builtins.int, builtins.float]"
+
+TERR2 = Dict[T4, T3]  # TODO should be an error  \
+                      # Type parameter "T4" has a default type that refers to one or more type variables that are out of scope
+
+def func_error_alias2(
+    a: TERR2,
+    b: TERR2[int],
+    c: TERR2[int, float],
+) -> None:
+    reveal_type(a)  # N: Revealed type is "builtins.dict[Any, builtins.str]"
+    reveal_type(b)  # N: Revealed type is "builtins.dict[builtins.int, builtins.str]"
+    reveal_type(c)  # N: Revealed type is "builtins.dict[builtins.int, builtins.float]"
+[builtins fixtures/dict.pyi]
+
 [case testTypeVarDefaultsFunctions]
 from typing import TypeVar, ParamSpec, List, Union, Callable, Tuple
 from typing_extensions import TypeVarTuple, Unpack
@@ -351,11 +419,12 @@ def func_c4(
 
 [case testTypeVarDefaultsClassRecursive1]
 # flags: --disallow-any-generics
-from typing import Generic, TypeVar
+from typing import Generic, TypeVar, List
 
 T1 = TypeVar("T1", default=str)
 T2 = TypeVar("T2", default=T1)
 T3 = TypeVar("T3", default=T2)
+T4 = TypeVar("T4", default=List[T1])
 
 class ClassD1(Generic[T1, T2]): ...
 
@@ -397,12 +466,30 @@ def func_d2(
     n = ClassD2[int, float, str]()
     reveal_type(n)  # N: Revealed type is "__main__.ClassD2[builtins.int, builtins.float, builtins.str]"
 
+class ClassD3(Generic[T1, T4]): ...
+
+def func_d3(
+    a: ClassD3,
+    b: ClassD3[int],
+    c: ClassD3[int, float],
+) -> None:
+    reveal_type(a)  # N: Revealed type is "__main__.ClassD3[builtins.str, builtins.list[builtins.str]]"
+    reveal_type(b)  # N: Revealed type is "__main__.ClassD3[builtins.int, builtins.list[builtins.int]]"
+    reveal_type(c)  # N: Revealed type is "__main__.ClassD3[builtins.int, builtins.float]"
+
+    # k = ClassD3()
+    # reveal_type(k)  # Revealed type is "__main__.ClassD3[builtins.str, builtins.list[builtins.str]]"  # TODO
+    l = ClassD3[int]()
+    reveal_type(l)  # N: Revealed type is "__main__.ClassD3[builtins.int, builtins.list[builtins.int]]"
+    m = ClassD3[int, float]()
+    reveal_type(m)  # N: Revealed type is "__main__.ClassD3[builtins.int, builtins.float]"
+
 [case testTypeVarDefaultsClassRecursiveMultipleFiles]
 # flags: --disallow-any-generics
 from typing import Generic, TypeVar
 from file2 import T as T2
 
-T = TypeVar('T', default=T2)
+T = TypeVar("T", default=T2)
 
 class ClassG1(Generic[T2, T]):
     pass
@@ -587,3 +674,46 @@ def func_c4(
     # reveal_type(b)  # Revealed type is "Tuple[builtins.int, builtins.str]"  # TODO
     reveal_type(c)  # N: Revealed type is "Tuple[builtins.int, builtins.float]"
 [builtins fixtures/tuple.pyi]
+
+[case testTypeVarDefaultsTypeAliasRecursive1]
+# flags: --disallow-any-generics
+from typing import Dict, List, TypeVar
+
+T1 = TypeVar("T1")
+T2 = TypeVar("T2", default=T1)
+
+TD1 = Dict[T1, T2]
+
+def func_d1(
+    a: TD1,  # E: Missing type parameters for generic type "TD1"
+    b: TD1[int],
+    c: TD1[int, float],
+) -> None:
+    reveal_type(a)  # N: Revealed type is "builtins.dict[Any, Any]"
+    reveal_type(b)  # N: Revealed type is "builtins.dict[builtins.int, builtins.int]"
+    reveal_type(c)  # N: Revealed type is "builtins.dict[builtins.int, builtins.float]"
+[builtins fixtures/dict.pyi]
+
+[case testTypeVarDefaultsTypeAliasRecursive2]
+from typing import Any, Dict, Generic, TypeVar
+
+T1 = TypeVar("T1", default=str)
+T2 = TypeVar("T2", default=T1)
+Alias1 = Dict[T1, T2]
+T3 = TypeVar("T3")
+class A(Generic[T3]): ...
+
+T4 = TypeVar("T4", default=A[Alias1])
+class B(Generic[T4]): ...
+
+def func_d3(
+    a: B,
+    b: B[A[Alias1[int]]],
+    c: B[A[Alias1[int, float]]],
+    d: B[int],
+) -> None:
+    reveal_type(a)  # N: Revealed type is "__main__.B[__main__.A[builtins.dict[builtins.str, builtins.str]]]"
+    reveal_type(b)  # N: Revealed type is "__main__.B[__main__.A[builtins.dict[builtins.int, builtins.int]]]"
+    reveal_type(c)  # N: Revealed type is "__main__.B[__main__.A[builtins.dict[builtins.int, builtins.float]]]"
+    reveal_type(d)  # N: Revealed type is "__main__.B[builtins.int]"
+[builtins fixtures/dict.pyi]
