From b205241a3912336bd1eb8154a7ff5523b71d0223 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Mon, 7 Nov 2022 00:29:02 +0000 Subject: [PATCH 01/25] Start working on Self type --- mypy/checkmember.py | 4 ++- mypy/nodes.py | 8 +++++ mypy/plugins/dataclasses.py | 26 ++++++++++---- mypy/semanal.py | 51 +++++++++++++++++++++++---- mypy/semanal_shared.py | 5 +++ mypy/subtypes.py | 4 ++- mypy/typeanal.py | 38 +++++++++++++++++--- test-data/unit/check-dataclasses.test | 20 +++++++++++ test-data/unit/check-protocols.test | 20 +++++++++++ test-data/unit/check-selftype.test | 13 +++++++ test-data/unit/lib-stub/typing.pyi | 1 + 11 files changed, 170 insertions(+), 20 deletions(-) diff --git a/mypy/checkmember.py b/mypy/checkmember.py index 6c9da4a6ce7c..75b5f2ac3fc2 100644 --- a/mypy/checkmember.py +++ b/mypy/checkmember.py @@ -6,7 +6,7 @@ from mypy import meet, message_registry, subtypes from mypy.erasetype import erase_typevars -from mypy.expandtype import expand_type_by_instance, freshen_function_type_vars +from mypy.expandtype import expand_type, expand_type_by_instance, freshen_function_type_vars from mypy.maptype import map_instance_to_supertype from mypy.messages import MessageBuilder from mypy.nodes import ( @@ -723,6 +723,8 @@ def analyze_var( if mx.is_lvalue and var.is_classvar: mx.msg.cant_assign_to_classvar(name, mx.context) t = get_proper_type(expand_type_by_instance(typ, itype)) + if var.info.self_type is not None and not var.is_property: + t = expand_type(t, {var.info.self_type.id: mx.original_type}) result: Type = t typ = get_proper_type(typ) if ( diff --git a/mypy/nodes.py b/mypy/nodes.py index 9221ec48aa61..49e50d37a644 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -2810,6 +2810,7 @@ class is generic then it will be a type constructor of higher kind. "has_type_var_tuple_type", "type_var_tuple_prefix", "type_var_tuple_suffix", + "self_type", ) _fullname: Bogus[str] # Fully qualified name @@ -2950,6 +2951,9 @@ class is generic then it will be a type constructor of higher kind. # in case we are doing multiple semantic analysis passes. special_alias: TypeAlias | None + # Shared type variable for typing.Self in this class (if used, otherwise None). + self_type: mypy.types.TypeVarType | None + FLAGS: Final = [ "is_abstract", "is_enum", @@ -3002,6 +3006,7 @@ def __init__(self, names: SymbolTable, defn: ClassDef, module_name: str) -> None self.is_newtype = False self.is_intersection = False self.metadata = {} + self.self_type = None def add_type_vars(self) -> None: self.has_type_var_tuple_type = False @@ -3219,6 +3224,7 @@ def serialize(self) -> JsonDict: "metadata": self.metadata, "slots": list(sorted(self.slots)) if self.slots is not None else None, "deletable_attributes": self.deletable_attributes, + "self_type": self.self_type.serialize() if self.self_type is not None else None, } return data @@ -3275,6 +3281,8 @@ def deserialize(cls, data: JsonDict) -> TypeInfo: ti.slots = set(data["slots"]) if data["slots"] is not None else None ti.deletable_attributes = data["deletable_attributes"] set_flags(ti, data["flags"]) + st = data["self_type"] + ti.self_type = mypy.types.TypeVarType.deserialize(st) if st is not None else None return ti diff --git a/mypy/plugins/dataclasses.py b/mypy/plugins/dataclasses.py index 26bc8ae80fdb..ccb95dd52dce 100644 --- a/mypy/plugins/dataclasses.py +++ b/mypy/plugins/dataclasses.py @@ -2,8 +2,10 @@ from __future__ import annotations +from typing import Optional from typing_extensions import Final +from mypy.expandtype import expand_type from mypy.nodes import ( ARG_NAMED, ARG_NAMED_OPT, @@ -52,6 +54,8 @@ ) # The set of decorators that generate dataclasses. +from mypy.typevars import fill_typevars + dataclass_makers: Final = {"dataclass", "dataclasses.dataclass"} # The set of functions that generate dataclass fields. field_makers: Final = {"dataclasses.field"} @@ -83,7 +87,7 @@ def __init__( self.info = info self.kw_only = kw_only - def to_argument(self) -> Argument: + def to_argument(self, current_info: TypeInfo) -> Argument: arg_kind = ARG_POS if self.kw_only and self.has_default: arg_kind = ARG_NAMED_OPT @@ -92,11 +96,19 @@ def to_argument(self) -> Argument: elif not self.kw_only and self.has_default: arg_kind = ARG_OPT return Argument( - variable=self.to_var(), type_annotation=self.type, initializer=None, kind=arg_kind + variable=self.to_var(current_info), + type_annotation=self.expand_type(current_info), + initializer=None, + kind=arg_kind, ) - def to_var(self) -> Var: - return Var(self.name, self.type) + def expand_type(self, current_info: TypeInfo) -> Optional[Type]: + if self.type is not None and self.info.self_type is not None: + return expand_type(self.type, {self.info.self_type.id: fill_typevars(current_info)}) + return self.type + + def to_var(self, current_info: TypeInfo) -> Var: + return Var(self.name, self.expand_type(current_info)) def serialize(self) -> JsonDict: assert self.type @@ -176,7 +188,7 @@ def transform(self) -> bool: ): args = [ - attr.to_argument() + attr.to_argument(info) for attr in attributes if attr.is_in_init and not self._is_kw_only_type(attr.type) ] @@ -548,7 +560,7 @@ def _freeze(self, attributes: list[DataclassAttribute]) -> None: if isinstance(var, Var): var.is_property = True else: - var = attr.to_var() + var = attr.to_var(info) var.info = info var.is_property = True var._fullname = info.fullname + "." + var.name @@ -567,7 +579,7 @@ def _propertize_callables( info = self._ctx.cls.info for attr in attributes: if isinstance(get_proper_type(attr.type), CallableType): - var = attr.to_var() + var = attr.to_var(info) var.info = info var.is_property = True var.is_settable_property = settable diff --git a/mypy/semanal.py b/mypy/semanal.py index 77555648ba7e..fc7c27354f3a 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -225,6 +225,7 @@ analyze_type_alias, check_for_explicit_any, detect_diverging_alias, + find_self_type, fix_instance_types, has_any_from_unimported_type, no_subscript_builtin_alias, @@ -272,7 +273,7 @@ invalid_recursive_alias, is_named_instance, ) -from mypy.typevars import fill_typevars +from mypy.typevars import fill_typevars, fill_typevars_with_any from mypy.util import ( correct_relative_import, is_dunder, @@ -812,7 +813,10 @@ def analyze_func_def(self, defn: FuncDef) -> None: if defn.type: assert isinstance(defn.type, CallableType) - self.update_function_type_variables(defn.type, defn) + has_self_type = self.update_function_type_variables(defn.type, defn) + else: + has_self_type = False + self.function_stack.pop() if self.is_class_scope(): @@ -823,7 +827,7 @@ def analyze_func_def(self, defn: FuncDef) -> None: assert isinstance(defn.type, CallableType) if isinstance(get_proper_type(defn.type.ret_type), AnyType): defn.type = defn.type.copy_modified(ret_type=NoneType()) - self.prepare_method_signature(defn, self.type) + self.prepare_method_signature(defn, self.type, has_self_type) # Analyze function signature with self.tvar_scope_frame(self.tvar_scope.method_frame()): @@ -842,6 +846,10 @@ def analyze_func_def(self, defn: FuncDef) -> None: assert isinstance(result, ProperType) if isinstance(result, CallableType): result = self.remove_unpack_kwargs(defn, result) + if has_self_type and self.type is not None: + info = self.type + if info.self_type is not None: + result.variables = [info.self_type] + list(result.variables) defn.type = result self.add_type_alias_deps(analyzer.aliases_used) self.check_function_signature(defn) @@ -914,7 +922,7 @@ def remove_unpack_kwargs(self, defn: FuncDef, typ: CallableType) -> CallableType new_arg_types = typ.arg_types[:-1] + [last_type] return typ.copy_modified(arg_types=new_arg_types, unpack_kwargs=True) - def prepare_method_signature(self, func: FuncDef, info: TypeInfo) -> None: + def prepare_method_signature(self, func: FuncDef, info: TypeInfo, has_self_type: bool) -> None: """Check basic signature validity and tweak annotation of self/cls argument.""" # Only non-static methods are special. functype = func.type @@ -926,10 +934,18 @@ def prepare_method_signature(self, func: FuncDef, info: TypeInfo) -> None: elif isinstance(functype, CallableType): self_type = get_proper_type(functype.arg_types[0]) if isinstance(self_type, AnyType): - leading_type: Type = fill_typevars(info) + if has_self_type: + assert self.type is not None and self.type.self_type is not None + leading_type: Type = self.type.self_type + else: + leading_type = fill_typevars(info) if func.is_class or func.name == "__new__": leading_type = self.class_type(leading_type) func.type = replace_implicit_first_type(functype, leading_type) + elif has_self_type: + self.fail("Method cannot have explicit self annotation and Self type", func) + elif has_self_type: + self.fail("Static methods cannot use Self type", func) def set_original_def(self, previous: Node | None, new: FuncDef | Decorator) -> bool: """If 'new' conditionally redefine 'previous', set 'previous' as original @@ -954,7 +970,7 @@ def f(): ... # Error: 'f' redefined else: return False - def update_function_type_variables(self, fun_type: CallableType, defn: FuncItem) -> None: + def update_function_type_variables(self, fun_type: CallableType, defn: FuncItem) -> bool: """Make any type variables in the signature of defn explicit. Update the signature of defn to contain type variable definitions @@ -962,7 +978,23 @@ def update_function_type_variables(self, fun_type: CallableType, defn: FuncItem) """ with self.tvar_scope_frame(self.tvar_scope.method_frame()): a = self.type_analyzer() - fun_type.variables = a.bind_function_type_variables(fun_type, defn) + fun_type.variables, has_self_type = a.bind_function_type_variables(fun_type, defn) + if has_self_type and self.type is not None: + self.setup_self_type() + return has_self_type + + def setup_self_type(self) -> None: + assert self.type is not None + info = self.type + if info.self_type is not None: + return + info.self_type = TypeVarType( + "Self", + f"{info.fullname}.Self", + self.tvar_scope.new_unique_func_id(), + [], + fill_typevars_with_any(info), + ) def visit_overloaded_func_def(self, defn: OverloadedFuncDef) -> None: self.statement = defn @@ -3103,6 +3135,11 @@ def process_type_annotation(self, s: AssignmentStmt) -> None: if s.type: lvalue = s.lvalues[-1] allow_tuple_literal = isinstance(lvalue, TupleExpr) + has_self_type = find_self_type( + s.type, lambda name: self.lookup_qualified(name, s, suppress_errors=True) + ) + if has_self_type and self.type: + self.setup_self_type() analyzed = self.anal_type(s.type, allow_tuple_literal=allow_tuple_literal) # Don't store not ready types (including placeholders). if analyzed is None or has_placeholder(analyzed): diff --git a/mypy/semanal_shared.py b/mypy/semanal_shared.py index 63f4f5516f79..db55f01cf7ca 100644 --- a/mypy/semanal_shared.py +++ b/mypy/semanal_shared.py @@ -119,6 +119,11 @@ def is_stub_file(self) -> bool: def is_func_scope(self) -> bool: raise NotImplementedError + @property + @abstractmethod + def type(self) -> TypeInfo | None: + raise NotImplementedError + @trait class SemanticAnalyzerInterface(SemanticAnalyzerCoreInterface): diff --git a/mypy/subtypes.py b/mypy/subtypes.py index 2724379ab878..c0bcf63b2bd2 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -8,7 +8,7 @@ import mypy.constraints import mypy.typeops from mypy.erasetype import erase_type -from mypy.expandtype import expand_type_by_instance +from mypy.expandtype import expand_type, expand_type_by_instance from mypy.maptype import map_instance_to_supertype # Circular import; done in the function instead. @@ -1194,6 +1194,8 @@ def find_node_type( ) else: typ = node.type + if typ is not None and node.info.self_type is not None and not node.is_property: + typ = expand_type(typ, {node.info.self_type.id: subtype}) p_typ = get_proper_type(typ) if typ is None: return AnyType(TypeOfAny.from_error) diff --git a/mypy/typeanal.py b/mypy/typeanal.py index 35f60f54605a..dd095cc7d9ad 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -117,6 +117,8 @@ "asyncio.futures.Future", } +SELF_TYPE_NAMES: Final = {"typing.Self", "typing_extensions.Self"} + def analyze_type_alias( node: Expression, @@ -575,6 +577,11 @@ def try_analyze_special_unbound_type(self, t: UnboundType, fullname: str) -> Typ self.fail("Unpack[...] requires exactly one type argument", t) return AnyType(TypeOfAny.from_error) return UnpackType(self.anal_type(t.args[0]), line=t.line, column=t.column) + elif fullname in SELF_TYPE_NAMES: + if self.api.type is None or self.api.type.self_type is None: + self.fail("Self type is only allowed in annotations within class definition", t) + return AnyType(TypeOfAny.from_error) + return self.api.type.self_type return None def get_omitted_any(self, typ: Type, fullname: str | None = None) -> AnyType: @@ -853,7 +860,7 @@ def visit_callable_type(self, t: CallableType, nested: bool = True) -> Type: if self.defining_alias: variables = t.variables else: - variables = self.bind_function_type_variables(t, t) + variables, _ = self.bind_function_type_variables(t, t) special = self.anal_type_guard(t.ret_type) arg_kinds = t.arg_kinds if len(arg_kinds) >= 2 and arg_kinds[-2] == ARG_STAR and arg_kinds[-1] == ARG_STAR2: @@ -1347,19 +1354,26 @@ def infer_type_variables(self, type: CallableType) -> list[tuple[str, TypeVarLik def bind_function_type_variables( self, fun_type: CallableType, defn: Context - ) -> Sequence[TypeVarLikeType]: + ) -> tuple[Sequence[TypeVarLikeType], bool]: """Find the type variables of the function type and bind them in our tvar_scope""" + has_self_type = False if fun_type.variables: defs = [] for var in fun_type.variables: var_node = self.lookup_qualified(var.name, defn) assert var_node, "Binding for function type variable not found within function" var_expr = var_node.node + if var_node.fullname in SELF_TYPE_NAMES: + has_self_type = True + continue assert isinstance(var_expr, TypeVarLikeExpr) binding = self.tvar_scope.bind_new(var.name, var_expr) defs.append(binding) - return defs + return defs, has_self_type typevars = self.infer_type_variables(fun_type) + has_self_type = find_self_type( + fun_type, lambda name: self.api.lookup_qualified(name, defn, suppress_errors=True) + ) # Do not define a new type variable if already defined in scope. typevars = [ (name, tvar) for name, tvar in typevars if not self.is_defined_type_var(name, defn) @@ -1375,7 +1389,7 @@ def bind_function_type_variables( binding = self.tvar_scope.bind_new(name, tvar) defs.append(binding) - return defs + return defs, has_self_type def is_defined_type_var(self, tvar: str, context: Context) -> bool: tvar_node = self.lookup_qualified(tvar, context) @@ -1961,3 +1975,19 @@ def visit_instance(self, typ: Instance) -> None: python_version=self.python_version, use_generic_error=True, ) + + +def find_self_type(typ: Type, lookup: Callable[[str], SymbolTableNode | None]) -> bool: + return typ.accept(HasSelfType(lookup)) + + +class HasSelfType(TypeQuery[bool]): + def __init__(self, lookup: Callable[[str], SymbolTableNode | None]) -> None: + self.lookup = lookup + super().__init__(any) + + def visit_unbound_type(self, t: UnboundType) -> bool: + sym = self.lookup(t.name) + if sym and sym.fullname in SELF_TYPE_NAMES: + return True + return super().visit_unbound_type(t) diff --git a/test-data/unit/check-dataclasses.test b/test-data/unit/check-dataclasses.test index 3ec4c60e6929..20bf8230b12e 100644 --- a/test-data/unit/check-dataclasses.test +++ b/test-data/unit/check-dataclasses.test @@ -1932,3 +1932,23 @@ B = List[C] class C(CC): ... class CC: ... [builtins fixtures/dataclasses.pyi] + +[case testDataclassSelfType] +# flags: --strict-optional +from dataclasses import dataclass +from typing import Self, TypeVar, Generic, Optional + +T = TypeVar("T") + +@dataclass +class LinkedList(Generic[T]): + value: T + next: Optional[Self] = None + +@dataclass +class SubLinkedList(LinkedList[int]): ... + +lst = SubLinkedList(1, LinkedList(2)) # E: Argument 2 to "SubLinkedList" has incompatible type "LinkedList[int]"; expected "SubLinkedList" +reveal_type(lst.next) # N: Revealed type is "Union[__main__.SubLinkedList, None]" +reveal_type(SubLinkedList) # N: Revealed type is "def (value: builtins.int, next: __main__.SubLinkedList =) -> __main__.SubLinkedList" +[builtins fixtures/dataclasses.pyi] diff --git a/test-data/unit/check-protocols.test b/test-data/unit/check-protocols.test index 113b2000fc22..b686a0b7ef8b 100644 --- a/test-data/unit/check-protocols.test +++ b/test-data/unit/check-protocols.test @@ -3843,3 +3843,23 @@ def f() -> str: ... [file package/badmod.py] def nothing() -> int: ... [builtins fixtures/module.pyi] + +[case testProtocolSelfTypeNewSyntax] +from typing import Protocol, Self + +class P(Protocol): + @property + def next(self) -> Self: ... + +class C: + next: C +class S: + next: Self + +x: P = C() +y: P = S() + +z: P +reveal_type(S().next) # N: Revealed type is "__main__.S" +reveal_type(z.next) # N: Revealed type is "__main__.P" +[builtins fixtures/property.pyi] diff --git a/test-data/unit/check-selftype.test b/test-data/unit/check-selftype.test index bfb0eb5a4d89..2e0b271170b3 100644 --- a/test-data/unit/check-selftype.test +++ b/test-data/unit/check-selftype.test @@ -1267,3 +1267,16 @@ class Test(Generic[T]): a: deque[List[T]] # previously this failed with 'Incompatible types in assignment (expression has type "deque[List[List[T]]]", variable has type "deque[List[T]]")' b: deque[List[T]] = a.copy() + +[case testTypingSelfBasic] +from typing import Self, List + +class C: + attr: List[Self] + def meth(self) -> List[Self]: ... +class D(C): ... + +reveal_type(C.meth) # N: Revealed type is "def [Self <: __main__.C] (self: Self`-1) -> builtins.list[Self`-1]" +C.attr # E: Access to generic instance variables via class is ambiguous +reveal_type(D().meth()) # N: Revealed type is "builtins.list[__main__.D]" +reveal_type(D().attr) # N: Revealed type is "builtins.list[__main__.D]" diff --git a/test-data/unit/lib-stub/typing.pyi b/test-data/unit/lib-stub/typing.pyi index 23d97704d934..f3850d3936b4 100644 --- a/test-data/unit/lib-stub/typing.pyi +++ b/test-data/unit/lib-stub/typing.pyi @@ -27,6 +27,7 @@ NoReturn = 0 Never = 0 NewType = 0 ParamSpec = 0 +Self = 0 TYPE_CHECKING = 0 T = TypeVar('T') From 5e5bd6fbb950b14075fdf568986d37d7c7ae3c9d Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Mon, 7 Nov 2022 21:53:35 +0000 Subject: [PATCH 02/25] Add support for special forms; some fixes --- mypy/plugins/dataclasses.py | 15 ++++++--- mypy/semanal.py | 6 ++++ mypy/semanal_namedtuple.py | 29 +++++++++------- mypy/semanal_shared.py | 10 ++++++ mypy/semanal_typeddict.py | 29 +++++++++------- mypy/typeanal.py | 9 +++++ test-data/unit/check-dataclasses.test | 6 ++-- test-data/unit/check-namedtuple.test | 29 ++++++++++++++++ test-data/unit/check-selftype.test | 33 +++++++++++++++++++ test-data/unit/check-typeddict.test | 27 +++++++++++++++ test-data/unit/fixtures/typing-namedtuple.pyi | 1 + test-data/unit/fixtures/typing-typeddict.pyi | 1 + 12 files changed, 165 insertions(+), 30 deletions(-) diff --git a/mypy/plugins/dataclasses.py b/mypy/plugins/dataclasses.py index ccb95dd52dce..430b34ffd636 100644 --- a/mypy/plugins/dataclasses.py +++ b/mypy/plugins/dataclasses.py @@ -104,6 +104,10 @@ def to_argument(self, current_info: TypeInfo) -> Argument: def expand_type(self, current_info: TypeInfo) -> Optional[Type]: if self.type is not None and self.info.self_type is not None: + # In general, it is not safe to call `expand_type()` during semantic analyzis, + # however this plugin is called very late, so all types should be fully ready. + # Also, it is tricky to avoid eager expansion of Self types here (e.g. because + # we serialize attributes). return expand_type(self.type, {self.info.self_type.id: fill_typevars(current_info)}) return self.type @@ -187,11 +191,12 @@ def transform(self) -> bool: and attributes ): - args = [ - attr.to_argument(info) - for attr in attributes - if attr.is_in_init and not self._is_kw_only_type(attr.type) - ] + with state.strict_optional_set(ctx.api.options.strict_optional): + args = [ + attr.to_argument(info) + for attr in attributes + if attr.is_in_init and not self._is_kw_only_type(attr.type) + ] if info.fallback_to_any: # Make positional args optional since we don't know their order. diff --git a/mypy/semanal.py b/mypy/semanal.py index fc7c27354f3a..5f0a0979e860 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -3139,6 +3139,8 @@ def process_type_annotation(self, s: AssignmentStmt) -> None: s.type, lambda name: self.lookup_qualified(name, s, suppress_errors=True) ) if has_self_type and self.type: + if self.type.typeddict_type or self.type.tuple_type: + return self.setup_self_type() analyzed = self.anal_type(s.type, allow_tuple_literal=allow_tuple_literal) # Don't store not ready types (including placeholders). @@ -6140,6 +6142,7 @@ def type_analyzer( allow_required: bool = False, allow_param_spec_literals: bool = False, report_invalid_types: bool = True, + self_type_override: Type | None = None, ) -> TypeAnalyser: if tvar_scope is None: tvar_scope = self.tvar_scope @@ -6155,6 +6158,7 @@ def type_analyzer( allow_placeholder=allow_placeholder, allow_required=allow_required, allow_param_spec_literals=allow_param_spec_literals, + self_type_override=self_type_override, ) tpan.in_dynamic_func = bool(self.function_stack and self.function_stack[-1].is_dynamic()) tpan.global_scope = not self.type and not self.function_stack @@ -6174,6 +6178,7 @@ def anal_type( allow_required: bool = False, allow_param_spec_literals: bool = False, report_invalid_types: bool = True, + self_type_override: Type | None = None, third_pass: bool = False, ) -> Type | None: """Semantically analyze a type. @@ -6204,6 +6209,7 @@ def anal_type( allow_required=allow_required, allow_param_spec_literals=allow_param_spec_literals, report_invalid_types=report_invalid_types, + self_type_override=self_type_override, ) tag = self.track_incomplete_refs() typ = typ.accept(a) diff --git a/mypy/semanal_namedtuple.py b/mypy/semanal_namedtuple.py index 1727c18b6fd9..1f6f7653b8dd 100644 --- a/mypy/semanal_namedtuple.py +++ b/mypy/semanal_namedtuple.py @@ -48,6 +48,7 @@ calculate_tuple_fallback, has_placeholder, set_callable_name, + special_self_type, ) from mypy.types import ( TYPED_NAMEDTUPLE_NAMES, @@ -107,16 +108,16 @@ def analyze_namedtuple_classdef( if isinstance(base_expr, RefExpr): self.api.accept(base_expr) if base_expr.fullname in TYPED_NAMEDTUPLE_NAMES: - result = self.check_namedtuple_classdef(defn, is_stub_file) + existing_info = None + if isinstance(defn.analyzed, NamedTupleExpr): + existing_info = defn.analyzed.info + result = self.check_namedtuple_classdef(defn, is_stub_file, existing_info) if result is None: # This is a valid named tuple, but some types are incomplete. return True, None items, types, default_items = result if is_func_scope and "@" not in defn.name: defn.name += "@" + str(defn.line) - existing_info = None - if isinstance(defn.analyzed, NamedTupleExpr): - existing_info = defn.analyzed.info info = self.build_namedtuple_typeinfo( defn.name, items, types, default_items, defn.line, existing_info ) @@ -129,7 +130,7 @@ def analyze_namedtuple_classdef( return False, None def check_namedtuple_classdef( - self, defn: ClassDef, is_stub_file: bool + self, defn: ClassDef, is_stub_file: bool, existing_info: TypeInfo | None ) -> tuple[list[str], list[Type], dict[str, Expression]] | None: """Parse and validate fields in named tuple class definition. @@ -178,6 +179,7 @@ def check_namedtuple_classdef( stmt.type, allow_placeholder=not self.options.disable_recursive_aliases and not self.api.is_func_scope(), + self_type_override=special_self_type(existing_info, defn), ) if analyzed is None: # Something is incomplete. We need to defer this named tuple. @@ -229,7 +231,10 @@ def check_namedtuple( is_typed = True else: return None, None, [] - result = self.parse_namedtuple_args(call, fullname) + existing_info = None + if isinstance(node.analyzed, NamedTupleExpr): + existing_info = node.analyzed.info + result = self.parse_namedtuple_args(call, fullname, existing_info) if result: items, types, defaults, typename, tvar_defs, ok = result else: @@ -277,9 +282,6 @@ def check_namedtuple( else: default_items = {} - existing_info = None - if isinstance(node.analyzed, NamedTupleExpr): - existing_info = node.analyzed.info info = self.build_namedtuple_typeinfo( name, items, types, default_items, node.line, existing_info ) @@ -317,7 +319,7 @@ def store_namedtuple_info( call.analyzed.set_line(call) def parse_namedtuple_args( - self, call: CallExpr, fullname: str + self, call: CallExpr, fullname: str, existing_info: TypeInfo | None ) -> None | (tuple[list[str], list[Type], list[Expression], str, list[TypeVarLikeType], bool]): """Parse a namedtuple() call into data needed to construct a type. @@ -394,7 +396,9 @@ def parse_namedtuple_args( ] tvar_defs = self.api.get_and_bind_all_tvars(type_exprs) # The fields argument contains (name, type) tuples. - result = self.parse_namedtuple_fields_with_types(listexpr.items, call) + result = self.parse_namedtuple_fields_with_types( + listexpr.items, call, existing_info + ) if result is None: # One of the types is not ready, defer. return None @@ -416,7 +420,7 @@ def parse_namedtuple_args( return items, types, defaults, typename, tvar_defs, True def parse_namedtuple_fields_with_types( - self, nodes: list[Expression], context: Context + self, nodes: list[Expression], call: CallExpr, existing_info: TypeInfo | None ) -> tuple[list[str], list[Type], list[Expression], bool] | None: """Parse typed named tuple fields. @@ -445,6 +449,7 @@ def parse_namedtuple_fields_with_types( type, allow_placeholder=not self.options.disable_recursive_aliases and not self.api.is_func_scope(), + self_type_override=special_self_type(existing_info, call), ) # Workaround #4987 and avoid introducing a bogus UnboundType if isinstance(analyzed, UnboundType): diff --git a/mypy/semanal_shared.py b/mypy/semanal_shared.py index db55f01cf7ca..25b0374d491d 100644 --- a/mypy/semanal_shared.py +++ b/mypy/semanal_shared.py @@ -33,6 +33,7 @@ ProperType, TupleType, Type, + TypeAliasType, TypeVarId, TypeVarLikeType, get_proper_type, @@ -167,6 +168,7 @@ def anal_type( allow_required: bool = False, allow_placeholder: bool = False, report_invalid_types: bool = True, + self_type_override: Type | None = None, ) -> Type | None: raise NotImplementedError @@ -329,3 +331,11 @@ def visit_placeholder_type(self, t: PlaceholderType) -> bool: def has_placeholder(typ: Type) -> bool: """Check if a type contains any placeholder types (recursively).""" return typ.accept(HasPlaceholders()) + + +def special_self_type(info: TypeInfo | None, defn: Context) -> Type: + if info is not None: + assert info.special_alias is not None + return TypeAliasType(info.special_alias, list(info.defn.type_vars)) + else: + return PlaceholderType(None, [], defn.line) diff --git a/mypy/semanal_typeddict.py b/mypy/semanal_typeddict.py index b864c2a30615..46959b6867fb 100644 --- a/mypy/semanal_typeddict.py +++ b/mypy/semanal_typeddict.py @@ -31,7 +31,7 @@ TypeInfo, ) from mypy.options import Options -from mypy.semanal_shared import SemanticAnalyzerInterface, has_placeholder +from mypy.semanal_shared import SemanticAnalyzerInterface, has_placeholder, special_self_type from mypy.typeanal import check_for_explicit_any, has_any_from_unimported_type from mypy.types import ( TPDICT_NAMES, @@ -94,7 +94,9 @@ def analyze_typeddict_classdef(self, defn: ClassDef) -> tuple[bool, TypeInfo | N and defn.base_type_exprs[0].fullname in TPDICT_NAMES ): # Building a new TypedDict - fields, types, statements, required_keys = self.analyze_typeddict_classdef_fields(defn) + fields, types, statements, required_keys = self.analyze_typeddict_classdef_fields( + defn, existing_info + ) if fields is None: return True, None # Defer info = self.build_typeddict_typeinfo( @@ -146,7 +148,7 @@ def analyze_typeddict_classdef(self, defn: ClassDef) -> tuple[bool, TypeInfo | N new_types, new_statements, new_required_keys, - ) = self.analyze_typeddict_classdef_fields(defn, keys) + ) = self.analyze_typeddict_classdef_fields(defn, existing_info, keys) if new_keys is None: return True, None # Defer keys.extend(new_keys) @@ -257,7 +259,7 @@ def map_items_to_base( return mapped_items def analyze_typeddict_classdef_fields( - self, defn: ClassDef, oldfields: list[str] | None = None + self, defn: ClassDef, existing_info: TypeInfo | None, oldfields: list[str] | None = None ) -> tuple[list[str] | None, list[Type], list[Statement], set[str]]: """Analyze fields defined in a TypedDict class definition. @@ -305,6 +307,7 @@ def analyze_typeddict_classdef_fields( allow_required=True, allow_placeholder=not self.options.disable_recursive_aliases and not self.api.is_func_scope(), + self_type_override=special_self_type(existing_info, defn), ) if analyzed is None: return None, [], [], set() # Need to defer @@ -356,7 +359,10 @@ def check_typeddict( fullname = callee.fullname if fullname not in TPDICT_NAMES: return False, None, [] - res = self.parse_typeddict_args(call) + existing_info = None + if isinstance(node.analyzed, TypedDictExpr): + existing_info = node.analyzed.info + res = self.parse_typeddict_args(call, existing_info) if res is None: # This is a valid typed dict, but some type is not ready. # The caller should defer this until next iteration. @@ -386,9 +392,6 @@ def check_typeddict( types = [ # unwrap Required[T] to just T t.item if isinstance(t, RequiredType) else t for t in types ] - existing_info = None - if isinstance(node.analyzed, TypedDictExpr): - existing_info = node.analyzed.info info = self.build_typeddict_typeinfo( name, items, types, required_keys, call.line, existing_info ) @@ -403,7 +406,7 @@ def check_typeddict( return True, info, tvar_defs def parse_typeddict_args( - self, call: CallExpr + self, call: CallExpr, existing_info: TypeInfo | None ) -> tuple[str, list[str], list[Type], bool, list[TypeVarLikeType], bool] | None: """Parse typed dict call expression. @@ -440,7 +443,7 @@ def parse_typeddict_args( ) dictexpr = args[1] tvar_defs = self.api.get_and_bind_all_tvars([t for k, t in dictexpr.items]) - res = self.parse_typeddict_fields_with_types(dictexpr.items, call) + res = self.parse_typeddict_fields_with_types(dictexpr.items, call, existing_info) if res is None: # One of the types is not ready, defer. return None @@ -458,7 +461,10 @@ def parse_typeddict_args( return args[0].value, items, types, total, tvar_defs, ok def parse_typeddict_fields_with_types( - self, dict_items: list[tuple[Expression | None, Expression]], context: Context + self, + dict_items: list[tuple[Expression | None, Expression]], + call: CallExpr, + existing_info: TypeInfo | None, ) -> tuple[list[str], list[Type], bool] | None: """Parse typed dict items passed as pairs (name expression, type expression). @@ -500,6 +506,7 @@ def parse_typeddict_fields_with_types( allow_required=True, allow_placeholder=not self.options.disable_recursive_aliases and not self.api.is_func_scope(), + self_type_override=special_self_type(existing_info, call), ) if analyzed is None: return None diff --git a/mypy/typeanal.py b/mypy/typeanal.py index dd095cc7d9ad..3b1fc9272d2f 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -198,6 +198,7 @@ def __init__( allow_required: bool = False, allow_param_spec_literals: bool = False, report_invalid_types: bool = True, + self_type_override: Type | None = None, ) -> None: self.api = api self.lookup_qualified = api.lookup_qualified @@ -233,6 +234,10 @@ def __init__( self.is_typeshed_stub = is_typeshed_stub # Names of type aliases encountered while analysing a type will be collected here. self.aliases_used: set[str] = set() + # Used by special forms like TypedDicts and NamedTuples, where Self type + # needs a special handling (for such types the target TypeInfo not be + # created yet when analyzing Self type, unlike for regular classes). + self.self_type_override = self_type_override def visit_unbound_type(self, t: UnboundType, defining_literal: bool = False) -> Type: typ = self.visit_unbound_type_nonoptional(t, defining_literal) @@ -578,6 +583,10 @@ def try_analyze_special_unbound_type(self, t: UnboundType, fullname: str) -> Typ return AnyType(TypeOfAny.from_error) return UnpackType(self.anal_type(t.args[0]), line=t.line, column=t.column) elif fullname in SELF_TYPE_NAMES: + if self.self_type_override is not None: + # For various special forms that can't be inherited but use Self + # for convenience we eagerly replace Self. + return self.self_type_override if self.api.type is None or self.api.type.self_type is None: self.fail("Self type is only allowed in annotations within class definition", t) return AnyType(TypeOfAny.from_error) diff --git a/test-data/unit/check-dataclasses.test b/test-data/unit/check-dataclasses.test index 20bf8230b12e..d9f4e9c17c2b 100644 --- a/test-data/unit/check-dataclasses.test +++ b/test-data/unit/check-dataclasses.test @@ -1945,10 +1945,12 @@ class LinkedList(Generic[T]): value: T next: Optional[Self] = None +l_int: LinkedList[int] = LinkedList(1, LinkedList("no", None)) # E: Argument 1 to "LinkedList" has incompatible type "str"; expected "int" + @dataclass class SubLinkedList(LinkedList[int]): ... -lst = SubLinkedList(1, LinkedList(2)) # E: Argument 2 to "SubLinkedList" has incompatible type "LinkedList[int]"; expected "SubLinkedList" +lst = SubLinkedList(1, LinkedList(2)) # E: Argument 2 to "SubLinkedList" has incompatible type "LinkedList[int]"; expected "Optional[SubLinkedList]" reveal_type(lst.next) # N: Revealed type is "Union[__main__.SubLinkedList, None]" -reveal_type(SubLinkedList) # N: Revealed type is "def (value: builtins.int, next: __main__.SubLinkedList =) -> __main__.SubLinkedList" +reveal_type(SubLinkedList) # N: Revealed type is "def (value: builtins.int, next: Union[__main__.SubLinkedList, None] =) -> __main__.SubLinkedList" [builtins fixtures/dataclasses.pyi] diff --git a/test-data/unit/check-namedtuple.test b/test-data/unit/check-namedtuple.test index 438e17a6ba0a..82d3add825bb 100644 --- a/test-data/unit/check-namedtuple.test +++ b/test-data/unit/check-namedtuple.test @@ -1306,3 +1306,32 @@ class C( [builtins fixtures/tuple.pyi] [typing fixtures/typing-namedtuple.pyi] + +[case testNamedTupleSelfTypeNewSyntaxGeneric] +# flags: --strict-optional +from typing import Self, NamedTuple, Optional, Generic, TypeVar + +T = TypeVar("T") +class NT(NamedTuple, Generic[T]): + val: T + next: Optional[Self] + +reveal_type(NT) # N: Revealed type is "def [T] (val: T`1, next: Union[Tuple[T`1, Union[..., None], fallback=__main__.NT[T`1]], None]) -> Tuple[T`1, Union[Tuple[T`1, Union[..., None], fallback=__main__.NT[T`1]], None], fallback=__main__.NT[T`1]]" +n: NT[int] +reveal_type(n.next) # N: Revealed type is "Union[Tuple[builtins.int, Union[..., None], fallback=__main__.NT[builtins.int]], None]" +reveal_type(n[1]) # N: Revealed type is "Union[Tuple[builtins.int, Union[..., None], fallback=__main__.NT[builtins.int]], None]" + +m: NT[int] = NT(1, NT("no", None)) # E: Argument 1 to "NT" has incompatible type "str"; expected "int" +[builtins fixtures/tuple.pyi] +[typing fixtures/typing-namedtuple.pyi] + +[case testNamedTupleSelfTypeNewSyntaxCall] +# flags: --strict-optional +from typing import Self, NamedTuple, Optional + +NTC = NamedTuple("NTC", [("val", int), ("next", Optional[Self])]) +nc: NTC +reveal_type(nc.next) # N: Revealed type is "Union[Tuple[builtins.int, Union[..., None], fallback=__main__.NTC], None]" +reveal_type(nc[1]) # N: Revealed type is "Union[Tuple[builtins.int, Union[..., None], fallback=__main__.NTC], None]" +[builtins fixtures/tuple.pyi] +[typing fixtures/typing-namedtuple.pyi] diff --git a/test-data/unit/check-selftype.test b/test-data/unit/check-selftype.test index 2e0b271170b3..37ab6d594fe3 100644 --- a/test-data/unit/check-selftype.test +++ b/test-data/unit/check-selftype.test @@ -1280,3 +1280,36 @@ reveal_type(C.meth) # N: Revealed type is "def [Self <: __main__.C] (self: Self C.attr # E: Access to generic instance variables via class is ambiguous reveal_type(D().meth()) # N: Revealed type is "builtins.list[__main__.D]" reveal_type(D().attr) # N: Revealed type is "builtins.list[__main__.D]" + +-- Test various errors + +-- Test all nodes: property, classmethod, overload, ..., and both incrementals + +-- Test Self nested in a generic alias + +[case testTypingSelfMixedTypeVars] +from typing import Self, TypeVar, Generic, Tuple + +T = TypeVar("T") +S = TypeVar("S") + +class C(Generic[T]): + def meth(self, arg: S) -> Tuple[Self, S, T]: ... + +class D(C[int]): ... + +c: C[int] +d: D +reveal_type(c.meth("test")) # N: Revealed type is "Tuple[__main__.C[builtins.int], builtins.str, builtins.int]" +reveal_type(d.meth("test")) # N: Revealed type is "Tuple[__main__.D, builtins.str, builtins.int]" +[builtins fixtures/tuple.pyi] + +[case testTypingSelfRecursiveInit] +from typing import Self + +class C: + def __init__(self, other: Self) -> None: ... +class D(C): ... + +reveal_type(C) # N: Revealed type is "def (other: __main__.C) -> __main__.C" +reveal_type(D) # N: Revealed type is "def (other: __main__.D) -> __main__.D" diff --git a/test-data/unit/check-typeddict.test b/test-data/unit/check-typeddict.test index 796f2f547528..469c82fb159d 100644 --- a/test-data/unit/check-typeddict.test +++ b/test-data/unit/check-typeddict.test @@ -2590,3 +2590,30 @@ TD[str](key=0, value=0) # E: Incompatible types (expression has type "int", Typ TD[str]({"key": 0, "value": 0}) # E: Incompatible types (expression has type "int", TypedDict item "value" has type "str") [builtins fixtures/dict.pyi] [typing fixtures/typing-typeddict.pyi] + +[case testTypedDictSelfTypeNewSyntaxGeneric] +# flags: --strict-optional +from typing import Self, TypedDict, Optional, Generic, TypeVar + +T = TypeVar("T") +class TD(TypedDict, Generic[T]): + val: T + next: Optional[Self] + +reveal_type(TD) # N: Revealed type is "def [T] (*, val: T`1, next: Union[TypedDict('__main__.TD', {'val': T`1, 'next': Union[..., None]}), None]) -> TypedDict('__main__.TD', {'val': T`1, 'next': Union[TypedDict('__main__.TD', {'val': T`1, 'next': Union[..., None]}), None]})" +t: TD[int] +reveal_type(t["next"]) # N: Revealed type is "Union[TypedDict('__main__.TD', {'val': builtins.int, 'next': Union[..., None]}), None]" + +s: TD[int] = TD(val=1, next=TD(val="no", next=None)) # E: Incompatible types (expression has type "str", TypedDict item "val" has type "int") +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypedDictSelfTypeNewSyntaxCall] +# flags: --strict-optional +from typing import Self, TypedDict, Optional + +TDC = TypedDict("TDC", {"val": int, "next": Optional[Self]}) +tc: TDC +reveal_type(tc["next"]) # N: Revealed type is "Union[TypedDict('__main__.TDC', {'val': builtins.int, 'next': Union[..., None]}), None]" +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] diff --git a/test-data/unit/fixtures/typing-namedtuple.pyi b/test-data/unit/fixtures/typing-namedtuple.pyi index d51134ead599..1a31549463b6 100644 --- a/test-data/unit/fixtures/typing-namedtuple.pyi +++ b/test-data/unit/fixtures/typing-namedtuple.pyi @@ -5,6 +5,7 @@ overload = 0 Type = 0 Literal = 0 Optional = 0 +Self = 0 T_co = TypeVar('T_co', covariant=True) KT = TypeVar('KT') diff --git a/test-data/unit/fixtures/typing-typeddict.pyi b/test-data/unit/fixtures/typing-typeddict.pyi index 378570b4c19c..e398dff3fc6b 100644 --- a/test-data/unit/fixtures/typing-typeddict.pyi +++ b/test-data/unit/fixtures/typing-typeddict.pyi @@ -25,6 +25,7 @@ TypedDict = 0 NoReturn = 0 Required = 0 NotRequired = 0 +Self = 0 T = TypeVar('T') T_co = TypeVar('T_co', covariant=True) From 7f91f3feb112818a3b091fc26385e4fd617eb325 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Mon, 7 Nov 2022 23:39:08 +0000 Subject: [PATCH 03/25] More fixes; more tests --- mypy/checker.py | 8 +++- mypy/checkmember.py | 5 +-- mypy/expandtype.py | 19 +++++++- mypy/subtypes.py | 6 +-- test-data/unit/check-incremental.test | 2 + test-data/unit/check-selftype.test | 62 +++++++++++++++++++++++++-- test-data/unit/fine-grained.test | 2 + 7 files changed, 93 insertions(+), 11 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index 8973ade98228..c53e6c974b41 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -39,7 +39,7 @@ from mypy.erasetype import erase_type, erase_typevars, remove_instance_last_known_values from mypy.errorcodes import TYPE_VAR, UNUSED_AWAITABLE, UNUSED_COROUTINE, ErrorCode from mypy.errors import Errors, ErrorWatcher, report_internal_error -from mypy.expandtype import expand_type, expand_type_by_instance +from mypy.expandtype import expand_self_type, expand_type, expand_type_by_instance from mypy.join import join_types from mypy.literals import Key, literal, literal_hash from mypy.maptype import map_instance_to_supertype @@ -2468,6 +2468,10 @@ class C(B, A[int]): ... # this is unsafe because... second_sig = self.bind_and_map_method(second, second_type, ctx, base2) ok = is_subtype(first_sig, second_sig, ignore_pos_arg_names=True) elif first_type and second_type: + if isinstance(first.node, Var): + first_type = expand_self_type(first.node, first_type, fill_typevars(base1)) + if isinstance(second.node, Var): + second_type = expand_self_type(second.node, second_type, fill_typevars(base2)) ok = is_equivalent(first_type, second_type) if not ok: second_node = base2[name].node @@ -3048,6 +3052,8 @@ def lvalue_type_from_base( if base_var: base_node = base_var.node base_type = base_var.type + if isinstance(base_node, Var) and base_type is not None: + base_type = expand_self_type(base_node, base_type, fill_typevars(base)) if isinstance(base_node, Decorator): base_node = base_node.func base_type = base_node.type diff --git a/mypy/checkmember.py b/mypy/checkmember.py index 75b5f2ac3fc2..f092c4998d4f 100644 --- a/mypy/checkmember.py +++ b/mypy/checkmember.py @@ -6,7 +6,7 @@ from mypy import meet, message_registry, subtypes from mypy.erasetype import erase_typevars -from mypy.expandtype import expand_type, expand_type_by_instance, freshen_function_type_vars +from mypy.expandtype import expand_self_type, expand_type_by_instance, freshen_function_type_vars from mypy.maptype import map_instance_to_supertype from mypy.messages import MessageBuilder from mypy.nodes import ( @@ -723,8 +723,7 @@ def analyze_var( if mx.is_lvalue and var.is_classvar: mx.msg.cant_assign_to_classvar(name, mx.context) t = get_proper_type(expand_type_by_instance(typ, itype)) - if var.info.self_type is not None and not var.is_property: - t = expand_type(t, {var.info.self_type.id: mx.original_type}) + t = get_proper_type(expand_self_type(var, t, mx.original_type)) result: Type = t typ = get_proper_type(typ) if ( diff --git a/mypy/expandtype.py b/mypy/expandtype.py index 08bc216689fb..5a56857e1114 100644 --- a/mypy/expandtype.py +++ b/mypy/expandtype.py @@ -2,7 +2,7 @@ from typing import Iterable, Mapping, Sequence, TypeVar, cast, overload -from mypy.nodes import ARG_STAR +from mypy.nodes import ARG_STAR, Var from mypy.types import ( AnyType, CallableType, @@ -383,3 +383,20 @@ def expand_unpack_with_variables( raise NotImplementedError(f"Invalid type replacement to expand: {repl}") else: raise NotImplementedError(f"Invalid type to expand: {t.type}") + + +@overload +def expand_self_type(var: Var, typ: ProperType, replacement: ProperType) -> ProperType: + ... + + +@overload +def expand_self_type(var: Var, typ: Type, replacement: Type) -> Type: + ... + + +def expand_self_type(var: Var, typ: Type, replacement: Type) -> Type: + """Expand appearances of Self type in a variable type.""" + if var.info.self_type is not None and not var.is_property: + return expand_type(typ, {var.info.self_type.id: replacement}) + return typ diff --git a/mypy/subtypes.py b/mypy/subtypes.py index c0bcf63b2bd2..1898ea7c9a21 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -8,7 +8,7 @@ import mypy.constraints import mypy.typeops from mypy.erasetype import erase_type -from mypy.expandtype import expand_type, expand_type_by_instance +from mypy.expandtype import expand_self_type, expand_type_by_instance from mypy.maptype import map_instance_to_supertype # Circular import; done in the function instead. @@ -1194,8 +1194,8 @@ def find_node_type( ) else: typ = node.type - if typ is not None and node.info.self_type is not None and not node.is_property: - typ = expand_type(typ, {node.info.self_type.id: subtype}) + if typ is not None: + typ = expand_self_type(node, typ, subtype) p_typ = get_proper_type(typ) if typ is None: return AnyType(TypeOfAny.from_error) diff --git a/test-data/unit/check-incremental.test b/test-data/unit/check-incremental.test index 3ec0ed2c63f5..7b8f9a7e18f7 100644 --- a/test-data/unit/check-incremental.test +++ b/test-data/unit/check-incremental.test @@ -6286,3 +6286,5 @@ class C: ... [out] [out2] [out3] + +[case testTypingSelfCoarse] diff --git a/test-data/unit/check-selftype.test b/test-data/unit/check-selftype.test index 37ab6d594fe3..64ef7a015f6e 100644 --- a/test-data/unit/check-selftype.test +++ b/test-data/unit/check-selftype.test @@ -1281,11 +1281,67 @@ C.attr # E: Access to generic instance variables via class is ambiguous reveal_type(D().meth()) # N: Revealed type is "builtins.list[__main__.D]" reveal_type(D().attr) # N: Revealed type is "builtins.list[__main__.D]" --- Test various errors +[case testTypingSelfInvalid] +from typing import Self, Callable --- Test all nodes: property, classmethod, overload, ..., and both incrementals +var: Self # E: Self type is only allowed in annotations within class definition +reveal_type(var) # N: Revealed type is "Any" +def foo() -> Self: ... # E: Self type is only allowed in annotations within class definition +reveal_type(foo) # N: Revealed type is "def () -> Any" +bad: Callable[[Self], Self] # E: Self type is only allowed in annotations within class definition +reveal_type(bad) # N: Revealed type is "def (Any) -> Any" --- Test Self nested in a generic alias +def func() -> None: + var: Self # E: Self type is only allowed in annotations within class definition + +class C(Self): ... # E: Self type is only allowed in annotations within class definition +Alias = Callable[[Self], Self] # E: Self type is only allowed in annotations within class definition +a: Alias +reveal_type(a) # N: Revealed type is "def (Any) -> Any" + +[case testTypingSelfProperty] + +[case testTypingSelfClassMethod] + +[case testTypingSelfOverload] + +[case testTypingSelfNestedInAlias] +from typing import Generic, Self, TypeVar, List, Tuple + +T = TypeVar("T") +Pairs = List[Tuple[T, T]] + +class C(Generic[T]): + def pairs(self) -> Pairs[Self]: ... +class D(C[T]): ... +reveal_type(D[int]().pairs()) # N: Revealed type is "builtins.list[Tuple[__main__.D[builtins.int], __main__.D[builtins.int]]]" +[builtins fixtures/tuple.pyi] + +[case testTypingSelfOverrideVar] +from typing import Self + +class C: + x: Self + +class D(C): + x: D +class Bad(C): + x: int # E: Incompatible types in assignment (expression has type "int", base class "C" defined the type as "C") + +[case testTypingSelfOverrideVarMulti] +from typing import Self + +class C: + x: Self +class D: + x: int +class E: + x: C + +class Bad(D, C): # E: Definition of "x" in base class "D" is incompatible with definition in base class "C" + ... +class Good(E, C): + ... [case testTypingSelfMixedTypeVars] from typing import Self, TypeVar, Generic, Tuple diff --git a/test-data/unit/fine-grained.test b/test-data/unit/fine-grained.test index 32c4ff2eecf0..2704417ed793 100644 --- a/test-data/unit/fine-grained.test +++ b/test-data/unit/fine-grained.test @@ -10130,3 +10130,5 @@ b.py:2: error: "int" not callable a.py:1: error: Unsupported operand types for + ("int" and "str") 1 + '' ^~ + +[case testTypingSelfFine] From 80a11f656934fac1b4706de563330269d748a773 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Tue, 8 Nov 2022 11:11:22 +0000 Subject: [PATCH 04/25] Improve a test --- test-data/unit/check-selftype.test | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/test-data/unit/check-selftype.test b/test-data/unit/check-selftype.test index 64ef7a015f6e..e8a575e67294 100644 --- a/test-data/unit/check-selftype.test +++ b/test-data/unit/check-selftype.test @@ -1318,15 +1318,16 @@ reveal_type(D[int]().pairs()) # N: Revealed type is "builtins.list[Tuple[__main [builtins fixtures/tuple.pyi] [case testTypingSelfOverrideVar] -from typing import Self +from typing import Self, TypeVar, Generic -class C: +T = TypeVar("T") +class C(Generic[T]): x: Self -class D(C): +class D(C[int]): x: D -class Bad(C): - x: int # E: Incompatible types in assignment (expression has type "int", base class "C" defined the type as "C") +class Bad(C[int]): + x: int # E: Incompatible types in assignment (expression has type "int", base class "C" defined the type as "C[int]") [case testTypingSelfOverrideVarMulti] from typing import Self From 2eb0db1b96b1a34a22256bb866cc6ad416be87c7 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Tue, 8 Nov 2022 17:06:42 +0000 Subject: [PATCH 05/25] More tests few more fixes --- mypy/checkmember.py | 8 ++-- mypy/semanal.py | 7 +++- mypy/typeanal.py | 2 +- mypy/types.py | 6 ++- test-data/unit/check-selftype.test | 60 ++++++++++++++++++++++++++++-- 5 files changed, 71 insertions(+), 12 deletions(-) diff --git a/mypy/checkmember.py b/mypy/checkmember.py index f092c4998d4f..94b5e2b6a8ed 100644 --- a/mypy/checkmember.py +++ b/mypy/checkmember.py @@ -682,12 +682,12 @@ def analyze_descriptor_access(descriptor_type: Type, mx: MemberContext) -> Type: return inferred_dunder_get_type.ret_type -def is_instance_var(var: Var, info: TypeInfo) -> bool: +def is_instance_var(var: Var) -> bool: """Return if var is an instance variable according to PEP 526.""" return ( # check the type_info node is the var (not a decorated function, etc.) - var.name in info.names - and info.names[var.name].node is var + var.name in var.info.names + and var.info.names[var.name].node is var and not var.is_classvar # variables without annotations are treated as classvar and not var.is_inferred @@ -728,7 +728,7 @@ def analyze_var( typ = get_proper_type(typ) if ( var.is_initialized_in_class - and (not is_instance_var(var, info) or mx.is_operator) + and (not is_instance_var(var) or mx.is_operator) and isinstance(typ, FunctionLike) and not typ.is_type_obj() ): diff --git a/mypy/semanal.py b/mypy/semanal.py index 5f0a0979e860..4d2301687645 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -942,8 +942,11 @@ def prepare_method_signature(self, func: FuncDef, info: TypeInfo, has_self_type: if func.is_class or func.name == "__new__": leading_type = self.class_type(leading_type) func.type = replace_implicit_first_type(functype, leading_type) - elif has_self_type: - self.fail("Method cannot have explicit self annotation and Self type", func) + elif has_self_type and isinstance(func.unanalyzed_type, CallableType): + if not isinstance(get_proper_type(func.unanalyzed_type.arg_types[0]), AnyType): + self.fail( + "Method cannot have explicit self annotation and Self type", func + ) elif has_self_type: self.fail("Static methods cannot use Self type", func) diff --git a/mypy/typeanal.py b/mypy/typeanal.py index 3b1fc9272d2f..d244b6ae5162 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -590,7 +590,7 @@ def try_analyze_special_unbound_type(self, t: UnboundType, fullname: str) -> Typ if self.api.type is None or self.api.type.self_type is None: self.fail("Self type is only allowed in annotations within class definition", t) return AnyType(TypeOfAny.from_error) - return self.api.type.self_type + return self.api.type.self_type.copy_modified(line=t.line, column=t.column) return None def get_omitted_any(self, typ: Type, fullname: str | None = None) -> AnyType: diff --git a/mypy/types.py b/mypy/types.py index e322cf02505f..cc845784da79 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -524,6 +524,8 @@ def copy_modified( values: Bogus[list[Type]] = _dummy, upper_bound: Bogus[Type] = _dummy, id: Bogus[TypeVarId | int] = _dummy, + line: Bogus[int] = _dummy, + column: Bogus[int] = _dummy, ) -> TypeVarType: return TypeVarType( self.name, @@ -532,8 +534,8 @@ def copy_modified( self.values if values is _dummy else values, self.upper_bound if upper_bound is _dummy else upper_bound, self.variance, - self.line, - self.column, + self.line if line is _dummy else line, + self.column if column is _dummy else column, ) def accept(self, visitor: TypeVisitor[T]) -> T: diff --git a/test-data/unit/check-selftype.test b/test-data/unit/check-selftype.test index e8a575e67294..470461ffa957 100644 --- a/test-data/unit/check-selftype.test +++ b/test-data/unit/check-selftype.test @@ -445,15 +445,15 @@ reveal_type(B().ft()) # N: Revealed type is "Tuple[builtins.int, builtins.int, [builtins fixtures/property.pyi] [case testSelfTypeProperSupertypeAttributeMeta] -from typing import Callable, TypeVar, Type +from typing import Callable, TypeVar, Type, ClassVar T = TypeVar('T') class A(type): @property def g(cls: object) -> int: return 0 @property def gt(cls: T) -> T: return cls - f: Callable[[object], int] - ft: Callable[[T], T] + f: ClassVar[Callable[[object], int]] + ft: ClassVar[Callable[[T], T]] class B(A): pass @@ -1299,11 +1299,65 @@ Alias = Callable[[Self], Self] # E: Self type is only allowed in annotations wi a: Alias reveal_type(a) # N: Revealed type is "def (Any) -> Any" +[case testTypingSelfConflict] +from typing import Self, TypeVar, Tuple + +T = TypeVar("T") +class C: + def meth(self: T) -> Tuple[Self, T]: ... # E: Method cannot have explicit self annotation and Self type +reveal_type(C().meth()) # N: Revealed type is "Tuple[, __main__.C]" +[builtins fixtures/property.pyi] + [case testTypingSelfProperty] +from typing import Self, Tuple +class C: + @property + def attr(self) -> Tuple[Self, ...]: ... +class D(C): ... + +reveal_type(D().attr) # N: Revealed type is "builtins.tuple[__main__.D, ...]" +[builtins fixtures/property.pyi] + +[case testTypingSelfCallableVar] +from typing import Self, Callable + +class C: + x: Callable[[Self], Self] + def meth(self) -> Callable[[Self], Self]: ... +class D(C): ... + +reveal_type(C().x) # N: Revealed type is "def (__main__.C) -> __main__.C" +reveal_type(D().x) # N: Revealed type is "def (__main__.D) -> __main__.D" +reveal_type(D().meth()) # N: Revealed type is "def (__main__.D) -> __main__.D" [case testTypingSelfClassMethod] +from typing import Self + +class C: + @classmethod + def meth(cls) -> Self: ... + @staticmethod + def bad() -> Self: ... # E: Static methods cannot use Self type \ + # E: A function returning TypeVar should receive at least one argument containing the same TypeVar \ + # N: Consider using the upper bound "C" instead + +class D(C): ... +reveal_type(D.meth()) # N: Revealed type is "__main__.D" +reveal_type(D.bad()) # N: Revealed type is "" +[builtins fixtures/classmethod.pyi] [case testTypingSelfOverload] +from typing import Self, overload, Union + +class C: + @overload + def foo(self, other: Self) -> Self: ... + @overload + def foo(self, other: int) -> int: ... + def foo(self, other: Union[Self, int]) -> Union[Self, int]: + return other +class D(C): ... +reveal_type(D().foo) # N: Revealed type is "Overload(def (other: __main__.D) -> __main__.D, def (other: builtins.int) -> builtins.int)" [case testTypingSelfNestedInAlias] from typing import Generic, Self, TypeVar, List, Tuple From 535d9363c0066428d82ec59ef40771c10581fac9 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Tue, 8 Nov 2022 19:40:52 +0000 Subject: [PATCH 06/25] Add incremental tests --- test-data/unit/check-incremental.test | 20 ++++++++++++++++++++ test-data/unit/fine-grained.test | 21 +++++++++++++++++++++ 2 files changed, 41 insertions(+) diff --git a/test-data/unit/check-incremental.test b/test-data/unit/check-incremental.test index 7b8f9a7e18f7..5a37463ab8d7 100644 --- a/test-data/unit/check-incremental.test +++ b/test-data/unit/check-incremental.test @@ -6288,3 +6288,23 @@ class C: ... [out3] [case testTypingSelfCoarse] +import m +[file lib.py] +from typing import Self + +class C: + def meth(self, other: Self) -> Self: ... + +[file m.py] +import lib +class D: ... +[file m.py.2] +import lib +class D(lib.C): ... + +reveal_type(D.meth) +reveal_type(D().meth) +[out] +[out2] +tmp/m.py:4: note: Revealed type is "def [Self <: lib.C] (self: Self`-1, other: Self`-1) -> Self`-1" +tmp/m.py:5: note: Revealed type is "def (other: m.D) -> m.D" diff --git a/test-data/unit/fine-grained.test b/test-data/unit/fine-grained.test index 2704417ed793..8a8c0a3532ed 100644 --- a/test-data/unit/fine-grained.test +++ b/test-data/unit/fine-grained.test @@ -10132,3 +10132,24 @@ a.py:1: error: Unsupported operand types for + ("int" and "str") ^~ [case testTypingSelfFine] +import m +[file lib.py] +from typing import Any + +class C: + def meth(self, other: Any) -> C: ... +[file lib.py.2] +from typing import Self + +class C: + def meth(self, other: Self) -> Self: ... + +[file m.py] +import lib +class D(lib.C): + def meth(self, other: int) -> D: ... +[out] +== +m.py:3: error: Argument 1 of "meth" is incompatible with supertype "C"; supertype defines the argument type as "D" +m.py:3: note: This violates the Liskov substitution principle +m.py:3: note: See https://mypy.readthedocs.io/en/stable/common_issues.html#incompatible-overrides From ca7c7e82ac35410162cbb2cbb1176185bd3b0301 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Tue, 8 Nov 2022 20:27:18 +0000 Subject: [PATCH 07/25] Add docs --- docs/source/generics.rst | 50 ++++++++++++++++++++++++++++++++-------- 1 file changed, 40 insertions(+), 10 deletions(-) diff --git a/docs/source/generics.rst b/docs/source/generics.rst index 3ae616f78691..45b2e4b97406 100644 --- a/docs/source/generics.rst +++ b/docs/source/generics.rst @@ -264,15 +264,8 @@ Generic methods and generic self You can also define generic methods — just use a type variable in the method signature that is different from class type variables. In particular, ``self`` may also be generic, allowing a method to return the most precise -type known at the point of access. - -.. note:: - - This feature is experimental. Checking code with type annotations for self - arguments is still not fully implemented. Mypy may disallow valid code or - allow unsafe code. - -In this way, for example, you can typecheck chaining of setter methods: +type known at the point of access. In this way, for example, you can typecheck +chaining of setter methods: .. code-block:: python @@ -335,6 +328,43 @@ possibly by making use of the ``Any`` type. For some advanced uses of self-types see :ref:`additional examples `. +Automatic self types using typing.Self +************************************** + +The patterns described above are quite common, so there is a syntactic sugar +for them introduced in :pep:`673`. Instead of defining a type variable and +using an explicit ``self`` annotation, you can import a magic type ``typing.Self`` +that is automatically transformed into a type variable with an upper bound of +current class, and you don't need an annotation for ``self`` (or ``cls`` for +class methods). The above example can thus be rewritten as: + +.. code-block:: python + + from typing import Self + + class Friend: + other: Self | None = None + + @classmethod + def make_pair(cls) -> tuple[Self, Self]: + a, b = cls(), cls() + a.other = b + b.other = a + return a, b + + class SuperFriend(Friend): + pass + + a, b = SuperFriend.make_pair() + +This is more compact than using explicit type variables, plus additionally +you can use ``Self`` in attribute annotations, not just in methods. + +.. note:: + + This feature is available on Python 3.11 or newer. On older Python versions + you can import backported ``Self`` from latest ``typing_extensions``. + .. _variance-of-generics: Variance of generic types @@ -548,7 +578,7 @@ Note that class decorators are handled differently than function decorators in mypy: decorating a class does not erase its type, even if the decorator has incomplete type annotations. -Suppose we have the following decorator, not type annotated yet, +Suppose we have the following decorator, not type annotated yet, that preserves the original function's signature and merely prints the decorated function's name: .. code-block:: python From 2ee66ec731a72c8fd59f5269edfb50f4c5326ad1 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Tue, 8 Nov 2022 20:53:09 +0000 Subject: [PATCH 08/25] Minor cleanup --- mypy/plugins/dataclasses.py | 3 +-- mypy/semanal.py | 9 ++++++++- mypy/semanal_namedtuple.py | 4 ++-- mypy/semanal_typeddict.py | 4 ++-- mypy/typeanal.py | 2 +- test-data/unit/check-selftype.test | 8 ++++++++ 6 files changed, 22 insertions(+), 8 deletions(-) diff --git a/mypy/plugins/dataclasses.py b/mypy/plugins/dataclasses.py index 430b34ffd636..75496d5e56f9 100644 --- a/mypy/plugins/dataclasses.py +++ b/mypy/plugins/dataclasses.py @@ -52,10 +52,9 @@ TypeVarType, get_proper_type, ) - -# The set of decorators that generate dataclasses. from mypy.typevars import fill_typevars +# The set of decorators that generate dataclasses. dataclass_makers: Final = {"dataclass", "dataclasses.dataclass"} # The set of functions that generate dataclass fields. field_makers: Final = {"dataclasses.field"} diff --git a/mypy/semanal.py b/mypy/semanal.py index 4d2301687645..9c08810bb50d 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -987,6 +987,12 @@ def update_function_type_variables(self, fun_type: CallableType, defn: FuncItem) return has_self_type def setup_self_type(self) -> None: + """Setup a (shared) Self type variable for current class. + + We intentionally don't add it to the class symbol table, + so it can be accessed only by mypy and will not cause + clashes with user defined names. + """ assert self.type is not None info = self.type if info.self_type is not None: @@ -3142,7 +3148,8 @@ def process_type_annotation(self, s: AssignmentStmt) -> None: s.type, lambda name: self.lookup_qualified(name, s, suppress_errors=True) ) if has_self_type and self.type: - if self.type.typeddict_type or self.type.tuple_type: + if self.type.typeddict_type or self.type.is_named_tuple: + # Annotations have special meaning in TypedDicts and NamedTuples. return self.setup_self_type() analyzed = self.anal_type(s.type, allow_tuple_literal=allow_tuple_literal) diff --git a/mypy/semanal_namedtuple.py b/mypy/semanal_namedtuple.py index 1f6f7653b8dd..82db27417eae 100644 --- a/mypy/semanal_namedtuple.py +++ b/mypy/semanal_namedtuple.py @@ -420,7 +420,7 @@ def parse_namedtuple_args( return items, types, defaults, typename, tvar_defs, True def parse_namedtuple_fields_with_types( - self, nodes: list[Expression], call: CallExpr, existing_info: TypeInfo | None + self, nodes: list[Expression], context: Context, existing_info: TypeInfo | None ) -> tuple[list[str], list[Type], list[Expression], bool] | None: """Parse typed named tuple fields. @@ -449,7 +449,7 @@ def parse_namedtuple_fields_with_types( type, allow_placeholder=not self.options.disable_recursive_aliases and not self.api.is_func_scope(), - self_type_override=special_self_type(existing_info, call), + self_type_override=special_self_type(existing_info, context), ) # Workaround #4987 and avoid introducing a bogus UnboundType if isinstance(analyzed, UnboundType): diff --git a/mypy/semanal_typeddict.py b/mypy/semanal_typeddict.py index 46959b6867fb..47375c9a2776 100644 --- a/mypy/semanal_typeddict.py +++ b/mypy/semanal_typeddict.py @@ -463,7 +463,7 @@ def parse_typeddict_args( def parse_typeddict_fields_with_types( self, dict_items: list[tuple[Expression | None, Expression]], - call: CallExpr, + context: Context, existing_info: TypeInfo | None, ) -> tuple[list[str], list[Type], bool] | None: """Parse typed dict items passed as pairs (name expression, type expression). @@ -506,7 +506,7 @@ def parse_typeddict_fields_with_types( allow_required=True, allow_placeholder=not self.options.disable_recursive_aliases and not self.api.is_func_scope(), - self_type_override=special_self_type(existing_info, call), + self_type_override=special_self_type(existing_info, context), ) if analyzed is None: return None diff --git a/mypy/typeanal.py b/mypy/typeanal.py index d244b6ae5162..7f175e807288 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -235,7 +235,7 @@ def __init__( # Names of type aliases encountered while analysing a type will be collected here. self.aliases_used: set[str] = set() # Used by special forms like TypedDicts and NamedTuples, where Self type - # needs a special handling (for such types the target TypeInfo not be + # needs a special handling (for such types the target TypeInfo may not be # created yet when analyzing Self type, unlike for regular classes). self.self_type_override = self_type_override diff --git a/test-data/unit/check-selftype.test b/test-data/unit/check-selftype.test index 470461ffa957..f490db2d0b3e 100644 --- a/test-data/unit/check-selftype.test +++ b/test-data/unit/check-selftype.test @@ -1424,3 +1424,11 @@ class D(C): ... reveal_type(C) # N: Revealed type is "def (other: __main__.C) -> __main__.C" reveal_type(D) # N: Revealed type is "def (other: __main__.D) -> __main__.D" + +[case testTypingSelfCorrectName] +from typing import Self, List + +class C: + Self = List[C] + def meth(self) -> Self: ... +reveal_type(C.meth) # N: Revealed type is "def (self: __main__.C) -> builtins.list[__main__.C]" From ccb74a78cdb7707a902564fa4d01b5ec8e918d49 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Tue, 8 Nov 2022 21:27:23 +0000 Subject: [PATCH 09/25] Fix self-compilation --- mypy/semanal.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index 9c08810bb50d..e83de3994387 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -340,7 +340,7 @@ class SemanticAnalyzer( # Nested block depths of scopes block_depth: list[int] # TypeInfo of directly enclosing class (or None) - type: TypeInfo | None = None + _type: TypeInfo | None = None # Stack of outer classes (the second tuple item contains tvars). type_stack: list[TypeInfo | None] # Type variables bound by the current scope, be it class or function @@ -419,7 +419,7 @@ def __init__( FuncItem | GeneratorExpr | DictionaryComprehension, SymbolTable ] = {} self.imports = set() - self.type = None + self._type = None self.type_stack = [] # Are the namespaces of classes being processed complete? self.incomplete_type_stack: list[bool] = [] @@ -459,6 +459,10 @@ def __init__( # mypyc doesn't properly handle implementing an abstractproperty # with a regular attribute so we make them properties + @property + def type(self) -> TypeInfo | None: + return self._type + @property def is_stub_file(self) -> bool: return self._is_stub_file @@ -772,7 +776,7 @@ def file_context( if active_type: scope.leave_class() self.leave_class() - self.type = None + self._type = None self.incomplete_type_stack.pop() del self.options @@ -1682,7 +1686,7 @@ def enter_class(self, info: TypeInfo) -> None: self.locals.append(None) # Add class scope self.is_comprehension_stack.append(False) self.block_depth.append(-1) # The class body increments this to 0 - self.type = info + self._type = info self.missing_names.append(set()) def leave_class(self) -> None: @@ -1690,7 +1694,7 @@ def leave_class(self) -> None: self.block_depth.pop() self.locals.pop() self.is_comprehension_stack.pop() - self.type = self.type_stack.pop() + self._type = self.type_stack.pop() self.missing_names.pop() def analyze_class_decorator(self, defn: ClassDef, decorator: Expression) -> None: From 504fe2c2cdda593dd75453978c48c16198b54b61 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Tue, 8 Nov 2022 23:29:15 +0000 Subject: [PATCH 10/25] Best effort support for unusual locations for self --- mypy/typeanal.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/mypy/typeanal.py b/mypy/typeanal.py index 7f175e807288..976444770bdc 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -89,6 +89,7 @@ get_proper_type, ) from mypy.typetraverser import TypeTraverserVisitor +from mypy.typevars import fill_typevars T = TypeVar("T") @@ -587,10 +588,16 @@ def try_analyze_special_unbound_type(self, t: UnboundType, fullname: str) -> Typ # For various special forms that can't be inherited but use Self # for convenience we eagerly replace Self. return self.self_type_override - if self.api.type is None or self.api.type.self_type is None: + if self.api.type is None: self.fail("Self type is only allowed in annotations within class definition", t) return AnyType(TypeOfAny.from_error) - return self.api.type.self_type.copy_modified(line=t.line, column=t.column) + if self.api.type.self_type is not None: + return self.api.type.self_type.copy_modified(line=t.line, column=t.column) + # Attributes and methods are handled above, this is best effort support for + # other things: simply use the current class instead of Self. This may be + # useful for e.g. cast(Self, ...) etc, to avoid repeating long class name. + # TODO: can we have more case by case logic here? + return fill_typevars(self.api.type) return None def get_omitted_any(self, typ: Type, fullname: str | None = None) -> AnyType: From 1a9996177b2a25c5df18473c8c63609bf5d32d3a Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Wed, 9 Nov 2022 00:29:10 +0000 Subject: [PATCH 11/25] Some cleanups --- mypy/checkexpr.py | 5 +++++ mypy/checkmember.py | 7 +++++- mypy/typeanal.py | 2 ++ test-data/unit/check-selftype.test | 34 ++++++++++++++++++++++++++++++ 4 files changed, 47 insertions(+), 1 deletion(-) diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index ac16f9c9c813..4d4f74fceb22 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -2666,6 +2666,10 @@ def analyze_ordinary_member_access(self, e: MemberExpr, is_lvalue: bool) -> Type if isinstance(base, RefExpr) and isinstance(base.node, MypyFile): module_symbol_table = base.node.names + if isinstance(base, RefExpr) and isinstance(base.node, Var): + is_self = base.node.is_self + else: + is_self = False member_type = analyze_member_access( e.name, @@ -2679,6 +2683,7 @@ def analyze_ordinary_member_access(self, e: MemberExpr, is_lvalue: bool) -> Type chk=self.chk, in_literal_context=self.is_literal_context(), module_symbol_table=module_symbol_table, + is_self=is_self, ) return member_type diff --git a/mypy/checkmember.py b/mypy/checkmember.py index 94b5e2b6a8ed..6df1ba4d4a2b 100644 --- a/mypy/checkmember.py +++ b/mypy/checkmember.py @@ -90,6 +90,7 @@ def __init__( self_type: Type | None, module_symbol_table: SymbolTable | None = None, no_deferral: bool = False, + is_self: bool = False, ) -> None: self.is_lvalue = is_lvalue self.is_super = is_super @@ -101,6 +102,7 @@ def __init__( self.chk = chk self.module_symbol_table = module_symbol_table self.no_deferral = no_deferral + self.is_self = is_self def named_type(self, name: str) -> Instance: return self.chk.named_type(name) @@ -152,6 +154,7 @@ def analyze_member_access( self_type: Type | None = None, module_symbol_table: SymbolTable | None = None, no_deferral: bool = False, + is_self: bool = False, ) -> Type: """Return the type of attribute 'name' of 'typ'. @@ -187,6 +190,7 @@ def analyze_member_access( self_type=self_type, module_symbol_table=module_symbol_table, no_deferral=no_deferral, + is_self=is_self, ) result = _analyze_member_access(name, typ, mx, override_info) possible_literal = get_proper_type(result) @@ -723,7 +727,8 @@ def analyze_var( if mx.is_lvalue and var.is_classvar: mx.msg.cant_assign_to_classvar(name, mx.context) t = get_proper_type(expand_type_by_instance(typ, itype)) - t = get_proper_type(expand_self_type(var, t, mx.original_type)) + if not (mx.is_self or mx.is_super): + t = get_proper_type(expand_self_type(var, t, mx.original_type)) result: Type = t typ = get_proper_type(typ) if ( diff --git a/mypy/typeanal.py b/mypy/typeanal.py index 976444770bdc..51c44b7b483a 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -591,6 +591,8 @@ def try_analyze_special_unbound_type(self, t: UnboundType, fullname: str) -> Typ if self.api.type is None: self.fail("Self type is only allowed in annotations within class definition", t) return AnyType(TypeOfAny.from_error) + if self.api.type.has_base("builtins.type"): + self.fail("Self type cannot be used in a metaclass", t) if self.api.type.self_type is not None: return self.api.type.self_type.copy_modified(line=t.line, column=t.column) # Attributes and methods are handled above, this is best effort support for diff --git a/test-data/unit/check-selftype.test b/test-data/unit/check-selftype.test index f490db2d0b3e..7199f36e87df 100644 --- a/test-data/unit/check-selftype.test +++ b/test-data/unit/check-selftype.test @@ -1432,3 +1432,37 @@ class C: Self = List[C] def meth(self) -> Self: ... reveal_type(C.meth) # N: Revealed type is "def (self: __main__.C) -> builtins.list[__main__.C]" + +[case testTypingSelfClassVar] +from typing import Self, ClassVar + +class C: + DEFAULT: ClassVar[Self] +reveal_type(C.DEFAULT) # E: Access to generic class variables is ambiguous \ + # N: Revealed type is "Any" + +[case testTypingSelfMetaClassDisabled] +from typing import Self + +class Meta(type): + def meth(cls) -> Self: ... # E: Self type cannot be used in a metaclass + +[case testTypingSelfNonAnnotationUses] +from typing import Self, List, cast + +class C: + A = List[Self] + B = cast(Self, ...) + def meth(self) -> A: ... + +class D(C): ... +reveal_type(D().meth()) # N: Revealed type is "builtins.list[__main__.C]" +reveal_type(D().B) # N: Revealed type is "__main__.C" + +[case testTypingSelfInternalSafe] +from typing import Self + +class C: + x: Self + def __init__(self, x: C) -> None: + self.x = x # E: Incompatible types in assignment (expression has type "C", variable has type "Self") From ce8d34556c07563591b8a33012ff82a80577b7e1 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Wed, 9 Nov 2022 00:55:25 +0000 Subject: [PATCH 12/25] Enable ClassVar (to some safe extent) --- mypy/checkmember.py | 8 +++++++- mypy/types.py | 11 +++++++---- test-data/unit/check-selftype.test | 10 +++++++--- 3 files changed, 21 insertions(+), 8 deletions(-) diff --git a/mypy/checkmember.py b/mypy/checkmember.py index 6df1ba4d4a2b..63cfc1d1226d 100644 --- a/mypy/checkmember.py +++ b/mypy/checkmember.py @@ -951,7 +951,12 @@ def analyze_class_attribute_access( # x: T # C.x # Error, ambiguous access # C[int].x # Also an error, since C[int] is same as C at runtime - if isinstance(t, TypeVarType) or has_type_vars(t): + # Exception is Self type wrapped in ClassVar, that is safe. + if node.node.info.self_type is not None and node.node.is_classvar: + exclude = {node.node.info.self_type.id} + else: + exclude = set() + if isinstance(t, TypeVarType) and t.id not in exclude or has_type_vars(t, exclude): # Exception: access on Type[...], including first argument of class methods is OK. if not isinstance(get_proper_type(mx.original_type), TypeType) or node.implicit: if node.node.is_classvar: @@ -964,6 +969,7 @@ def analyze_class_attribute_access( # In the above example this means that we infer following types: # C.x -> Any # C[int].x -> int + t = get_proper_type(expand_self_type(node.node, t, itype)) t = erase_typevars(expand_type_by_instance(t, isuper)) is_classmethod = (is_decorated and cast(Decorator, node.node).func.is_class) or ( diff --git a/mypy/types.py b/mypy/types.py index cc845784da79..53c818cd91c1 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -3236,16 +3236,19 @@ def replace_alias_tvars( class HasTypeVars(TypeQuery[bool]): - def __init__(self) -> None: + def __init__(self, exclude: set[TypeVarId] | None = None) -> None: super().__init__(any) + if exclude is None: + exclude = set() + self.exclude = exclude def visit_type_var(self, t: TypeVarType) -> bool: - return True + return t.id not in self.exclude -def has_type_vars(typ: Type) -> bool: +def has_type_vars(typ: Type, exclude: set[TypeVarId] | None = None) -> bool: """Check if a type contains any type variables (recursively).""" - return typ.accept(HasTypeVars()) + return typ.accept(HasTypeVars(exclude)) class HasRecursiveType(TypeQuery[bool]): diff --git a/test-data/unit/check-selftype.test b/test-data/unit/check-selftype.test index 7199f36e87df..4c4db9da4143 100644 --- a/test-data/unit/check-selftype.test +++ b/test-data/unit/check-selftype.test @@ -1434,12 +1434,16 @@ class C: reveal_type(C.meth) # N: Revealed type is "def (self: __main__.C) -> builtins.list[__main__.C]" [case testTypingSelfClassVar] -from typing import Self, ClassVar +from typing import Self, ClassVar, Generic, TypeVar class C: DEFAULT: ClassVar[Self] -reveal_type(C.DEFAULT) # E: Access to generic class variables is ambiguous \ - # N: Revealed type is "Any" +reveal_type(C.DEFAULT) # N: Revealed type is "__main__.C" + +T = TypeVar("T") +class G(Generic[T]): + BAD: ClassVar[Self] # E: ClassVar cannot contain type variables +reveal_type(G.BAD) # N: Revealed type is "__main__.G[Any]" [case testTypingSelfMetaClassDisabled] from typing import Self From 324eff2bef6b076340f67add43f77dee49cea5a4 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Wed, 9 Nov 2022 23:08:31 +0000 Subject: [PATCH 13/25] Allow redundant Self by default; add error code --- docs/source/error_code_list2.rst | 19 +++++++++++++ mypy/errorcodes.py | 6 ++++ mypy/semanal.py | 37 +++++++++++++++++++++++-- test-data/unit/check-selftype.test | 44 ++++++++++++++++++++++++++++++ 4 files changed, 103 insertions(+), 3 deletions(-) diff --git a/docs/source/error_code_list2.rst b/docs/source/error_code_list2.rst index cac19e705361..5c586be93449 100644 --- a/docs/source/error_code_list2.rst +++ b/docs/source/error_code_list2.rst @@ -82,6 +82,25 @@ Example: # Error: Redundant cast to "int" [redundant-cast] return cast(int, x) +Check that methods do not have redundant Self annotations [redundant-self] +-------------------------------------------------------------------------- + +Such annotations are allowed by :pep:`673` but are redundant, so if you want +warnings about them, enable this error code. + +Example: + +.. code-block:: python + + # mypy: enable-error-code="redundant-self" + + from typing import Self + + class C: + # Error: Redundant Self annotation on method first argument + def copy(self: Self) -> Self: + return type(self)() + Check that comparisons are overlapping [comparison-overlap] ----------------------------------------------------------- diff --git a/mypy/errorcodes.py b/mypy/errorcodes.py index f2a74c332b2e..767b8ad7ff12 100644 --- a/mypy/errorcodes.py +++ b/mypy/errorcodes.py @@ -180,6 +180,12 @@ def __str__(self) -> str: "General", default_enabled=False, ) +REDUNDANT_SELF_TYPE = ErrorCode( + "redundant-self", + "Warn about redundant Self type annotations on method first argument", + "General", + default_enabled=False, +) # Syntax errors are often blocking. diff --git a/mypy/semanal.py b/mypy/semanal.py index e83de3994387..9f4219791348 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -219,6 +219,7 @@ from mypy.semanal_typeddict import TypedDictAnalyzer from mypy.tvar_scope import TypeVarLikeScope from mypy.typeanal import ( + SELF_TYPE_NAMES, TypeAnalyser, TypeVarLikeList, TypeVarLikeQuery, @@ -948,12 +949,42 @@ def prepare_method_signature(self, func: FuncDef, info: TypeInfo, has_self_type: func.type = replace_implicit_first_type(functype, leading_type) elif has_self_type and isinstance(func.unanalyzed_type, CallableType): if not isinstance(get_proper_type(func.unanalyzed_type.arg_types[0]), AnyType): - self.fail( - "Method cannot have explicit self annotation and Self type", func - ) + if self.is_expected_self_type( + self_type, func.is_class or func.name == "__new__" + ): + # This error is off by default, since it is explicitly allowed + # by the PEP 673. + self.fail( + "Redundant Self annotation on method first argument", + func, + code=codes.REDUNDANT_SELF_TYPE, + ) + else: + self.fail( + "Method cannot have explicit self annotation and Self type", func + ) elif has_self_type: self.fail("Static methods cannot use Self type", func) + def is_expected_self_type(self, typ: Type, is_classmethod: bool) -> bool: + """Does this (analyzed or not) type represent the expected Self type for a method?""" + assert self.type is not None + typ = get_proper_type(typ) + if is_classmethod: + if isinstance(typ, TypeType): + return self.is_expected_self_type(typ.item, is_classmethod=False) + if isinstance(typ, UnboundType): + sym = self.lookup_qualified(typ.name, typ, suppress_errors=True) + if sym is not None and sym.fullname == "typing.Type" and typ.args: + return self.is_expected_self_type(typ.args[0], is_classmethod=False) + return False + if isinstance(typ, TypeVarType): + return typ == self.type.self_type + if isinstance(typ, UnboundType): + sym = self.lookup_qualified(typ.name, typ, suppress_errors=True) + return sym is not None and sym.fullname in SELF_TYPE_NAMES + return False + def set_original_def(self, previous: Node | None, new: FuncDef | Decorator) -> bool: """If 'new' conditionally redefine 'previous', set 'previous' as original diff --git a/test-data/unit/check-selftype.test b/test-data/unit/check-selftype.test index 4c4db9da4143..057dc3820166 100644 --- a/test-data/unit/check-selftype.test +++ b/test-data/unit/check-selftype.test @@ -1470,3 +1470,47 @@ class C: x: Self def __init__(self, x: C) -> None: self.x = x # E: Incompatible types in assignment (expression has type "C", variable has type "Self") + +[case testTypingSelfRedundantAllowed] +from typing import Self, Type + +class C: + def f(self: Self) -> Self: + d: Defer + class Defer: ... + return self + + @classmethod + def g(cls: Type[Self]) -> Self: + d: DeferAgain + class DeferAgain: ... + return cls() +[builtins fixtures/classmethod.pyi] + +[case testTypingSelfRedundantWarning] +# mypy: enable-error-code="redundant-self" + +from typing import Self, Type + +class C: + def copy(self: Self) -> Self: # E: Redundant Self annotation on method first argument + d: Defer + class Defer: ... + return self + + @classmethod + def g(cls: Type[Self]) -> Self: # E: Redundant Self annotation on method first argument + d: DeferAgain + class DeferAgain: ... + return cls() +[builtins fixtures/classmethod.pyi] + +-- Handle unusual locations consistently using a flag + +-- Self in cast and type assert should be a Self (and probably actually everywhere) + +-- Always prohibit in type aliases + +-- Protocol test with ClassVar default + +-- NamedTuple with Self (class)method From d96cfdc7e136b3480a652557283648dbae79eeed Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Wed, 9 Nov 2022 23:16:51 +0000 Subject: [PATCH 14/25] Prohibit Self with arguments --- mypy/typeanal.py | 2 ++ test-data/unit/check-selftype.test | 10 +++++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/mypy/typeanal.py b/mypy/typeanal.py index 51c44b7b483a..ddfd881bb483 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -584,6 +584,8 @@ def try_analyze_special_unbound_type(self, t: UnboundType, fullname: str) -> Typ return AnyType(TypeOfAny.from_error) return UnpackType(self.anal_type(t.args[0]), line=t.line, column=t.column) elif fullname in SELF_TYPE_NAMES: + if t.args: + self.fail("Self type cannot have type arguments", t) if self.self_type_override is not None: # For various special forms that can't be inherited but use Self # for convenience we eagerly replace Self. diff --git a/test-data/unit/check-selftype.test b/test-data/unit/check-selftype.test index 057dc3820166..4a59b2507217 100644 --- a/test-data/unit/check-selftype.test +++ b/test-data/unit/check-selftype.test @@ -1281,7 +1281,7 @@ C.attr # E: Access to generic instance variables via class is ambiguous reveal_type(D().meth()) # N: Revealed type is "builtins.list[__main__.D]" reveal_type(D().attr) # N: Revealed type is "builtins.list[__main__.D]" -[case testTypingSelfInvalid] +[case testTypingSelfInvalidLocations] from typing import Self, Callable var: Self # E: Self type is only allowed in annotations within class definition @@ -1299,6 +1299,14 @@ Alias = Callable[[Self], Self] # E: Self type is only allowed in annotations wi a: Alias reveal_type(a) # N: Revealed type is "def (Any) -> Any" +[case testTypingSelfInvalidArgs] +from typing import Self, List + +class C: + x: Self[int] # E: Self type cannot have type arguments + def meth(self) -> List[Self[int]]: # E: Self type cannot have type arguments + ... + [case testTypingSelfConflict] from typing import Self, TypeVar, Tuple From 0b953cf64de7da4a96a32a88f351a00763d6627e Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Thu, 10 Nov 2022 01:28:07 +0000 Subject: [PATCH 15/25] Address CR; minor cleanups --- docs/source/generics.rst | 27 +++++++++++++++++++++++++-- mypy/checker.py | 6 +++--- mypy/checkmember.py | 6 +++--- mypy/semanal_shared.py | 6 ++++++ mypy/types.py | 8 +++----- test-data/unit/check-dataclasses.test | 3 +++ test-data/unit/check-protocols.test | 21 +++++++++++++++++++++ test-data/unit/check-selftype.test | 15 ++++++++++----- test-data/unit/fine-grained.test | 14 ++++++++------ 9 files changed, 82 insertions(+), 24 deletions(-) diff --git a/docs/source/generics.rst b/docs/source/generics.rst index 45b2e4b97406..3d512239d82e 100644 --- a/docs/source/generics.rst +++ b/docs/source/generics.rst @@ -326,6 +326,29 @@ or a deserialization method returns the actual type of self. Therefore you may need to silence mypy inside these methods (but not at the call site), possibly by making use of the ``Any`` type. +Note that this feature may accept some unsafe code for the purpose of +*practicality*. For example: + +.. code-block:: python + + from typing import TypeVar + + T = TypeVar("T") + class Base: + def compare(self: T, other: T) -> bool: + return False + + class Sub(Base): + def __init__(self, x: int) -> None: + self.x = x + # This is unsafe (see below), but allowed because it is + # a common pattern, and rarely causes issues in practice. + def compare(self, other: Sub) -> bool: + return self.x > other.x + + b: Base = Sub(42) + b.compare(Base()) # Runtime error here: 'Base' object has no attribute 'x' + For some advanced uses of self-types see :ref:`additional examples `. Automatic self types using typing.Self @@ -362,8 +385,8 @@ you can use ``Self`` in attribute annotations, not just in methods. .. note:: - This feature is available on Python 3.11 or newer. On older Python versions - you can import backported ``Self`` from latest ``typing_extensions``. + To use this feature on versions of Python before 3.11, you will need to + import ``Self`` from ``typing_extensions`` version 4.0 or newer. .. _variance-of-generics: diff --git a/mypy/checker.py b/mypy/checker.py index c53e6c974b41..abc3b8158b4d 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -2469,9 +2469,9 @@ class C(B, A[int]): ... # this is unsafe because... ok = is_subtype(first_sig, second_sig, ignore_pos_arg_names=True) elif first_type and second_type: if isinstance(first.node, Var): - first_type = expand_self_type(first.node, first_type, fill_typevars(base1)) + first_type = expand_self_type(first.node, first_type, fill_typevars(ctx)) if isinstance(second.node, Var): - second_type = expand_self_type(second.node, second_type, fill_typevars(base2)) + second_type = expand_self_type(second.node, second_type, fill_typevars(ctx)) ok = is_equivalent(first_type, second_type) if not ok: second_node = base2[name].node @@ -3053,7 +3053,7 @@ def lvalue_type_from_base( base_node = base_var.node base_type = base_var.type if isinstance(base_node, Var) and base_type is not None: - base_type = expand_self_type(base_node, base_type, fill_typevars(base)) + base_type = expand_self_type(base_node, base_type, fill_typevars(expr_node.info)) if isinstance(base_node, Decorator): base_node = base_node.func base_type = base_node.type diff --git a/mypy/checkmember.py b/mypy/checkmember.py index 63cfc1d1226d..5c4bd4e33d2e 100644 --- a/mypy/checkmember.py +++ b/mypy/checkmember.py @@ -953,10 +953,10 @@ def analyze_class_attribute_access( # C[int].x # Also an error, since C[int] is same as C at runtime # Exception is Self type wrapped in ClassVar, that is safe. if node.node.info.self_type is not None and node.node.is_classvar: - exclude = {node.node.info.self_type.id} + exclude = node.node.info.self_type.id else: - exclude = set() - if isinstance(t, TypeVarType) and t.id not in exclude or has_type_vars(t, exclude): + exclude = None + if isinstance(t, TypeVarType) and t.id != exclude or has_type_vars(t, exclude): # Exception: access on Type[...], including first argument of class methods is OK. if not isinstance(get_proper_type(mx.original_type), TypeType) or node.implicit: if node.node.is_classvar: diff --git a/mypy/semanal_shared.py b/mypy/semanal_shared.py index 25b0374d491d..cb65dfa44366 100644 --- a/mypy/semanal_shared.py +++ b/mypy/semanal_shared.py @@ -334,6 +334,12 @@ def has_placeholder(typ: Type) -> bool: def special_self_type(info: TypeInfo | None, defn: Context) -> Type: + """Eager expansion of Self type in special forms like NamedTuple and TypedDict. + + We special case them because: + * They need special handling because TypeInfo may not exist yet when it is analyzed. + * Attribute/item types TypedDict and NamedTuple can't be overriden. + """ if info is not None: assert info.special_alias is not None return TypeAliasType(info.special_alias, list(info.defn.type_vars)) diff --git a/mypy/types.py b/mypy/types.py index 53c818cd91c1..285826e34047 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -3236,17 +3236,15 @@ def replace_alias_tvars( class HasTypeVars(TypeQuery[bool]): - def __init__(self, exclude: set[TypeVarId] | None = None) -> None: + def __init__(self, exclude: TypeVarId | None = None) -> None: super().__init__(any) - if exclude is None: - exclude = set() self.exclude = exclude def visit_type_var(self, t: TypeVarType) -> bool: - return t.id not in self.exclude + return t.id != self.exclude -def has_type_vars(typ: Type, exclude: set[TypeVarId] | None = None) -> bool: +def has_type_vars(typ: Type, exclude: TypeVarId | None = None) -> bool: """Check if a type contains any type variables (recursively).""" return typ.accept(HasTypeVars(exclude)) diff --git a/test-data/unit/check-dataclasses.test b/test-data/unit/check-dataclasses.test index d9f4e9c17c2b..c651a751511c 100644 --- a/test-data/unit/check-dataclasses.test +++ b/test-data/unit/check-dataclasses.test @@ -1945,6 +1945,9 @@ class LinkedList(Generic[T]): value: T next: Optional[Self] = None + def meth(self) -> None: + reveal_type(self.next) # N: Revealed type is "Union[Self`-1, None]" + l_int: LinkedList[int] = LinkedList(1, LinkedList("no", None)) # E: Argument 1 to "LinkedList" has incompatible type "str"; expected "int" @dataclass diff --git a/test-data/unit/check-protocols.test b/test-data/unit/check-protocols.test index b686a0b7ef8b..bf44d120c5a9 100644 --- a/test-data/unit/check-protocols.test +++ b/test-data/unit/check-protocols.test @@ -3863,3 +3863,24 @@ z: P reveal_type(S().next) # N: Revealed type is "__main__.S" reveal_type(z.next) # N: Revealed type is "__main__.P" [builtins fixtures/property.pyi] + +[case testProtocolSelfTypeNewSyntaxSubProtocol] +from typing import Protocol, Self + +class P(Protocol): + @property + def next(self) -> Self: ... +class PS(P, Protocol): + @property + def other(self) -> Self: ... + +class C: + next: C + other: C +class S: + next: Self + other: Self + +x: PS = C() +y: PS = S() +[builtins fixtures/property.pyi] diff --git a/test-data/unit/check-selftype.test b/test-data/unit/check-selftype.test index 4a59b2507217..6c806d1af9b6 100644 --- a/test-data/unit/check-selftype.test +++ b/test-data/unit/check-selftype.test @@ -1274,6 +1274,11 @@ from typing import Self, List class C: attr: List[Self] def meth(self) -> List[Self]: ... + def test(self) -> Self: + if bool(): + return C() # E: Incompatible return value type (got "C", expected "Self") + else: + return self class D(C): ... reveal_type(C.meth) # N: Revealed type is "def [Self <: __main__.C] (self: Self`-1) -> builtins.list[Self`-1]" @@ -1389,7 +1394,7 @@ class C(Generic[T]): class D(C[int]): x: D class Bad(C[int]): - x: int # E: Incompatible types in assignment (expression has type "int", base class "C" defined the type as "C[int]") + x: C[int] # E: Incompatible types in assignment (expression has type "C[int]", base class "C" defined the type as "Bad") [case testTypingSelfOverrideVarMulti] from typing import Self @@ -1397,9 +1402,9 @@ from typing import Self class C: x: Self class D: - x: int -class E: x: C +class E: + x: Good class Bad(D, C): # E: Definition of "x" in base class "D" is incompatible with definition in base class "C" ... @@ -1513,11 +1518,11 @@ class C: return cls() [builtins fixtures/classmethod.pyi] --- Handle unusual locations consistently using a flag +-- Handle unusual locations consistently (w.r.t. presence of Self annotaions) -- Self in cast and type assert should be a Self (and probably actually everywhere) --- Always prohibit in type aliases +-- Always prohibit in type aliases (PEP says so) -- Protocol test with ClassVar default diff --git a/test-data/unit/fine-grained.test b/test-data/unit/fine-grained.test index 8a8c0a3532ed..2fa3deadd16c 100644 --- a/test-data/unit/fine-grained.test +++ b/test-data/unit/fine-grained.test @@ -10144,12 +10144,14 @@ from typing import Self class C: def meth(self, other: Self) -> Self: ... -[file m.py] +[file n.py] import lib -class D(lib.C): - def meth(self, other: int) -> D: ... +class D(lib.C): ... +[file m.py] +from n import D +d = D() +def test() -> None: + d.meth(42) [out] == -m.py:3: error: Argument 1 of "meth" is incompatible with supertype "C"; supertype defines the argument type as "D" -m.py:3: note: This violates the Liskov substitution principle -m.py:3: note: See https://mypy.readthedocs.io/en/stable/common_issues.html#incompatible-overrides +m.py:4: error: Argument 1 to "meth" of "C" has incompatible type "int"; expected "D" From 582980459f3e579bdd6da0bbc941330b60202822 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Thu, 10 Nov 2022 12:35:26 +0000 Subject: [PATCH 16/25] Prohibit unclear cases; some more tests --- mypy/semanal.py | 11 +++----- mypy/semanal_namedtuple.py | 5 ++-- mypy/semanal_shared.py | 17 +---------- mypy/semanal_typeddict.py | 6 ++-- mypy/typeanal.py | 15 ++++------ test-data/unit/check-namedtuple.test | 42 ++++++++++++++-------------- test-data/unit/check-protocols.test | 10 +++++++ test-data/unit/check-selftype.test | 15 +++------- test-data/unit/check-typeddict.test | 28 ++++--------------- 9 files changed, 57 insertions(+), 92 deletions(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index 9f4219791348..b34ce8e6be43 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -3183,9 +3183,6 @@ def process_type_annotation(self, s: AssignmentStmt) -> None: s.type, lambda name: self.lookup_qualified(name, s, suppress_errors=True) ) if has_self_type and self.type: - if self.type.typeddict_type or self.type.is_named_tuple: - # Annotations have special meaning in TypedDicts and NamedTuples. - return self.setup_self_type() analyzed = self.anal_type(s.type, allow_tuple_literal=allow_tuple_literal) # Don't store not ready types (including placeholders). @@ -6187,7 +6184,7 @@ def type_analyzer( allow_required: bool = False, allow_param_spec_literals: bool = False, report_invalid_types: bool = True, - self_type_override: Type | None = None, + prohibit_self_type: str | None = None, ) -> TypeAnalyser: if tvar_scope is None: tvar_scope = self.tvar_scope @@ -6203,7 +6200,7 @@ def type_analyzer( allow_placeholder=allow_placeholder, allow_required=allow_required, allow_param_spec_literals=allow_param_spec_literals, - self_type_override=self_type_override, + prohibit_self_type=prohibit_self_type, ) tpan.in_dynamic_func = bool(self.function_stack and self.function_stack[-1].is_dynamic()) tpan.global_scope = not self.type and not self.function_stack @@ -6223,7 +6220,7 @@ def anal_type( allow_required: bool = False, allow_param_spec_literals: bool = False, report_invalid_types: bool = True, - self_type_override: Type | None = None, + prohibit_self_type: str | None = None, third_pass: bool = False, ) -> Type | None: """Semantically analyze a type. @@ -6254,7 +6251,7 @@ def anal_type( allow_required=allow_required, allow_param_spec_literals=allow_param_spec_literals, report_invalid_types=report_invalid_types, - self_type_override=self_type_override, + prohibit_self_type=prohibit_self_type, ) tag = self.track_incomplete_refs() typ = typ.accept(a) diff --git a/mypy/semanal_namedtuple.py b/mypy/semanal_namedtuple.py index 82db27417eae..0af30e25613c 100644 --- a/mypy/semanal_namedtuple.py +++ b/mypy/semanal_namedtuple.py @@ -48,7 +48,6 @@ calculate_tuple_fallback, has_placeholder, set_callable_name, - special_self_type, ) from mypy.types import ( TYPED_NAMEDTUPLE_NAMES, @@ -179,7 +178,7 @@ def check_namedtuple_classdef( stmt.type, allow_placeholder=not self.options.disable_recursive_aliases and not self.api.is_func_scope(), - self_type_override=special_self_type(existing_info, defn), + prohibit_self_type="NamedTuple item type", ) if analyzed is None: # Something is incomplete. We need to defer this named tuple. @@ -449,7 +448,7 @@ def parse_namedtuple_fields_with_types( type, allow_placeholder=not self.options.disable_recursive_aliases and not self.api.is_func_scope(), - self_type_override=special_self_type(existing_info, context), + prohibit_self_type="NamedTuple item type", ) # Workaround #4987 and avoid introducing a bogus UnboundType if isinstance(analyzed, UnboundType): diff --git a/mypy/semanal_shared.py b/mypy/semanal_shared.py index cb65dfa44366..ee9218f02b3e 100644 --- a/mypy/semanal_shared.py +++ b/mypy/semanal_shared.py @@ -33,7 +33,6 @@ ProperType, TupleType, Type, - TypeAliasType, TypeVarId, TypeVarLikeType, get_proper_type, @@ -168,7 +167,7 @@ def anal_type( allow_required: bool = False, allow_placeholder: bool = False, report_invalid_types: bool = True, - self_type_override: Type | None = None, + prohibit_self_type: str | None = None, ) -> Type | None: raise NotImplementedError @@ -331,17 +330,3 @@ def visit_placeholder_type(self, t: PlaceholderType) -> bool: def has_placeholder(typ: Type) -> bool: """Check if a type contains any placeholder types (recursively).""" return typ.accept(HasPlaceholders()) - - -def special_self_type(info: TypeInfo | None, defn: Context) -> Type: - """Eager expansion of Self type in special forms like NamedTuple and TypedDict. - - We special case them because: - * They need special handling because TypeInfo may not exist yet when it is analyzed. - * Attribute/item types TypedDict and NamedTuple can't be overriden. - """ - if info is not None: - assert info.special_alias is not None - return TypeAliasType(info.special_alias, list(info.defn.type_vars)) - else: - return PlaceholderType(None, [], defn.line) diff --git a/mypy/semanal_typeddict.py b/mypy/semanal_typeddict.py index 47375c9a2776..7e0b6f82ab0c 100644 --- a/mypy/semanal_typeddict.py +++ b/mypy/semanal_typeddict.py @@ -31,7 +31,7 @@ TypeInfo, ) from mypy.options import Options -from mypy.semanal_shared import SemanticAnalyzerInterface, has_placeholder, special_self_type +from mypy.semanal_shared import SemanticAnalyzerInterface, has_placeholder from mypy.typeanal import check_for_explicit_any, has_any_from_unimported_type from mypy.types import ( TPDICT_NAMES, @@ -307,7 +307,7 @@ def analyze_typeddict_classdef_fields( allow_required=True, allow_placeholder=not self.options.disable_recursive_aliases and not self.api.is_func_scope(), - self_type_override=special_self_type(existing_info, defn), + prohibit_self_type="TypedDict item type", ) if analyzed is None: return None, [], [], set() # Need to defer @@ -506,7 +506,7 @@ def parse_typeddict_fields_with_types( allow_required=True, allow_placeholder=not self.options.disable_recursive_aliases and not self.api.is_func_scope(), - self_type_override=special_self_type(existing_info, context), + prohibit_self_type="TypedDict item type", ) if analyzed is None: return None diff --git a/mypy/typeanal.py b/mypy/typeanal.py index ddfd881bb483..2920d9fd609b 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -151,6 +151,7 @@ def analyze_type_alias( is_typeshed_stub, defining_alias=True, allow_placeholder=allow_placeholder, + prohibit_self_type="type alias target", ) analyzer.in_dynamic_func = in_dynamic_func analyzer.global_scope = global_scope @@ -199,7 +200,7 @@ def __init__( allow_required: bool = False, allow_param_spec_literals: bool = False, report_invalid_types: bool = True, - self_type_override: Type | None = None, + prohibit_self_type: str | None = None, ) -> None: self.api = api self.lookup_qualified = api.lookup_qualified @@ -235,10 +236,7 @@ def __init__( self.is_typeshed_stub = is_typeshed_stub # Names of type aliases encountered while analysing a type will be collected here. self.aliases_used: set[str] = set() - # Used by special forms like TypedDicts and NamedTuples, where Self type - # needs a special handling (for such types the target TypeInfo may not be - # created yet when analyzing Self type, unlike for regular classes). - self.self_type_override = self_type_override + self.prohibit_self_type = prohibit_self_type def visit_unbound_type(self, t: UnboundType, defining_literal: bool = False) -> Type: typ = self.visit_unbound_type_nonoptional(t, defining_literal) @@ -586,10 +584,9 @@ def try_analyze_special_unbound_type(self, t: UnboundType, fullname: str) -> Typ elif fullname in SELF_TYPE_NAMES: if t.args: self.fail("Self type cannot have type arguments", t) - if self.self_type_override is not None: - # For various special forms that can't be inherited but use Self - # for convenience we eagerly replace Self. - return self.self_type_override + if self.prohibit_self_type is not None: + self.fail(f"Self type cannot be used in {self.prohibit_self_type}", t) + return AnyType(TypeOfAny.from_error) if self.api.type is None: self.fail("Self type is only allowed in annotations within class definition", t) return AnyType(TypeOfAny.from_error) diff --git a/test-data/unit/check-namedtuple.test b/test-data/unit/check-namedtuple.test index 82d3add825bb..4eda14c2c592 100644 --- a/test-data/unit/check-namedtuple.test +++ b/test-data/unit/check-namedtuple.test @@ -1307,31 +1307,31 @@ class C( [builtins fixtures/tuple.pyi] [typing fixtures/typing-namedtuple.pyi] -[case testNamedTupleSelfTypeNewSyntaxGeneric] -# flags: --strict-optional -from typing import Self, NamedTuple, Optional, Generic, TypeVar - -T = TypeVar("T") -class NT(NamedTuple, Generic[T]): - val: T - next: Optional[Self] - -reveal_type(NT) # N: Revealed type is "def [T] (val: T`1, next: Union[Tuple[T`1, Union[..., None], fallback=__main__.NT[T`1]], None]) -> Tuple[T`1, Union[Tuple[T`1, Union[..., None], fallback=__main__.NT[T`1]], None], fallback=__main__.NT[T`1]]" -n: NT[int] -reveal_type(n.next) # N: Revealed type is "Union[Tuple[builtins.int, Union[..., None], fallback=__main__.NT[builtins.int]], None]" -reveal_type(n[1]) # N: Revealed type is "Union[Tuple[builtins.int, Union[..., None], fallback=__main__.NT[builtins.int]], None]" +[case testNamedTupleSelfItemNotAllowed] +from typing import Self, NamedTuple, Optional -m: NT[int] = NT(1, NT("no", None)) # E: Argument 1 to "NT" has incompatible type "str"; expected "int" +class NT(NamedTuple): + val: int + next: Optional[Self] # E: Self type cannot be used in NamedTuple item type +NTC = NamedTuple("NTC", [("val", int), ("next", Optional[Self])]) # E: Self type cannot be used in NamedTuple item type [builtins fixtures/tuple.pyi] [typing fixtures/typing-namedtuple.pyi] -[case testNamedTupleSelfTypeNewSyntaxCall] -# flags: --strict-optional -from typing import Self, NamedTuple, Optional +[case testNamedTupleTypingSelfMethod] +from typing import Self, NamedTuple, TypeVar, Generic -NTC = NamedTuple("NTC", [("val", int), ("next", Optional[Self])]) -nc: NTC -reveal_type(nc.next) # N: Revealed type is "Union[Tuple[builtins.int, Union[..., None], fallback=__main__.NTC], None]" -reveal_type(nc[1]) # N: Revealed type is "Union[Tuple[builtins.int, Union[..., None], fallback=__main__.NTC], None]" +T = TypeVar("T") +class NT(NamedTuple, Generic[T]): + key: str + val: T + def meth(self) -> Self: + nt: NT[int] + if bool(): + return nt._replace() # E: Incompatible return value type (got "NT[int]", expected "Self") + else: + return self._replace() + +class SNT(NT[int]): ... +reveal_type(SNT("test", 42).meth()) # N: Revealed type is "Tuple[builtins.str, builtins.int, fallback=__main__.SNT]" [builtins fixtures/tuple.pyi] [typing fixtures/typing-namedtuple.pyi] diff --git a/test-data/unit/check-protocols.test b/test-data/unit/check-protocols.test index bf44d120c5a9..7ce4001d5f36 100644 --- a/test-data/unit/check-protocols.test +++ b/test-data/unit/check-protocols.test @@ -3884,3 +3884,13 @@ class S: x: PS = C() y: PS = S() [builtins fixtures/property.pyi] + +[case testProtocolClassVarSelfType] +from typing import ClassVar, Self, Protocol + +class P(Protocol): + DEFAULT: ClassVar[Self] +class C: + DEFAULT: ClassVar[C] + +x: P = C() diff --git a/test-data/unit/check-selftype.test b/test-data/unit/check-selftype.test index 6c806d1af9b6..1229b7c1fc7d 100644 --- a/test-data/unit/check-selftype.test +++ b/test-data/unit/check-selftype.test @@ -1291,8 +1291,10 @@ from typing import Self, Callable var: Self # E: Self type is only allowed in annotations within class definition reveal_type(var) # N: Revealed type is "Any" + def foo() -> Self: ... # E: Self type is only allowed in annotations within class definition reveal_type(foo) # N: Revealed type is "def () -> Any" + bad: Callable[[Self], Self] # E: Self type is only allowed in annotations within class definition reveal_type(bad) # N: Revealed type is "def (Any) -> Any" @@ -1300,9 +1302,6 @@ def func() -> None: var: Self # E: Self type is only allowed in annotations within class definition class C(Self): ... # E: Self type is only allowed in annotations within class definition -Alias = Callable[[Self], Self] # E: Self type is only allowed in annotations within class definition -a: Alias -reveal_type(a) # N: Revealed type is "def (Any) -> Any" [case testTypingSelfInvalidArgs] from typing import Self, List @@ -1468,12 +1467,12 @@ class Meta(type): from typing import Self, List, cast class C: - A = List[Self] + A = List[Self] # E: Self type cannot be used in type alias target B = cast(Self, ...) def meth(self) -> A: ... class D(C): ... -reveal_type(D().meth()) # N: Revealed type is "builtins.list[__main__.C]" +reveal_type(D().meth()) # N: Revealed type is "builtins.list[Any]" reveal_type(D().B) # N: Revealed type is "__main__.C" [case testTypingSelfInternalSafe] @@ -1521,9 +1520,3 @@ class C: -- Handle unusual locations consistently (w.r.t. presence of Self annotaions) -- Self in cast and type assert should be a Self (and probably actually everywhere) - --- Always prohibit in type aliases (PEP says so) - --- Protocol test with ClassVar default - --- NamedTuple with Self (class)method diff --git a/test-data/unit/check-typeddict.test b/test-data/unit/check-typeddict.test index 469c82fb159d..24521062a5d4 100644 --- a/test-data/unit/check-typeddict.test +++ b/test-data/unit/check-typeddict.test @@ -2591,29 +2591,13 @@ TD[str]({"key": 0, "value": 0}) # E: Incompatible types (expression has type "i [builtins fixtures/dict.pyi] [typing fixtures/typing-typeddict.pyi] -[case testTypedDictSelfTypeNewSyntaxGeneric] -# flags: --strict-optional -from typing import Self, TypedDict, Optional, Generic, TypeVar - -T = TypeVar("T") -class TD(TypedDict, Generic[T]): - val: T - next: Optional[Self] - -reveal_type(TD) # N: Revealed type is "def [T] (*, val: T`1, next: Union[TypedDict('__main__.TD', {'val': T`1, 'next': Union[..., None]}), None]) -> TypedDict('__main__.TD', {'val': T`1, 'next': Union[TypedDict('__main__.TD', {'val': T`1, 'next': Union[..., None]}), None]})" -t: TD[int] -reveal_type(t["next"]) # N: Revealed type is "Union[TypedDict('__main__.TD', {'val': builtins.int, 'next': Union[..., None]}), None]" - -s: TD[int] = TD(val=1, next=TD(val="no", next=None)) # E: Incompatible types (expression has type "str", TypedDict item "val" has type "int") -[builtins fixtures/dict.pyi] -[typing fixtures/typing-typeddict.pyi] - -[case testTypedDictSelfTypeNewSyntaxCall] -# flags: --strict-optional +[case testTypedDictSelfItemNotAllowed] from typing import Self, TypedDict, Optional -TDC = TypedDict("TDC", {"val": int, "next": Optional[Self]}) -tc: TDC -reveal_type(tc["next"]) # N: Revealed type is "Union[TypedDict('__main__.TDC', {'val': builtins.int, 'next': Union[..., None]}), None]" +class TD(TypedDict): + val: int + next: Optional[Self] # E: Self type cannot be used in TypedDict item type +TDC = TypedDict("TDC", {"val": int, "next": Optional[Self]}) # E: Self type cannot be used in TypedDict item type + [builtins fixtures/dict.pyi] [typing fixtures/typing-typeddict.pyi] From 3ec47b95ccd2eb3402cec664638aca2ea8d0c2a9 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Thu, 10 Nov 2022 13:27:16 +0000 Subject: [PATCH 17/25] Make ClassVar in generics better --- mypy/message_registry.py | 1 + mypy/semanal.py | 16 ++++++++++----- test-data/unit/check-selftype.test | 33 ++++++++++++++++++++++++++---- 3 files changed, 41 insertions(+), 9 deletions(-) diff --git a/mypy/message_registry.py b/mypy/message_registry.py index c84ce120dbda..b2a68306ae14 100644 --- a/mypy/message_registry.py +++ b/mypy/message_registry.py @@ -231,6 +231,7 @@ def with_additional_msg(self, info: str) -> ErrorMessage: "variable" ) CLASS_VAR_WITH_TYPEVARS: Final = "ClassVar cannot contain type variables" +CLASS_VAR_WITH_GENERIC_SELF: Final = "ClassVar cannot contain Self type in generic classes" CLASS_VAR_OUTSIDE_OF_CLASS: Final = "ClassVar can only be used for assignments in class body" # Protocol diff --git a/mypy/semanal.py b/mypy/semanal.py index b34ce8e6be43..38acfea40b73 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -3179,11 +3179,6 @@ def process_type_annotation(self, s: AssignmentStmt) -> None: if s.type: lvalue = s.lvalues[-1] allow_tuple_literal = isinstance(lvalue, TupleExpr) - has_self_type = find_self_type( - s.type, lambda name: self.lookup_qualified(name, s, suppress_errors=True) - ) - if has_self_type and self.type: - self.setup_self_type() analyzed = self.anal_type(s.type, allow_tuple_literal=allow_tuple_literal) # Don't store not ready types (including placeholders). if analyzed is None or has_placeholder(analyzed): @@ -4148,6 +4143,12 @@ def check_classvar(self, s: AssignmentStmt) -> None: # See https://github.com/python/mypy/issues/11538 self.fail(message_registry.CLASS_VAR_WITH_TYPEVARS, s) + if ( + analyzed is not None + and self.type.self_type in get_type_vars(analyzed) + and self.type.defn.type_vars + ): + self.fail(message_registry.CLASS_VAR_WITH_GENERIC_SELF, s) elif not isinstance(lvalue, MemberExpr) or self.is_self_member_ref(lvalue): # In case of member access, report error only when assigning to self # Other kinds of member assignments should be already reported @@ -6243,6 +6244,11 @@ def anal_type( NOTE: The caller shouldn't defer even if this returns None or a placeholder type. """ + has_self_type = find_self_type( + typ, lambda name: self.lookup_qualified(name, typ, suppress_errors=True) + ) + if has_self_type and self.type and prohibit_self_type is None: + self.setup_self_type() a = self.type_analyzer( tvar_scope=tvar_scope, allow_unbound_tvars=allow_unbound_tvars, diff --git a/test-data/unit/check-selftype.test b/test-data/unit/check-selftype.test index 1229b7c1fc7d..dbb9c3c20033 100644 --- a/test-data/unit/check-selftype.test +++ b/test-data/unit/check-selftype.test @@ -1410,6 +1410,22 @@ class Bad(D, C): # E: Definition of "x" in base class "D" is incompatible with class Good(E, C): ... +[case testTypingSelfAlternativeGenericConstructor] +from typing import Self, Generic, TypeVar, Tuple + +T = TypeVar("T") +class C(Generic[T]): + def __init__(self, val: T) -> None: ... + @classmethod + def pair(cls, val: T) -> Tuple[Self, Self]: + return (cls(val), C(val)) # E: Incompatible return value type (got "Tuple[Self, C[Any]]", expected "Tuple[Self, Self]") + +class D(C[int]): pass +reveal_type(C.pair(42)) # N: Revealed type is "Tuple[__main__.C[builtins.int], __main__.C[builtins.int]]" +reveal_type(D.pair("no")) # N: Revealed type is "Tuple[__main__.D, __main__.D]" \ + # E: Argument 1 to "pair" of "C" has incompatible type "str"; expected "int" +[builtins fixtures/classmethod.pyi] + [case testTypingSelfMixedTypeVars] from typing import Self, TypeVar, Generic, Tuple @@ -1454,7 +1470,7 @@ reveal_type(C.DEFAULT) # N: Revealed type is "__main__.C" T = TypeVar("T") class G(Generic[T]): - BAD: ClassVar[Self] # E: ClassVar cannot contain type variables + BAD: ClassVar[Self] # E: ClassVar cannot contain Self type in generic classes reveal_type(G.BAD) # N: Revealed type is "__main__.G[Any]" [case testTypingSelfMetaClassDisabled] @@ -1473,7 +1489,7 @@ class C: class D(C): ... reveal_type(D().meth()) # N: Revealed type is "builtins.list[Any]" -reveal_type(D().B) # N: Revealed type is "__main__.C" +reveal_type(D().B) # N: Revealed type is "__main__.D" [case testTypingSelfInternalSafe] from typing import Self @@ -1517,6 +1533,15 @@ class C: return cls() [builtins fixtures/classmethod.pyi] --- Handle unusual locations consistently (w.r.t. presence of Self annotaions) +[case testTypingSelfAssertType] +from typing import Self, assert_type --- Self in cast and type assert should be a Self (and probably actually everywhere) +class C: + def foo(self) -> None: + assert_type(self, Self) # E: Expression is of type "C", not "Self" + assert_type(C(), Self) # E: Expression is of type "C", not "Self" + + def bar(self) -> Self: + assert_type(self, Self) # OK + assert_type(C(), Self) # E: Expression is of type "C", not "Self" + return self From 24dd6499d914eab4950a268bcca437d96bde60ed Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Thu, 10 Nov 2022 13:56:22 +0000 Subject: [PATCH 18/25] More cleanup --- docs/source/generics.rst | 1 + mypy/semanal_namedtuple.py | 26 ++++++++++++-------------- mypy/semanal_typeddict.py | 25 ++++++++++--------------- mypy/typeanal.py | 6 ++---- 4 files changed, 25 insertions(+), 33 deletions(-) diff --git a/docs/source/generics.rst b/docs/source/generics.rst index 3d512239d82e..59d4aa1a2dea 100644 --- a/docs/source/generics.rst +++ b/docs/source/generics.rst @@ -341,6 +341,7 @@ Note that this feature may accept some unsafe code for the purpose of class Sub(Base): def __init__(self, x: int) -> None: self.x = x + # This is unsafe (see below), but allowed because it is # a common pattern, and rarely causes issues in practice. def compare(self, other: Sub) -> bool: diff --git a/mypy/semanal_namedtuple.py b/mypy/semanal_namedtuple.py index 0af30e25613c..04308db99e63 100644 --- a/mypy/semanal_namedtuple.py +++ b/mypy/semanal_namedtuple.py @@ -107,16 +107,16 @@ def analyze_namedtuple_classdef( if isinstance(base_expr, RefExpr): self.api.accept(base_expr) if base_expr.fullname in TYPED_NAMEDTUPLE_NAMES: - existing_info = None - if isinstance(defn.analyzed, NamedTupleExpr): - existing_info = defn.analyzed.info - result = self.check_namedtuple_classdef(defn, is_stub_file, existing_info) + result = self.check_namedtuple_classdef(defn, is_stub_file) if result is None: # This is a valid named tuple, but some types are incomplete. return True, None items, types, default_items = result if is_func_scope and "@" not in defn.name: defn.name += "@" + str(defn.line) + existing_info = None + if isinstance(defn.analyzed, NamedTupleExpr): + existing_info = defn.analyzed.info info = self.build_namedtuple_typeinfo( defn.name, items, types, default_items, defn.line, existing_info ) @@ -129,7 +129,7 @@ def analyze_namedtuple_classdef( return False, None def check_namedtuple_classdef( - self, defn: ClassDef, is_stub_file: bool, existing_info: TypeInfo | None + self, defn: ClassDef, is_stub_file: bool ) -> tuple[list[str], list[Type], dict[str, Expression]] | None: """Parse and validate fields in named tuple class definition. @@ -230,10 +230,7 @@ def check_namedtuple( is_typed = True else: return None, None, [] - existing_info = None - if isinstance(node.analyzed, NamedTupleExpr): - existing_info = node.analyzed.info - result = self.parse_namedtuple_args(call, fullname, existing_info) + result = self.parse_namedtuple_args(call, fullname) if result: items, types, defaults, typename, tvar_defs, ok = result else: @@ -281,6 +278,9 @@ def check_namedtuple( else: default_items = {} + existing_info = None + if isinstance(node.analyzed, NamedTupleExpr): + existing_info = node.analyzed.info info = self.build_namedtuple_typeinfo( name, items, types, default_items, node.line, existing_info ) @@ -318,7 +318,7 @@ def store_namedtuple_info( call.analyzed.set_line(call) def parse_namedtuple_args( - self, call: CallExpr, fullname: str, existing_info: TypeInfo | None + self, call: CallExpr, fullname: str ) -> None | (tuple[list[str], list[Type], list[Expression], str, list[TypeVarLikeType], bool]): """Parse a namedtuple() call into data needed to construct a type. @@ -395,9 +395,7 @@ def parse_namedtuple_args( ] tvar_defs = self.api.get_and_bind_all_tvars(type_exprs) # The fields argument contains (name, type) tuples. - result = self.parse_namedtuple_fields_with_types( - listexpr.items, call, existing_info - ) + result = self.parse_namedtuple_fields_with_types(listexpr.items, call) if result is None: # One of the types is not ready, defer. return None @@ -419,7 +417,7 @@ def parse_namedtuple_args( return items, types, defaults, typename, tvar_defs, True def parse_namedtuple_fields_with_types( - self, nodes: list[Expression], context: Context, existing_info: TypeInfo | None + self, nodes: list[Expression], context: Context ) -> tuple[list[str], list[Type], list[Expression], bool] | None: """Parse typed named tuple fields. diff --git a/mypy/semanal_typeddict.py b/mypy/semanal_typeddict.py index 7e0b6f82ab0c..e8be82bd41be 100644 --- a/mypy/semanal_typeddict.py +++ b/mypy/semanal_typeddict.py @@ -94,9 +94,7 @@ def analyze_typeddict_classdef(self, defn: ClassDef) -> tuple[bool, TypeInfo | N and defn.base_type_exprs[0].fullname in TPDICT_NAMES ): # Building a new TypedDict - fields, types, statements, required_keys = self.analyze_typeddict_classdef_fields( - defn, existing_info - ) + fields, types, statements, required_keys = self.analyze_typeddict_classdef_fields(defn) if fields is None: return True, None # Defer info = self.build_typeddict_typeinfo( @@ -148,7 +146,7 @@ def analyze_typeddict_classdef(self, defn: ClassDef) -> tuple[bool, TypeInfo | N new_types, new_statements, new_required_keys, - ) = self.analyze_typeddict_classdef_fields(defn, existing_info, keys) + ) = self.analyze_typeddict_classdef_fields(defn, keys) if new_keys is None: return True, None # Defer keys.extend(new_keys) @@ -259,7 +257,7 @@ def map_items_to_base( return mapped_items def analyze_typeddict_classdef_fields( - self, defn: ClassDef, existing_info: TypeInfo | None, oldfields: list[str] | None = None + self, defn: ClassDef, oldfields: list[str] | None = None ) -> tuple[list[str] | None, list[Type], list[Statement], set[str]]: """Analyze fields defined in a TypedDict class definition. @@ -359,10 +357,7 @@ def check_typeddict( fullname = callee.fullname if fullname not in TPDICT_NAMES: return False, None, [] - existing_info = None - if isinstance(node.analyzed, TypedDictExpr): - existing_info = node.analyzed.info - res = self.parse_typeddict_args(call, existing_info) + res = self.parse_typeddict_args(call) if res is None: # This is a valid typed dict, but some type is not ready. # The caller should defer this until next iteration. @@ -392,6 +387,9 @@ def check_typeddict( types = [ # unwrap Required[T] to just T t.item if isinstance(t, RequiredType) else t for t in types ] + existing_info = None + if isinstance(node.analyzed, TypedDictExpr): + existing_info = node.analyzed.info info = self.build_typeddict_typeinfo( name, items, types, required_keys, call.line, existing_info ) @@ -406,7 +404,7 @@ def check_typeddict( return True, info, tvar_defs def parse_typeddict_args( - self, call: CallExpr, existing_info: TypeInfo | None + self, call: CallExpr ) -> tuple[str, list[str], list[Type], bool, list[TypeVarLikeType], bool] | None: """Parse typed dict call expression. @@ -443,7 +441,7 @@ def parse_typeddict_args( ) dictexpr = args[1] tvar_defs = self.api.get_and_bind_all_tvars([t for k, t in dictexpr.items]) - res = self.parse_typeddict_fields_with_types(dictexpr.items, call, existing_info) + res = self.parse_typeddict_fields_with_types(dictexpr.items, call) if res is None: # One of the types is not ready, defer. return None @@ -461,10 +459,7 @@ def parse_typeddict_args( return args[0].value, items, types, total, tvar_defs, ok def parse_typeddict_fields_with_types( - self, - dict_items: list[tuple[Expression | None, Expression]], - context: Context, - existing_info: TypeInfo | None, + self, dict_items: list[tuple[Expression | None, Expression]], context: Context ) -> tuple[list[str], list[Type], bool] | None: """Parse typed dict items passed as pairs (name expression, type expression). diff --git a/mypy/typeanal.py b/mypy/typeanal.py index 2920d9fd609b..3ddf71fb7412 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -594,10 +594,8 @@ def try_analyze_special_unbound_type(self, t: UnboundType, fullname: str) -> Typ self.fail("Self type cannot be used in a metaclass", t) if self.api.type.self_type is not None: return self.api.type.self_type.copy_modified(line=t.line, column=t.column) - # Attributes and methods are handled above, this is best effort support for - # other things: simply use the current class instead of Self. This may be - # useful for e.g. cast(Self, ...) etc, to avoid repeating long class name. - # TODO: can we have more case by case logic here? + # TODO: verify this is unreachable and replace with an assert? + self.fail("Unexpected Self type", t) return fill_typevars(self.api.type) return None From ac6234db0559122191839932cc0136de1f147f01 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Fri, 11 Nov 2022 00:00:35 +0000 Subject: [PATCH 19/25] Fix TypeVar id clash --- mypy/semanal.py | 6 +++++- mypy/tvar_scope.py | 11 +++++++++-- test-data/unit/check-selftype.test | 13 +++++++++++++ 3 files changed, 27 insertions(+), 3 deletions(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index 38acfea40b73..aa07e586903a 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -835,7 +835,11 @@ def analyze_func_def(self, defn: FuncDef) -> None: self.prepare_method_signature(defn, self.type, has_self_type) # Analyze function signature - with self.tvar_scope_frame(self.tvar_scope.method_frame()): + if has_self_type and self.type and self.type.self_type: + func_id = self.type.self_type.id.raw_id + else: + func_id = 0 + with self.tvar_scope_frame(self.tvar_scope.method_frame(func_id=func_id)): if defn.type: self.check_classvar_in_signature(defn.type) assert isinstance(defn.type, CallableType) diff --git a/mypy/tvar_scope.py b/mypy/tvar_scope.py index f926d0dfb883..1cd172b9661f 100644 --- a/mypy/tvar_scope.py +++ b/mypy/tvar_scope.py @@ -29,6 +29,7 @@ def __init__( is_class_scope: bool = False, prohibited: TypeVarLikeScope | None = None, namespace: str = "", + func_id: int = 0, ) -> None: """Initializer for TypeVarLikeScope @@ -37,6 +38,9 @@ def __init__( is_class_scope: True if this represents a generic class prohibited: Type variables that aren't strictly in scope exactly, but can't be bound because they're part of an outer class's scope. + func_id: override for parent func_id. This is needed if we bind some + synthetic type variables. For example, Self type is transformed into + a type variable, and we need to reserve its id. """ self.scope: dict[str, TypeVarLikeType] = {} self.parent = parent @@ -48,6 +52,9 @@ def __init__( if parent is not None: self.func_id = parent.func_id self.class_id = parent.class_id + if func_id < 0: + assert func_id <= self.func_id + self.func_id = func_id def get_function_scope(self) -> TypeVarLikeScope | None: """Get the nearest parent that's a function scope, not a class scope""" @@ -65,9 +72,9 @@ def allow_binding(self, fullname: str) -> bool: return False return True - def method_frame(self) -> TypeVarLikeScope: + def method_frame(self, func_id: int = 0) -> TypeVarLikeScope: """A new scope frame for binding a method""" - return TypeVarLikeScope(self, False, None) + return TypeVarLikeScope(self, False, None, func_id=func_id) def class_frame(self, namespace: str) -> TypeVarLikeScope: """A new scope frame for binding a class. Prohibits *this* class's tvars""" diff --git a/test-data/unit/check-selftype.test b/test-data/unit/check-selftype.test index dbb9c3c20033..e82c4869fdce 100644 --- a/test-data/unit/check-selftype.test +++ b/test-data/unit/check-selftype.test @@ -1545,3 +1545,16 @@ class C: assert_type(self, Self) # OK assert_type(C(), Self) # E: Expression is of type "C", not "Self" return self + +[case testTypingSelfTypeVarClash] +from typing import Self, TypeVar, Tuple + +S = TypeVar("S") +class C: + def bar(self) -> Self: ... + def foo(self, x: S) -> Tuple[Self, S]: ... + +c: C +reveal_type(C.foo) # N: Revealed type is "def [Self <: __main__.C, S] (self: Self`-1, x: S`-2) -> Tuple[Self`-1, S`-2]" +reveal_type(C().foo(42)) # N: Revealed type is "Tuple[__main__.C, builtins.int]" +[builtins fixtures/tuple.pyi] From 61c05895fac6da6d19e144b3540b173ae46dfbbc Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sat, 12 Nov 2022 00:36:03 +0000 Subject: [PATCH 20/25] Final tweaks + couple tests --- mypy/checkmember.py | 5 ++++- mypy/semanal.py | 2 +- test-data/unit/check-selftype.test | 29 ++++++++++++++++++++++++++++- 3 files changed, 33 insertions(+), 3 deletions(-) diff --git a/mypy/checkmember.py b/mypy/checkmember.py index 5c4bd4e33d2e..52b949ca1153 100644 --- a/mypy/checkmember.py +++ b/mypy/checkmember.py @@ -37,6 +37,7 @@ erase_to_bound, function_type, make_simplified_union, + supported_self_type, tuple_fallback, type_object_type_from_function, ) @@ -727,7 +728,9 @@ def analyze_var( if mx.is_lvalue and var.is_classvar: mx.msg.cant_assign_to_classvar(name, mx.context) t = get_proper_type(expand_type_by_instance(typ, itype)) - if not (mx.is_self or mx.is_super): + if not (mx.is_self or mx.is_super) or supported_self_type( + get_proper_type(mx.original_type) + ): t = get_proper_type(expand_self_type(var, t, mx.original_type)) result: Type = t typ = get_proper_type(typ) diff --git a/mypy/semanal.py b/mypy/semanal.py index aa07e586903a..7536fc130d14 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -835,7 +835,7 @@ def analyze_func_def(self, defn: FuncDef) -> None: self.prepare_method_signature(defn, self.type, has_self_type) # Analyze function signature - if has_self_type and self.type and self.type.self_type: + if self.type and self.type.self_type: func_id = self.type.self_type.id.raw_id else: func_id = 0 diff --git a/test-data/unit/check-selftype.test b/test-data/unit/check-selftype.test index e82c4869fdce..65aa03adaa9c 100644 --- a/test-data/unit/check-selftype.test +++ b/test-data/unit/check-selftype.test @@ -1554,7 +1554,34 @@ class C: def bar(self) -> Self: ... def foo(self, x: S) -> Tuple[Self, S]: ... -c: C reveal_type(C.foo) # N: Revealed type is "def [Self <: __main__.C, S] (self: Self`-1, x: S`-2) -> Tuple[Self`-1, S`-2]" reveal_type(C().foo(42)) # N: Revealed type is "Tuple[__main__.C, builtins.int]" [builtins fixtures/tuple.pyi] + +[case testTypingSelfTypeVarClashAttr] +from typing import Self, TypeVar, Tuple, Callable + +S = TypeVar("S") +class C: + def bar(self) -> Self: ... + foo: Callable[[S, Self], Tuple[Self, S]] + +reveal_type(C().foo) # N: Revealed type is "def [S] (S`-2, __main__.C) -> Tuple[__main__.C, S`-2]" +[builtins fixtures/tuple.pyi] + +[case testTypingSelfAttrOldVsNewStyle] +from typing import Self, TypeVar + +T = TypeVar("T", bound=C) +class C: + x: Self + def foo(self: T) -> T: + return self.x + def bar(self: T) -> T: + self.x = self + return self + def baz(self: Self) -> None: + self.x = self + def bad(self) -> None: + # This is unfortunate, but required by PEP 484 + self.x = self # E: Incompatible types in assignment (expression has type "C", variable has type "Self") From cbd97b122e13fd913505e09ce40b66a2c242e6e9 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sat, 12 Nov 2022 13:59:43 +0000 Subject: [PATCH 21/25] Fix another bug from mypy_primer --- mypy/checkmember.py | 4 ++-- test-data/unit/check-selftype.test | 15 +++++++++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/mypy/checkmember.py b/mypy/checkmember.py index 52b949ca1153..c81b3fbe4f7e 100644 --- a/mypy/checkmember.py +++ b/mypy/checkmember.py @@ -727,11 +727,11 @@ def analyze_var( mx.msg.read_only_property(name, itype.type, mx.context) if mx.is_lvalue and var.is_classvar: mx.msg.cant_assign_to_classvar(name, mx.context) - t = get_proper_type(expand_type_by_instance(typ, itype)) if not (mx.is_self or mx.is_super) or supported_self_type( get_proper_type(mx.original_type) ): - t = get_proper_type(expand_self_type(var, t, mx.original_type)) + typ = expand_self_type(var, typ, mx.original_type) + t = get_proper_type(expand_type_by_instance(typ, itype)) result: Type = t typ = get_proper_type(typ) if ( diff --git a/test-data/unit/check-selftype.test b/test-data/unit/check-selftype.test index 65aa03adaa9c..07df2a93d26e 100644 --- a/test-data/unit/check-selftype.test +++ b/test-data/unit/check-selftype.test @@ -1585,3 +1585,18 @@ class C: def bad(self) -> None: # This is unfortunate, but required by PEP 484 self.x = self # E: Incompatible types in assignment (expression has type "C", variable has type "Self") + +[case testTypingSelfClashUnrelated] +from typing import Self, Generic, TypeVar + +class B: ... + +T = TypeVar("T", bound=B) +class C(Generic[T]): + def __init__(self, val: T) -> None: + self.val = val + def foo(self) -> Self: ... + +def test(x: C[T]) -> T: + reveal_type(x.val) # N: Revealed type is "T`-1" + return x.val From 362d84a84ac63a11907e91ff960aabb2bcde71e9 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sat, 12 Nov 2022 14:08:42 +0000 Subject: [PATCH 22/25] Fix upper bound for Self --- docs/source/more_types.rst | 3 ++- mypy/semanal.py | 4 ++-- test-data/unit/check-selftype.test | 12 +++++++++++- 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/docs/source/more_types.rst b/docs/source/more_types.rst index 707411e95fef..722909a038b5 100644 --- a/docs/source/more_types.rst +++ b/docs/source/more_types.rst @@ -804,9 +804,10 @@ classes are generic, self-type allows giving them precise signatures: .. code-block:: python T = TypeVar('T') - Q = TypeVar('Q', bound='Base[Any]') class Base(Generic[T]): + Q = TypeVar('Q', bound='Base[T]') + def __init__(self, item: T) -> None: self.item = item diff --git a/mypy/semanal.py b/mypy/semanal.py index 7536fc130d14..8d50814ba26e 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -274,7 +274,7 @@ invalid_recursive_alias, is_named_instance, ) -from mypy.typevars import fill_typevars, fill_typevars_with_any +from mypy.typevars import fill_typevars from mypy.util import ( correct_relative_import, is_dunder, @@ -1041,7 +1041,7 @@ def setup_self_type(self) -> None: f"{info.fullname}.Self", self.tvar_scope.new_unique_func_id(), [], - fill_typevars_with_any(info), + fill_typevars(info), ) def visit_overloaded_func_def(self, defn: OverloadedFuncDef) -> None: diff --git a/test-data/unit/check-selftype.test b/test-data/unit/check-selftype.test index 07df2a93d26e..0bda073da3cf 100644 --- a/test-data/unit/check-selftype.test +++ b/test-data/unit/check-selftype.test @@ -1418,7 +1418,7 @@ class C(Generic[T]): def __init__(self, val: T) -> None: ... @classmethod def pair(cls, val: T) -> Tuple[Self, Self]: - return (cls(val), C(val)) # E: Incompatible return value type (got "Tuple[Self, C[Any]]", expected "Tuple[Self, Self]") + return (cls(val), C(val)) # E: Incompatible return value type (got "Tuple[Self, C[T]]", expected "Tuple[Self, Self]") class D(C[int]): pass reveal_type(C.pair(42)) # N: Revealed type is "Tuple[__main__.C[builtins.int], __main__.C[builtins.int]]" @@ -1600,3 +1600,13 @@ class C(Generic[T]): def test(x: C[T]) -> T: reveal_type(x.val) # N: Revealed type is "T`-1" return x.val + +[case testTypingSelfGenericBound] +from typing import Self, Generic, TypeVar + +T = TypeVar("T") +class C(Generic[T]): + val: T + def foo(self) -> Self: + reveal_type(self.val) # N: Revealed type is "T`1" + return self From a5740ebdcc8c68b89c30f4ede5e6990829e28144 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sat, 12 Nov 2022 14:23:10 +0000 Subject: [PATCH 23/25] More CR (docstring) --- mypy/semanal.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index 8d50814ba26e..993ec00a9e0e 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -1016,7 +1016,8 @@ def update_function_type_variables(self, fun_type: CallableType, defn: FuncItem) """Make any type variables in the signature of defn explicit. Update the signature of defn to contain type variable definitions - if defn is generic. + if defn is generic. Return True, if the signature contains typing.Self + type, or False otherwise. """ with self.tvar_scope_frame(self.tvar_scope.method_frame()): a = self.type_analyzer() From 6694f3b69c0f6553fca5be261edbedd5042c8623 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sat, 12 Nov 2022 17:41:15 +0000 Subject: [PATCH 24/25] Fix Self import; fix method bodies; simplify id handling --- mypy/semanal.py | 8 ++----- mypy/tvar_scope.py | 11 ++------- mypy/typeanal.py | 6 ++--- mypy/types.py | 3 ++- test-data/unit/check-dataclasses.test | 2 +- test-data/unit/check-incremental.test | 2 +- test-data/unit/check-selftype.test | 32 ++++++++++++++++++++++++--- 7 files changed, 40 insertions(+), 24 deletions(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index 993ec00a9e0e..1cb52eb57e15 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -835,11 +835,7 @@ def analyze_func_def(self, defn: FuncDef) -> None: self.prepare_method_signature(defn, self.type, has_self_type) # Analyze function signature - if self.type and self.type.self_type: - func_id = self.type.self_type.id.raw_id - else: - func_id = 0 - with self.tvar_scope_frame(self.tvar_scope.method_frame(func_id=func_id)): + with self.tvar_scope_frame(self.tvar_scope.method_frame()): if defn.type: self.check_classvar_in_signature(defn.type) assert isinstance(defn.type, CallableType) @@ -1040,7 +1036,7 @@ def setup_self_type(self) -> None: info.self_type = TypeVarType( "Self", f"{info.fullname}.Self", - self.tvar_scope.new_unique_func_id(), + 0, [], fill_typevars(info), ) diff --git a/mypy/tvar_scope.py b/mypy/tvar_scope.py index 1cd172b9661f..f926d0dfb883 100644 --- a/mypy/tvar_scope.py +++ b/mypy/tvar_scope.py @@ -29,7 +29,6 @@ def __init__( is_class_scope: bool = False, prohibited: TypeVarLikeScope | None = None, namespace: str = "", - func_id: int = 0, ) -> None: """Initializer for TypeVarLikeScope @@ -38,9 +37,6 @@ def __init__( is_class_scope: True if this represents a generic class prohibited: Type variables that aren't strictly in scope exactly, but can't be bound because they're part of an outer class's scope. - func_id: override for parent func_id. This is needed if we bind some - synthetic type variables. For example, Self type is transformed into - a type variable, and we need to reserve its id. """ self.scope: dict[str, TypeVarLikeType] = {} self.parent = parent @@ -52,9 +48,6 @@ def __init__( if parent is not None: self.func_id = parent.func_id self.class_id = parent.class_id - if func_id < 0: - assert func_id <= self.func_id - self.func_id = func_id def get_function_scope(self) -> TypeVarLikeScope | None: """Get the nearest parent that's a function scope, not a class scope""" @@ -72,9 +65,9 @@ def allow_binding(self, fullname: str) -> bool: return False return True - def method_frame(self, func_id: int = 0) -> TypeVarLikeScope: + def method_frame(self) -> TypeVarLikeScope: """A new scope frame for binding a method""" - return TypeVarLikeScope(self, False, None, func_id=func_id) + return TypeVarLikeScope(self, False, None) def class_frame(self, namespace: str) -> TypeVarLikeScope: """A new scope frame for binding a class. Prohibits *this* class's tvars""" diff --git a/mypy/typeanal.py b/mypy/typeanal.py index 3ddf71fb7412..0638b1d4a860 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -1375,12 +1375,12 @@ def bind_function_type_variables( if fun_type.variables: defs = [] for var in fun_type.variables: + if self.api.type and self.api.type.self_type and var == self.api.type.self_type: + has_self_type = True + continue var_node = self.lookup_qualified(var.name, defn) assert var_node, "Binding for function type variable not found within function" var_expr = var_node.node - if var_node.fullname in SELF_TYPE_NAMES: - has_self_type = True - continue assert isinstance(var_expr, TypeVarLikeExpr) binding = self.tvar_scope.bind_new(var.name, var_expr) defs.append(binding) diff --git a/mypy/types.py b/mypy/types.py index 285826e34047..9b24be79d9c3 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -402,7 +402,8 @@ class TypeVarId: # For plain variables (type parameters of generic classes and # functions) raw ids are allocated by semantic analysis, using # positive ids 1, 2, ... for generic class parameters and negative - # ids -1, ... for generic function type arguments. This convention + # ids -1, ... for generic function type arguments. A special value 0 + # is reserved for Self type variable (autogenerated). This convention # is only used to keep type variable ids distinct when allocating # them; the type checker makes no distinction between class and # function type variables. diff --git a/test-data/unit/check-dataclasses.test b/test-data/unit/check-dataclasses.test index c651a751511c..dd688eb240f6 100644 --- a/test-data/unit/check-dataclasses.test +++ b/test-data/unit/check-dataclasses.test @@ -1946,7 +1946,7 @@ class LinkedList(Generic[T]): next: Optional[Self] = None def meth(self) -> None: - reveal_type(self.next) # N: Revealed type is "Union[Self`-1, None]" + reveal_type(self.next) # N: Revealed type is "Union[Self`0, None]" l_int: LinkedList[int] = LinkedList(1, LinkedList("no", None)) # E: Argument 1 to "LinkedList" has incompatible type "str"; expected "int" diff --git a/test-data/unit/check-incremental.test b/test-data/unit/check-incremental.test index 5a37463ab8d7..7a4d00b37b33 100644 --- a/test-data/unit/check-incremental.test +++ b/test-data/unit/check-incremental.test @@ -6306,5 +6306,5 @@ reveal_type(D.meth) reveal_type(D().meth) [out] [out2] -tmp/m.py:4: note: Revealed type is "def [Self <: lib.C] (self: Self`-1, other: Self`-1) -> Self`-1" +tmp/m.py:4: note: Revealed type is "def [Self <: lib.C] (self: Self`0, other: Self`0) -> Self`0" tmp/m.py:5: note: Revealed type is "def (other: m.D) -> m.D" diff --git a/test-data/unit/check-selftype.test b/test-data/unit/check-selftype.test index 0bda073da3cf..4b20afa16d23 100644 --- a/test-data/unit/check-selftype.test +++ b/test-data/unit/check-selftype.test @@ -1281,7 +1281,7 @@ class C: return self class D(C): ... -reveal_type(C.meth) # N: Revealed type is "def [Self <: __main__.C] (self: Self`-1) -> builtins.list[Self`-1]" +reveal_type(C.meth) # N: Revealed type is "def [Self <: __main__.C] (self: Self`0) -> builtins.list[Self`0]" C.attr # E: Access to generic instance variables via class is ambiguous reveal_type(D().meth()) # N: Revealed type is "builtins.list[__main__.D]" reveal_type(D().attr) # N: Revealed type is "builtins.list[__main__.D]" @@ -1554,19 +1554,23 @@ class C: def bar(self) -> Self: ... def foo(self, x: S) -> Tuple[Self, S]: ... -reveal_type(C.foo) # N: Revealed type is "def [Self <: __main__.C, S] (self: Self`-1, x: S`-2) -> Tuple[Self`-1, S`-2]" +reveal_type(C.foo) # N: Revealed type is "def [Self <: __main__.C, S] (self: Self`0, x: S`-1) -> Tuple[Self`0, S`-1]" reveal_type(C().foo(42)) # N: Revealed type is "Tuple[__main__.C, builtins.int]" [builtins fixtures/tuple.pyi] [case testTypingSelfTypeVarClashAttr] from typing import Self, TypeVar, Tuple, Callable +class Defer(This): ... + S = TypeVar("S") class C: def bar(self) -> Self: ... foo: Callable[[S, Self], Tuple[Self, S]] -reveal_type(C().foo) # N: Revealed type is "def [S] (S`-2, __main__.C) -> Tuple[__main__.C, S`-2]" +reveal_type(C().foo) # N: Revealed type is "def [S] (S`-1, __main__.C) -> Tuple[__main__.C, S`-1]" +reveal_type(C().foo(42, C())) # N: Revealed type is "Tuple[__main__.C, builtins.int]" +class This: ... [builtins fixtures/tuple.pyi] [case testTypingSelfAttrOldVsNewStyle] @@ -1586,6 +1590,17 @@ class C: # This is unfortunate, but required by PEP 484 self.x = self # E: Incompatible types in assignment (expression has type "C", variable has type "Self") +[case testTypingSelfClashInBodies] +from typing import Self, TypeVar + +T = TypeVar("T") +class C: + def very_bad(self, x: T) -> None: + self.x = x # E: Incompatible types in assignment (expression has type "T", variable has type "Self") + x: Self + def baz(self: Self, x: T) -> None: + y: T = x + [case testTypingSelfClashUnrelated] from typing import Self, Generic, TypeVar @@ -1610,3 +1625,14 @@ class C(Generic[T]): def foo(self) -> Self: reveal_type(self.val) # N: Revealed type is "T`1" return self + +[case testTypingSelfDifferentImport] +import typing as t + +class Foo: + def foo(self) -> t.Self: + return self + @classmethod + def bar(cls) -> t.Self: + return cls() +[builtins fixtures/classmethod.pyi] From 3f86bf216c6ecb881dcb6b6417ebd424f4dd1f0e Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sat, 12 Nov 2022 18:09:52 +0000 Subject: [PATCH 25/25] Allow using plain class in final classes --- mypy/semanal.py | 8 +------- mypy/typeanal.py | 4 +++- test-data/unit/check-selftype.test | 8 ++++++++ 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index 1cb52eb57e15..87e2ab8c0f11 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -1033,13 +1033,7 @@ def setup_self_type(self) -> None: info = self.type if info.self_type is not None: return - info.self_type = TypeVarType( - "Self", - f"{info.fullname}.Self", - 0, - [], - fill_typevars(info), - ) + info.self_type = TypeVarType("Self", f"{info.fullname}.Self", 0, [], fill_typevars(info)) def visit_overloaded_func_def(self, defn: OverloadedFuncDef) -> None: self.statement = defn diff --git a/mypy/typeanal.py b/mypy/typeanal.py index 0638b1d4a860..96a3cbbb831f 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -593,10 +593,12 @@ def try_analyze_special_unbound_type(self, t: UnboundType, fullname: str) -> Typ if self.api.type.has_base("builtins.type"): self.fail("Self type cannot be used in a metaclass", t) if self.api.type.self_type is not None: + if self.api.type.is_final: + return fill_typevars(self.api.type) return self.api.type.self_type.copy_modified(line=t.line, column=t.column) # TODO: verify this is unreachable and replace with an assert? self.fail("Unexpected Self type", t) - return fill_typevars(self.api.type) + return AnyType(TypeOfAny.from_error) return None def get_omitted_any(self, typ: Type, fullname: str | None = None) -> AnyType: diff --git a/test-data/unit/check-selftype.test b/test-data/unit/check-selftype.test index 4b20afa16d23..5e331655728a 100644 --- a/test-data/unit/check-selftype.test +++ b/test-data/unit/check-selftype.test @@ -1636,3 +1636,11 @@ class Foo: def bar(cls) -> t.Self: return cls() [builtins fixtures/classmethod.pyi] + +[case testTypingSelfAllowAliasUseInFinalClasses] +from typing import Self, final + +@final +class C: + def meth(self) -> Self: + return C() # OK for final classes