diff --git a/pytype/abstract_utils.py b/pytype/abstract_utils.py index 2bce0a08a..9a51f4f62 100644 --- a/pytype/abstract_utils.py +++ b/pytype/abstract_utils.py @@ -1,8 +1,10 @@ +# Lint as: python3 """Utilities for abstract.py.""" import collections import hashlib import logging +from typing import Tuple, Union from pytype import compat from pytype import datatypes @@ -605,12 +607,14 @@ def check_classes(var, check): v.cls.isinstance_Class() and check(v.cls) for v in var.data if v.cls) -def match_type_container(typ, container_type_name): +def match_type_container(typ, container_type_name: Union[str, Tuple[str, ...]]): """Unpack the type parameter from ContainerType[T].""" if typ is None: return None + if isinstance(container_type_name, str): + container_type_name = (container_type_name,) if not (typ.isinstance_ParameterizedClass() and - typ.full_name == container_type_name): + typ.full_name in container_type_name): return None param = typ.get_formal_type_parameter(T) return param diff --git a/pytype/config.py b/pytype/config.py index 567033b0c..aeec0f03c 100644 --- a/pytype/config.py +++ b/pytype/config.py @@ -134,6 +134,10 @@ def add_basic_options(o): "--strict-import", action="store_true", dest="strict_import", default=False, help="Experimental: Only load submodules that are explicitly imported.") + o.add_argument( + "--check-variable-types", action="store_true", + dest="check_variable_types", default=False, + help="Experimental: Check variable values against their annotations.") o.add_argument( "--precise-return", action="store_true", dest="precise_return", default=False, help=("Experimental: Infer precise return types even for " diff --git a/pytype/matcher.py b/pytype/matcher.py index 0efa69092..edede2b2f 100644 --- a/pytype/matcher.py +++ b/pytype/matcher.py @@ -11,6 +11,7 @@ from pytype import mixin from pytype import special_builtins from pytype import utils +from pytype.overlays import dataclass_overlay from pytype.overlays import typing_overlay from pytype.pytd import pep484 from pytype.pytd import pytd @@ -102,6 +103,11 @@ def bad_matches(self, var, other_type, node): A list of all the views of var that didn't match. """ bad = [] + if (var.data == [self.vm.convert.unsolvable] or + other_type == self.vm.convert.unsolvable): + # An unsolvable matches everything. Since bad_matches doesn't need to + # compute substitutions, we can return immediately. + return bad views = abstract_utils.get_views([var], node) skip_future = None while True: @@ -397,6 +403,10 @@ def _match_type_against_type(self, left, other_type, subst, node, view): if new_subst is not None: return new_subst return None + elif isinstance(left, dataclass_overlay.FieldInstance) and left.default: + default = self.vm.merge_values(left.default.data) + return self._match_type_against_type( + default, other_type, subst, node, view) elif isinstance(left, abstract.SimpleAbstractValue): return self._match_instance_against_type( left, other_type, subst, node, view) diff --git a/pytype/overlays/attr_overlay.py b/pytype/overlays/attr_overlay.py index fd4de9f00..8e1aece5d 100644 --- a/pytype/overlays/attr_overlay.py +++ b/pytype/overlays/attr_overlay.py @@ -81,12 +81,13 @@ def decorate(self, node, cls): typ=typ, init=orig.data[0].init, default=orig.data[0].default) - self.check_default(node, attr.name, attr.typ, attr.default, local.stack, - allow_none=True) + self.vm.check_annotation_type_mismatch( + node, attr.name, attr.typ, attr.default, local.stack, + allow_none=True) own_attrs.append(attr) elif self.args[cls]["auto_attribs"]: if not match_classvar(typ): - self.check_default( + self.vm.check_annotation_type_mismatch( node, name, typ, orig, local.stack, allow_none=True) attr = Attribute(name=name, typ=typ, init=True, default=orig) if not orig: diff --git a/pytype/overlays/classgen.py b/pytype/overlays/classgen.py index 6ce26d8d0..13327a3bc 100644 --- a/pytype/overlays/classgen.py +++ b/pytype/overlays/classgen.py @@ -179,29 +179,6 @@ def get_base_class_attrs(self, cls, cls_attrs, metadata_key): base_attrs.append(a) return base_attrs - def check_default(self, node, name, typ, default, stack, allow_none=False): - """Check that the type annotation and the default value are consistent. - - Args: - node: node - name: variable name - typ: variable annotation - default: variable assignment or default value - stack: a frame stack for error reporting - allow_none: whether a default of None is allowed for any type - """ - if not typ or not default: - return - # Check for permitted uses of x: T = None - if (allow_none and - len(default.data) == 1 and - default.data[0].cls == self.vm.convert.none_type): - return - bad = self.vm.matcher.bad_matches(default, typ, node) - if bad: - binding = bad[0][default] - self.vm.errorlog.annotation_type_mismatch(stack, typ, binding, name) - def call(self, node, func, args): """Construct a decorator, and call it on the class.""" self.match_args(node, args) diff --git a/pytype/overlays/dataclass_overlay.py b/pytype/overlays/dataclass_overlay.py index 6909f92ab..0f8f0527f 100644 --- a/pytype/overlays/dataclass_overlay.py +++ b/pytype/overlays/dataclass_overlay.py @@ -82,8 +82,13 @@ def decorate(self, node, cls): else: init = True - # Check that default matches the declared type - self.check_default(node, name, typ, orig, local.stack) + if (not self.vm.options.check_variable_types or + orig and orig.data == [self.vm.convert.none]): + # vm._apply_annotation mostly takes care of checking that the default + # matches the declared type. However, it allows None defaults, and + # dataclasses do not. + self.vm.check_annotation_type_mismatch( + node, name, typ, orig, local.stack, allow_none=False) attr = classgen.Attribute(name=name, typ=typ, init=init, default=orig) own_attrs.append(attr) diff --git a/pytype/tests/py2/test_six_overlay.py b/pytype/tests/py2/test_six_overlay.py index 2b58b51fe..00837e15c 100644 --- a/pytype/tests/py2/test_six_overlay.py +++ b/pytype/tests/py2/test_six_overlay.py @@ -33,15 +33,15 @@ def test_string_types(self): b = [b] c = '' # type: Text if isinstance(c, six.string_types): - c = len(c) + d = len(c) """) - self.assertTypesMatchPytd( - ty, """ - from typing import List - six: module = ... - a: List[str] = ... - b: List[unicode] = ... - c: int = ... + self.assertTypesMatchPytd(ty, """ + from typing import List, Union + six: module + a: List[str] + b: List[unicode] + c: Union[str, unicode] + d: int """) diff --git a/pytype/tests/py3/test_attr.py b/pytype/tests/py3/test_attr.py index ed4f0b5ad..053b1d3c9 100644 --- a/pytype/tests/py3/test_attr.py +++ b/pytype/tests/py3/test_attr.py @@ -1,3 +1,4 @@ +# Lint as: python3 """Tests for attrs library in attr_overlay.py.""" from pytype import file_utils @@ -7,6 +8,12 @@ class TestAttrib(test_base.TargetPython3BasicTest): """Tests for attr.ib.""" + def setUp(self): + super().setUp() + # Checking field defaults against their types should work even when general + # variable checking is disabled. + self.options.tweak(check_variable_types=False) + def test_factory_function(self): ty = self.Infer(""" import attr @@ -31,6 +38,12 @@ def __init__(self, x: CustomClass = ...) -> None: ... class TestAttribPy3(test_base.TargetPython3FeatureTest): """Tests for attr.ib using PEP526 syntax.""" + def setUp(self): + super().setUp() + # Checking field defaults against their types should work even when general + # variable checking is disabled. + self.options.tweak(check_variable_types=False) + def test_variable_annotations(self): ty = self.Infer(""" import attr @@ -151,6 +164,12 @@ class Bar(object): class TestAttrs(test_base.TargetPython3FeatureTest): """Tests for attr.s.""" + def setUp(self): + super().setUp() + # Checking field defaults against their types should work even when general + # variable checking is disabled. + self.options.tweak(check_variable_types=False) + def test_kw_only(self): ty = self.Infer(""" import attr diff --git a/pytype/tests/py3/test_dataclasses.py b/pytype/tests/py3/test_dataclasses.py index 781fec237..0aeab1f1e 100644 --- a/pytype/tests/py3/test_dataclasses.py +++ b/pytype/tests/py3/test_dataclasses.py @@ -1,3 +1,4 @@ +# Lint as: python3 """Tests for the dataclasses overlay.""" from pytype.tests import test_base @@ -6,6 +7,12 @@ class TestDataclass(test_base.TargetPython3FeatureTest): """Tests for @dataclass.""" + def setUp(self): + super().setUp() + # Checking field defaults against their types should work even when general + # variable checking is disabled. + self.options.tweak(check_variable_types=False) + def test_basic(self): ty = self.Infer(""" import dataclasses diff --git a/pytype/tests/py3/test_six_overlay.py b/pytype/tests/py3/test_six_overlay.py index 092f7b523..0a2a7faa7 100644 --- a/pytype/tests/py3/test_six_overlay.py +++ b/pytype/tests/py3/test_six_overlay.py @@ -30,14 +30,14 @@ def test_string_types(self): a = [a] b = '' # type: Text if isinstance(b, six.string_types): - b = len(b) + c = len(b) """) - self.assertTypesMatchPytd( - ty, """ + self.assertTypesMatchPytd(ty, """ from typing import List - six: module = ... - a: List[str] = ... - b: int = ... + six: module + a: List[str] + b: str + c: int """) def test_integer_types(self): diff --git a/pytype/tests/py3/test_type_comments.py b/pytype/tests/py3/test_type_comments.py index 620c0b04a..fb769b128 100644 --- a/pytype/tests/py3/test_type_comments.py +++ b/pytype/tests/py3/test_type_comments.py @@ -14,12 +14,12 @@ def foo(x: int) -> float: """) def test_list_comprehension_comments(self): - ty = self.Infer(""" + ty, errors = self.InferWithErrors(""" from typing import List def f(x: str): pass def g(xs: List[str]) -> List[str]: - ys = [f(x) for x in xs] # type: List[str] + ys = [f(x) for x in xs] # type: List[str] # annotation-type-mismatch[e] return ys """) self.assertTypesMatchPytd(ty, """ @@ -27,6 +27,8 @@ def g(xs: List[str]) -> List[str]: def f(x: str) -> None: ... def g(xs: List[str]) -> List[str]: ... """) + self.assertErrorRegexes( + errors, {"e": r"Annotation: List\[str\].*Assignment: List\[None\]"}) class Py3TypeCommentTest(test_base.TargetPython3FeatureTest): diff --git a/pytype/tests/py3/test_variable_annotations.py b/pytype/tests/py3/test_variable_annotations.py index 9179c5763..ba0d68028 100644 --- a/pytype/tests/py3/test_variable_annotations.py +++ b/pytype/tests/py3/test_variable_annotations.py @@ -117,22 +117,24 @@ def test_shadow_none(self): """) def test_overwrite_annotation(self): - ty = self.Infer(""" + ty, errors = self.InferWithErrors(""" x: int - x = "" + x = "" # annotation-type-mismatch[e] """) self.assertTypesMatchPytd(ty, "x: str") + self.assertErrorRegexes(errors, {"e": r"Annotation: int.*Assignment: str"}) def test_overwrite_annotation_in_class(self): - ty = self.Infer(""" + ty, errors = self.InferWithErrors(""" class Foo: x: int - x = "" + x = "" # annotation-type-mismatch[e] """) self.assertTypesMatchPytd(ty, """ class Foo: x: str """) + self.assertErrorRegexes(errors, {"e": r"Annotation: int.*Assignment: str"}) def test_class_variable_forward_reference(self): ty = self.Infer(""" @@ -227,5 +229,11 @@ def f(): def f() -> Dict[str, Dict[str, bool]]: ... """) + def test_none_or_ellipsis_assignment(self): + self.Check(""" + v1: int = None + v2: str = ... + """) + test_base.main(globals(), __name__ == "__main__") diff --git a/pytype/tests/test_attr.py b/pytype/tests/test_attr.py index 55ae3b3e8..fb304a4ef 100644 --- a/pytype/tests/test_attr.py +++ b/pytype/tests/test_attr.py @@ -1,3 +1,4 @@ +# Lint as: python3 """Tests for attrs library in attr_overlay.py.""" from pytype.tests import test_base @@ -6,6 +7,12 @@ class TestAttrib(test_base.TargetIndependentTest): """Tests for attr.ib.""" + def setUp(self): + super().setUp() + # Checking field defaults against their types should work even when general + # variable checking is disabled. + self.options.tweak(check_variable_types=False) + def test_basic(self): ty = self.Infer(""" import attr @@ -647,6 +654,12 @@ def __init__(self, x = ...) -> None: ... class TestAttrs(test_base.TargetIndependentTest): """Tests for attr.s.""" + def setUp(self): + super().setUp() + # Checking field defaults against their types should work even when general + # variable checking is disabled. + self.options.tweak(check_variable_types=False) + def test_basic(self): ty = self.Infer(""" import attr diff --git a/pytype/tests/test_base.py b/pytype/tests/test_base.py index 286e8b0e0..e1fd4f37a 100644 --- a/pytype/tests/test_base.py +++ b/pytype/tests/test_base.py @@ -150,7 +150,8 @@ def t(name): # pylint: disable=invalid-name def setUp(self): super(BaseTest, self).setUp() - self.options = config.Options.create(python_version=self.python_version) + self.options = config.Options.create(python_version=self.python_version, + check_variable_types=True) @property def loader(self): diff --git a/pytype/tests/test_type_comments.py b/pytype/tests/test_type_comments.py index 12fdf30da..b5c903bb3 100644 --- a/pytype/tests/test_type_comments.py +++ b/pytype/tests/test_type_comments.py @@ -390,7 +390,7 @@ class A(object): def test_none_to_none_type(self): ty = self.Infer(""" - x = 0 # type: None + x = ... # type: None """, deep=False) self.assertTypesMatchPytd(ty, """ x = ... # type: None @@ -581,14 +581,14 @@ class A(object): """) def test_list_comprehension_comments(self): - ty = self.Infer(""" + ty, errors = self.InferWithErrors(""" from typing import List def f(x): # type: (str) -> None pass def g(xs): # type: (List[str]) -> List[str] - ys = [f(x) for x in xs] # type: List[str] + ys = [f(x) for x in xs] # type: List[str] # annotation-type-mismatch[e] return ys """) self.assertTypesMatchPytd(ty, """ @@ -596,6 +596,8 @@ def g(xs): def f(x: str) -> None: ... def g(xs: List[str]) -> List[str]: ... """) + self.assertErrorRegexes( + errors, {"e": r"Annotation: List\[str\].*Assignment: List\[None\]"}) def test_multiple_assignments(self): ty = self.Infer(""" diff --git a/pytype/tools/analyze_project/config.py b/pytype/tools/analyze_project/config.py index 8457e3a79..8656c1488 100644 --- a/pytype/tools/analyze_project/config.py +++ b/pytype/tools/analyze_project/config.py @@ -62,6 +62,8 @@ # The missing fields will be filled in by generate_sample_config_or_die. _PYTYPE_SINGLE_ITEMS = { + 'check_variable_types': Item( + None, 'False', ArgInfo('--check-variable-types', None), None), 'disable': Item( None, 'pyi-error', ArgInfo('--disable', ','.join), 'Comma or space separated list of error names to ignore.'), diff --git a/pytype/vm.py b/pytype/vm.py index c7463ff49..9970f05af 100644 --- a/pytype/vm.py +++ b/pytype/vm.py @@ -1282,12 +1282,48 @@ def _check_aliased_type_params(self, value): self.frames, "aliases of Unions with type parameters") def _apply_annotation(self, state, op, name, orig_val, local): + """Applies the type annotation, if any, associated with this object.""" typ, value = self.annotations_util.apply_annotation( state, op, name, orig_val) if local: self._record_local(state.node, op, name, typ, orig_val) + if typ is None and name in self.current_annotated_locals: + typ = self.current_annotated_locals[name].get_type(state.node, name) + if self.options.check_variable_types: + self.check_annotation_type_mismatch( + state.node, name, typ, orig_val, self.frames, allow_none=True) + # TODO(rechen): In cases like + # v: float + # v = 0 + # do we want to replace 0 with Instance(float)? return value + def check_annotation_type_mismatch( + self, node, name, typ, value, stack, allow_none): + """Checks for a mismatch between a variable's annotation and value. + + Args: + node: node + name: variable name + typ: variable annotation + value: variable value + stack: a frame stack for error reporting + allow_none: whether a value of None is allowed for any type + """ + if not typ or not value: + return + if (value.data == [self.convert.ellipsis] or + allow_none and value.data == [self.convert.none]): + return + contained_type = abstract_utils.match_type_container( + typ, ("typing.ClassVar", "dataclasses.InitVar")) + if contained_type: + typ = contained_type + bad = self.matcher.bad_matches(value, typ, node) + for view in bad: + binding = view[value] + self.errorlog.annotation_type_mismatch(stack, typ, binding, name) + def _pop_and_store(self, state, op, name, local): """Pop a value off the stack and store it in a variable.""" state, orig_val = state.pop()