# HG changeset patch # User Andrew Barnert # Date 1451933685 28800 # Mon Jan 04 10:54:45 2016 -0800 # Branch issue25864 # Node ID aca667ea6898c3c677d8926b67b93e4444e3623e # Parent 164b564e3c1ab51fbe1c2c29ca019de7fbf6ffc4 First patch diff -r 164b564e3c1a -r aca667ea6898 Doc/reference/datamodel.rst --- a/Doc/reference/datamodel.rst Sun Jan 03 17:58:24 2016 -0800 +++ b/Doc/reference/datamodel.rst Mon Jan 04 10:54:45 2016 -0800 @@ -1064,6 +1064,12 @@ operation raise an exception when no appropriate method is defined (typically :exc:`AttributeError` or :exc:`TypeError`). +Setting a special method to ``None`` indicates that the corresponding +operation is not available. For example, if a class sets +:meth:`__iter__` to ``None``, the class is not iterable, so calling +:func:`iter` on its instances will raise a :exc:`TypeError` (without +falling back to :meth:`__getitem__`). + When implementing a class that emulates any built-in type, it is important that the emulation only be implemented to the degree that it makes sense for the object being modelled. For example, some sequences may work well with retrieval diff -r 164b564e3c1a -r aca667ea6898 Lib/_collections_abc.py --- a/Lib/_collections_abc.py Sun Jan 03 17:58:24 2016 -0800 +++ b/Lib/_collections_abc.py Mon Jan 04 10:54:45 2016 -0800 @@ -200,8 +200,11 @@ @classmethod def __subclasshook__(cls, C): if cls is Iterable: - if any("__iter__" in B.__dict__ for B in C.__mro__): - return True + for B in C.__mro__: + if "__iter__" in B.__dict__: + if B.__dict__["__iter__"]: + return True + break return NotImplemented @@ -621,6 +624,8 @@ return NotImplemented return dict(self.items()) == dict(other.items()) + __reversed__ = None + Mapping.register(mappingproxy) diff -r 164b564e3c1a -r aca667ea6898 Lib/test/test_collections.py --- a/Lib/test/test_collections.py Sun Jan 03 17:58:24 2016 -0800 +++ b/Lib/test/test_collections.py Mon Jan 04 10:54:45 2016 -0800 @@ -688,6 +688,15 @@ self.assertFalse(issubclass(str, I)) self.validate_abstract_methods(Iterable, '__iter__') self.validate_isinstance(Iterable, '__iter__') + # Check None blocking + class J: + def __iter__(self): return iter([]) + class K(J): + __iter__ = None + self.assertTrue(issubclass(J, Iterable)) + self.assertTrue(isinstance(J(), Iterable)) + self.assertFalse(issubclass(K, Iterable)) + self.assertFalse(isinstance(K(), Iterable)) def test_Iterator(self): non_samples = [None, 42, 3.14, 1j, b"", "", (), [], {}, set()] @@ -1219,6 +1228,7 @@ def __iter__(self): return iter(()) self.validate_comparison(MyMapping()) + self.assertRaises(TypeError, reversed, MyMapping()) def test_MutableMapping(self): for sample in [dict]: diff -r 164b564e3c1a -r aca667ea6898 Lib/test/test_enumerate.py --- a/Lib/test/test_enumerate.py Sun Jan 03 17:58:24 2016 -0800 +++ b/Lib/test/test_enumerate.py Mon Jan 04 10:54:45 2016 -0800 @@ -232,6 +232,13 @@ ngi = NoGetItem() self.assertRaises(TypeError, reversed, ngi) + class Blocked(object): + def __getitem__(self): return 1 + def __len__(self): return 2 + __reversed__ = None + b = Blocked() + self.assertRaises(TypeError, reversed, b) + def test_pickle(self): for data in 'abc', range(5), tuple(enumerate('abc')), range(1,17,5): self.check_pickle(reversed(data), list(data)[::-1]) diff -r 164b564e3c1a -r aca667ea6898 Lib/test/test_iter.py --- a/Lib/test/test_iter.py Sun Jan 03 17:58:24 2016 -0800 +++ b/Lib/test/test_iter.py Mon Jan 04 10:54:45 2016 -0800 @@ -53,6 +53,14 @@ def __getitem__(self, i): return i +class DefaultIterClass: + pass + +class NoIterClass: + def __getitem__(self, i): + return i + __iter__ = None + # Main test suite class TestCase(unittest.TestCase): @@ -944,6 +952,10 @@ self.assertEqual(next(it), 0) self.assertEqual(next(it), 1) + def test_error_iter(self): + for typ in (DefaultIterClass, NoIterClass): + self.assertRaises(TypeError, iter, typ()) + def test_main(): run_unittest(TestCase) diff -r 164b564e3c1a -r aca667ea6898 Objects/enumobject.c --- a/Objects/enumobject.c Sun Jan 03 17:58:24 2016 -0800 +++ b/Objects/enumobject.c Mon Jan 04 10:54:45 2016 -0800 @@ -250,6 +250,12 @@ return NULL; reversed_meth = _PyObject_LookupSpecial(seq, &PyId___reversed__); + if (reversed_meth == Py_None) { + Py_DECREF(reversed_meth); + PyErr_SetString(PyExc_TypeError, + "argument to reversed() must be a sequence"); + return NULL; + } if (reversed_meth != NULL) { PyObject *res = PyObject_CallFunctionObjArgs(reversed_meth, NULL); Py_DECREF(reversed_meth); diff -r 164b564e3c1a -r aca667ea6898 Objects/typeobject.c --- a/Objects/typeobject.c Sun Jan 03 17:58:24 2016 -0800 +++ b/Objects/typeobject.c Mon Jan 04 10:54:45 2016 -0800 @@ -6204,6 +6204,13 @@ _Py_IDENTIFIER(__iter__); func = lookup_method(self, &PyId___iter__); + if (func == Py_None) { + Py_DECREF(func); + PyErr_Format(PyExc_TypeError, + "'%.200s' object is not iterable", + Py_TYPE(self)->tp_name); + return NULL; + } if (func != NULL) { PyObject *args; args = res = PyTuple_New(0); # HG changeset patch # User Andrew Barnert # Date 1451947928 28800 # Mon Jan 04 14:52:08 2016 -0800 # Branch issue25958 # Node ID b4af2e347762f6069214865ff7d847305d9016f8 # Parent aca667ea6898c3c677d8926b67b93e4444e3623e Make all collections.abc ABCs treat None as blocking Currently, for some collections.abc ABCs with subclass hooks, like `Hashable`, setting the special method to `None` means the class is definitely not a subclass, while with others, like `Iterable`, it means the opposite. This should be consistent. Just as setting `__hash__ = None` is the idiomatic way to prevent inheriting `object.__hash__` in `hash()`, setting `__iter__ = None` is the idiomatic way to block fallback to the old-style sequence protocol in `iter()`, and `__contains__ = None` to block fallback to iteration. Also, there were a few slightly different ways of implementing the check--they all seemed correct, but factoring it out to a single place means less chance of introducing bugs later. diff -r aca667ea6898 -r b4af2e347762 Lib/_collections_abc.py --- a/Lib/_collections_abc.py Mon Jan 04 10:54:45 2016 -0800 +++ b/Lib/_collections_abc.py Mon Jan 04 14:52:08 2016 -0800 @@ -62,6 +62,18 @@ ### ONE-TRICK PONIES ### +def _check_methods(C, *methods): + mro = C.__mro__ + for method in methods: + for B in mro: + if method in B.__dict__: + if B.__dict__[method] is None: + return NotImplemented + break + else: + return NotImplemented + return True + class Hashable(metaclass=ABCMeta): __slots__ = () @@ -72,12 +84,9 @@ @classmethod def __subclasshook__(cls, C): + print(cls) if cls is Hashable: - for B in C.__mro__: - if "__hash__" in B.__dict__: - if B.__dict__["__hash__"]: - return True - break + return _check_methods(C, "__hash__") return NotImplemented @@ -92,11 +101,7 @@ @classmethod def __subclasshook__(cls, C): if cls is Awaitable: - for B in C.__mro__: - if "__await__" in B.__dict__: - if B.__dict__["__await__"]: - return True - break + return _check_methods(C, "__await__") return NotImplemented @@ -137,14 +142,7 @@ @classmethod def __subclasshook__(cls, C): if cls is Coroutine: - mro = C.__mro__ - for method in ('__await__', 'send', 'throw', 'close'): - for base in mro: - if method in base.__dict__: - break - else: - return NotImplemented - return True + return _check_methods(C, '__await__', 'send', 'throw', 'close') return NotImplemented @@ -162,8 +160,7 @@ @classmethod def __subclasshook__(cls, C): if cls is AsyncIterable: - if any("__aiter__" in B.__dict__ for B in C.__mro__): - return True + return _check_methods(C, "__aiter__") return NotImplemented @@ -182,9 +179,7 @@ @classmethod def __subclasshook__(cls, C): if cls is AsyncIterator: - if (any("__anext__" in B.__dict__ for B in C.__mro__) and - any("__aiter__" in B.__dict__ for B in C.__mro__)): - return True + return _check_methods(C, "__anext__", "__aiter__") return NotImplemented @@ -200,11 +195,7 @@ @classmethod def __subclasshook__(cls, C): if cls is Iterable: - for B in C.__mro__: - if "__iter__" in B.__dict__: - if B.__dict__["__iter__"]: - return True - break + return _check_methods(C, "__iter__") return NotImplemented @@ -223,9 +214,7 @@ @classmethod def __subclasshook__(cls, C): if cls is Iterator: - if (any("__next__" in B.__dict__ for B in C.__mro__) and - any("__iter__" in B.__dict__ for B in C.__mro__)): - return True + return _check_methods(C, "__next__", "__iter__") return NotImplemented Iterator.register(bytes_iterator) @@ -286,14 +275,8 @@ @classmethod def __subclasshook__(cls, C): if cls is Generator: - mro = C.__mro__ - for method in ('__iter__', '__next__', 'send', 'throw', 'close'): - for base in mro: - if method in base.__dict__: - break - else: - return NotImplemented - return True + return _check_methods(C, '__iter__', '__next__', + 'send', 'throw', 'close') return NotImplemented @@ -311,8 +294,7 @@ @classmethod def __subclasshook__(cls, C): if cls is Sized: - if any("__len__" in B.__dict__ for B in C.__mro__): - return True + return _check_methods(C, "__len__") return NotImplemented @@ -327,8 +309,7 @@ @classmethod def __subclasshook__(cls, C): if cls is Container: - if any("__contains__" in B.__dict__ for B in C.__mro__): - return True + return _check_methods(C, "__contains__") return NotImplemented @@ -343,8 +324,7 @@ @classmethod def __subclasshook__(cls, C): if cls is Callable: - if any("__call__" in B.__dict__ for B in C.__mro__): - return True + return _check_methods(C, "__call__") return NotImplemented # HG changeset patch # User Andrew Barnert # Date 1451948495 28800 # Mon Jan 04 15:01:35 2016 -0800 # Branch issue25958 # Node ID 4a223e44ea253fdbfb9a60727c0146b319616743 # Parent b4af2e347762f6069214865ff7d847305d9016f8 Add collections.abc.Reversible As specified in #25987, but integrated with #25958 (that is, it uses the factored-out _check_methods for its subclass hook, to make __reversed__ = None a failure rather than a success). diff -r b4af2e347762 -r 4a223e44ea25 Doc/library/collections.abc.rst --- a/Doc/library/collections.abc.rst Mon Jan 04 14:52:08 2016 -0800 +++ b/Doc/library/collections.abc.rst Mon Jan 04 15:01:35 2016 -0800 @@ -42,10 +42,11 @@ :class:`Iterator` :class:`Iterable` ``__next__`` ``__iter__`` :class:`Generator` :class:`Iterator` ``send``, ``throw`` ``close``, ``__iter__``, ``__next__`` :class:`Sized` ``__len__`` +:class:`Reversible` :class:`Iterable` ``__reversed__`` :class:`Callable` ``__call__`` :class:`Sequence` :class:`Sized`, ``__getitem__``, ``__contains__``, ``__iter__``, ``__reversed__``, - :class:`Iterable`, ``__len__`` ``index``, and ``count`` + :class:`Reversible`, ``__len__`` ``index``, and ``count`` :class:`Container` :class:`MutableSequence` :class:`Sequence` ``__getitem__``, Inherited :class:`Sequence` methods and diff -r b4af2e347762 -r 4a223e44ea25 Lib/_collections_abc.py --- a/Lib/_collections_abc.py Mon Jan 04 14:52:08 2016 -0800 +++ b/Lib/_collections_abc.py Mon Jan 04 15:01:35 2016 -0800 @@ -11,7 +11,7 @@ __all__ = ["Awaitable", "Coroutine", "AsyncIterable", "AsyncIterator", "Hashable", "Iterable", "Iterator", "Generator", - "Sized", "Container", "Callable", + "Sized", "Container", "Reversible", "Callable", "Set", "MutableSet", "Mapping", "MutableMapping", "MappingView", "KeysView", "ItemsView", "ValuesView", @@ -328,6 +328,22 @@ return NotImplemented +class Reversible(Iterable): + + __slots__ = () + + @abstractmethod + def __reversed__(self): + while False: + yield None + + @classmethod + def __subclasshook__(cls, C): + if cls is Reversible: + return _check_methods(C, "__reversed__") + return NotImplemented + + ### SETS ### @@ -779,7 +795,7 @@ ### SEQUENCES ### -class Sequence(Sized, Iterable, Container): +class Sequence(Sized, Reversible, Container): """All the operations on a read-only sequence. # HG changeset patch # User Andrew Barnert # Date 1451949099 28800 # Mon Jan 04 15:11:39 2016 -0800 # Branch issue25958 # Node ID fcf76a4061f54e96003ecd07943ebb542b2287a3 # Parent 4a223e44ea253fdbfb9a60727c0146b319616743 Add tests for collections.abc.Reversible diff -r 4a223e44ea25 -r fcf76a4061f5 Lib/test/test_collections.py --- a/Lib/test/test_collections.py Mon Jan 04 15:01:35 2016 -0800 +++ b/Lib/test/test_collections.py Mon Jan 04 15:11:39 2016 -0800 @@ -21,7 +21,7 @@ from collections import deque from collections.abc import Awaitable, Coroutine, AsyncIterator, AsyncIterable from collections.abc import Hashable, Iterable, Iterator, Generator -from collections.abc import Sized, Container, Callable +from collections.abc import Sized, Container, Reversible, Callable from collections.abc import Set, MutableSet from collections.abc import Mapping, MutableMapping, KeysView, ItemsView from collections.abc import Sequence, MutableSequence @@ -698,6 +698,56 @@ self.assertFalse(issubclass(K, Iterable)) self.assertFalse(isinstance(K(), Iterable)) + def test_Reversible(self): + # Check some non-iterables + non_iterables = [None, 42, 3.14, 1j] + for x in non_iterables: + self.assertNotIsInstance(x, Reversible) + self.assertFalse(issubclass(type(x), Reversible), repr(type(x))) + # Check some non-reversible iterables + non_reversibles = [set(), frozenset(), dict(), dict().keys(), + dict().items(), dict().values(), + Counter(), Counter().keys(), Counter().items(), + Counter().values(), (lambda: (yield))(), + (x for x in []), iter([]), reversed([]) + ] + for x in non_reversibles: + self.assertNotIsInstance(x, Reversible) + self.assertFalse(issubclass(type(x), Reversible), repr(type(x))) + # Check some reversible iterables + samples = [bytes(), str(), + tuple(), list(), OrderedDict(), OrderedDict().keys(), + OrderedDict().items(), OrderedDict().values(), + ] + for x in samples: + self.assertIsInstance(x, Reversible) + self.assertTrue(issubclass(type(x), Reversible), repr(type(x))) + # Check direct subclassing + class R(Reversible): + def __iter__(self): + return super().__iter__() + def __reversed__(self): + return super().__reversed__() + self.assertEqual(list(R()), []) + self.assertEqual(list(reversed(R())), list(reversed([]))) + self.assertFalse(issubclass(str, R)) + self.validate_abstract_methods(Reversible, '__reversed__') + self.validate_isinstance(Reversible, '__reversed__') + # Check None blocking + class J: + def __iter__(self): return iter([]) + def __reversed__(self): return reversed([]) + class K(J): + __iter__ = None + class L(J): + __reversed__ = None + self.assertTrue(issubclass(J, Reversible)) + self.assertTrue(isinstance(J(), Reversible)) + self.assertFalse(issubclass(K, Reversible)) + self.assertFalse(isinstance(K(), Reversible)) + self.assertFalse(issubclass(L, Reversible)) + self.assertFalse(isinstance(L(), Reversible)) + def test_Iterator(self): non_samples = [None, 42, 3.14, 1j, b"", "", (), [], {}, set()] for x in non_samples: # HG changeset patch # User Andrew Barnert # Date 1451950587 28800 # Mon Jan 04 15:36:27 2016 -0800 # Branch issue25958 # Node ID 3971a514df3a132ea5a76a7dedee6a5095483fae # Parent fcf76a4061f54e96003ecd07943ebb542b2287a3 Special handling for __contains__ = None As with __hash__ since forever, and __iter__ and __reversed__ since earlier in this branch, setting __contains__ to None and then using the in operator will now raise a nice TypeError instead of an ugly one (and there's now a unit test for the error). diff -r fcf76a4061f5 -r 3971a514df3a Lib/test/test_contains.py --- a/Lib/test/test_contains.py Mon Jan 04 15:11:39 2016 -0800 +++ b/Lib/test/test_contains.py Mon Jan 04 15:36:27 2016 -0800 @@ -84,6 +84,27 @@ self.assertTrue(container == constructor(values)) self.assertTrue(container == container) + def test_block_fallback(self): + # blocking fallback with __contains__ = None + class ByContains(object): + def __contains__(self, other): + return False + c = ByContains() + class BlockContains(ByContains): + """Is not a container + + This class is a perfectly good iterable, and inherits from + a perfectly good container, but __contains__ = None blocks + both of these from working.""" + def __iter__(self): + while False: + yield None + __contains__ = None + bc = BlockContains() + self.assertFalse(0 in c) + self.assertFalse(0 in list(bc)) + self.assertRaises(TypeError, lambda x: 0 in x, bc) + if __name__ == '__main__': unittest.main() diff -r fcf76a4061f5 -r 3971a514df3a Objects/typeobject.c --- a/Objects/typeobject.c Mon Jan 04 15:11:39 2016 -0800 +++ b/Objects/typeobject.c Mon Jan 04 15:36:27 2016 -0800 @@ -5819,6 +5819,13 @@ _Py_IDENTIFIER(__contains__); func = lookup_maybe(self, &PyId___contains__); + if (func == Py_None) { + Py_DECREF(func); + PyErr_Format(PyExc_TypeError, + "argument of type '%.200s' is not a container", + Py_TYPE(self)->tp_name); + return NULL; + } if (func != NULL) { args = PyTuple_Pack(1, value); if (args == NULL) # HG changeset patch # User Andrew Barnert # Date 1451950978 28800 # Mon Jan 04 15:42:58 2016 -0800 # Branch issue25958 # Node ID 37750d1d2e83a482bc4f69c97427be92f85463f6 # Parent 3971a514df3a132ea5a76a7dedee6a5095483fae Test fix for Reversible diff -r 3971a514df3a -r 37750d1d2e83 Lib/test/test_collections.py --- a/Lib/test/test_collections.py Mon Jan 04 15:36:27 2016 -0800 +++ b/Lib/test/test_collections.py Mon Jan 04 15:42:58 2016 -0800 @@ -731,8 +731,7 @@ self.assertEqual(list(R()), []) self.assertEqual(list(reversed(R())), list(reversed([]))) self.assertFalse(issubclass(str, R)) - self.validate_abstract_methods(Reversible, '__reversed__') - self.validate_isinstance(Reversible, '__reversed__') + self.validate_abstract_methods(Reversible, '__reversed__', '__iter__') # Check None blocking class J: def __iter__(self): return iter([]) # HG changeset patch # User Andrew Barnert # Date 1451952740 28800 # Mon Jan 04 16:12:20 2016 -0800 # Branch issue25958 # Node ID 5ec3bbb7b2d63f2afa59aef1f0cedaa7a35f088a # Parent 37750d1d2e83a482bc4f69c97427be92f85463f6 Add tests for None blocking special method fallback * test_binop tests reverse fallback for __eq__, with subclass rules. * test_augassign tests fallback for __iadd__. diff -r 37750d1d2e83 -r 5ec3bbb7b2d6 Lib/test/test_augassign.py --- a/Lib/test/test_augassign.py Mon Jan 04 15:42:58 2016 -0800 +++ b/Lib/test/test_augassign.py Mon Jan 04 16:12:20 2016 -0800 @@ -83,6 +83,10 @@ def __iadd__(self, val): return aug_test3(self.val + val) + class aug_test4(aug_test3): + "Blocks inheritance, and fallback to __add__" + __iadd__ = None + x = aug_test(1) y = x x += 10 @@ -106,6 +110,9 @@ self.assertTrue(y is not x) self.assertEqual(x.val, 13) + x = aug_test4(4) + with self.assertRaises(TypeError): + x += 10 def testCustomMethods2(test_self): output = [] diff -r 37750d1d2e83 -r 5ec3bbb7b2d6 Lib/test/test_binop.py --- a/Lib/test/test_binop.py Mon Jan 04 15:42:58 2016 -0800 +++ b/Lib/test/test_binop.py Mon Jan 04 16:12:20 2016 -0800 @@ -388,6 +388,34 @@ self.assertEqual(op_sequence(eq, B, V), ['B.__eq__', 'V.__eq__']) self.assertEqual(op_sequence(le, B, V), ['B.__le__', 'V.__ge__']) +class E(object): + """Class that can test equality""" + def __eq__(self, other): + return True + +class S(E): + """Subclass of E that should fail""" + __eq__ = None +class F(object): + """Independent class that should fall back""" + +class X(object): + """Independent class that should fail""" + __eq__ = None + +class FallbackBlockingTests(unittest.TestCase): + def test_fallback_blocking(self): + e, f, s, x = E(), F(), S(), X() + self.assertEqual(e, e) + self.assertEqual(e, f) + self.assertEqual(f, e) + # left operand is checked first + self.assertEqual(e, x) + self.assertRaises(TypeError, eq, x, e) + # S is a subclass, so it's always checked first + self.assertRaises(TypeError, eq, e, s) + self.assertRaises(TypeError, eq, s, e) + if __name__ == "__main__": unittest.main()