PyCon Day 1: Descriptors and Metaclasses Understanding and Using Python's More Advanced Features
with Mike Müller
https://us.pycon.org/2015/schedule/presentation/302/
Introduction
You can definitely get away without knowing Descriptor and Metaclasses. But they are powerful and can provide elegant solutions to some problems. They’re not necessarily simple, can be opaque to other programmers and are easy to overuse. Everything in this tutoral comes with the caveat of use them with care.
Some libraries and frameworks use them, so it’s helpful to know them.
Descriptors and Metaclasses
Properties
import sys
sys.version
'3.4.0 (default, Apr 11 2014, 13:05:11) \n[GCC 4.8.2]'
class Square(object):
def __init__(self, side):
self.side = side
def aget(self):
return self.side ** 2
def aset(self, value):
raise Exception("Can't set the area")
def adel(self):
raise Exception("Can't delete the area")
area = property(aget, aset, adel, doc='Area of the square')
s = Square(4)
s.area
16
Notice that it’s a computed area, but it looks just like a property. We can do this a little bit better. Right now, the fact the the methods are actually properties are hidden/detacted all the way down at the bottom. We can fix that by using a decorator.
class Square(object):
"""A square using properties with decorators"""
def __init__(self, side):
self.side = side
@property
def area(self):
"""Calculate the area of the square when the property is accessed"""
return self.side ** 2
# If you leave this unimplemented you will get a standard exception when you try to access
@area.setter
def area(self, value):
raise Exception("Can't set the area")
@area.deleter
def area(self):
raise Exception("Can't delete the area")
s = Square(5)
print(s.area)
25
s.area = 16
---------------------------------------------------------------------------
Exception Traceback (most recent call last)
<ipython-input-6-738f5610e14d> in <module>()
----> 1 s.area = 16
<ipython-input-4-456197114e5d> in area(self, value)
12 @area.setter
13 def area(self, value):
---> 14 raise Exception("Can't set the area")
15
16 @area.deleter
Exception: Can't set the area
Now this is a little weird because now you have three functions that are all related, but are not actually grouped together. What if you forget to implement one and not the others? It would be nice to nest them together.
def nested_property(func):
names = func()
names['doc'] = func.__doc__
return property(**names)
class Square(object):
"""A square using properties with decorators"""
def __init__(self, side):
self.side = side
@nested_property
def area():
def fget(self):
return self.side ** 2
def fset(self, value):
raise Exception("Setting the area is not allowed")
def fdel(self):
raise Exception("Deleting the area is not allowed")
return locals()
s = Square(5)
print('area:', s.area)
area: 25
Descriptors
Descriptors provide fine grained control over attribute access and over how a user can use your program.
% load datadescriptor.py
"""A typical data descriptor.
"""
from __future__ import print_function
class DataDescriptor(object):
"""A simple descriptor.
"""
def __init__(self):
self.value = 0
def __get__(self, instance, cls):
print('data descriptor __get__')
return self.value
def __set__(self, instance, value):
print('data descriptor __set__')
try:
self.value = value.upper()
except AttributeError:
self.value = value
def __delete__(self, instance):
print("Don't like to be deleted." )
"""A typical data descriptor.
"""
from __future__ import print_function
class DataDescriptor(object):
"""A simple descriptor.
"""
def __init__(self):
self.value = 0
def __get__(self, instance, cls):
print('data descriptor __get__')
return self.value
def __set__(self, instance, value):
print('data descriptor __set__')
try:
self.value = value.upper()
except AttributeError:
self.value = value
def __delete__(self, instance):
print("Don't like to be deleted." )
class A(object):
attr = DataDescriptor()
a = A()
a.attr
data descriptor __get__
0
type(a).__dict__['attr'].__get__(a, type(a))
data descriptor __get__
0
So, a.attr causes the complicated lookup below.
A.attr
data descriptor __get__
0
A.__dict__['attr'].__get__(None, A)
data descriptor __get__
0
A.__dict__['attr'] = 12
---------------------------------------------------------------------------
TypeError Traceback (most recent call last)
<ipython-input-16-140b0a398f48> in <module>()
----> 1 A.__dict__['attr'] = 12
TypeError: 'mappingproxy' object does not support item assignment
a.attr = 5
data descriptor __set__
a.attr = 'hello'
a.attr
data descriptor __set__
data descriptor __get__
'HELLO'
Now we have very strong control over what happens when someone accesses an attribute.
a.__dict__ # it's an empty dict
{}
a.x = 100
a.__dict__
{'x': 100}
class NonDataDescriptor(object):
"""A simple descriptor.
"""
def __init__(self):
self.value = 0
def __get__(self, instance, cls):
print('data descriptor __get__')
return self.value + 10
class B(object):
attr = NonDataDescriptor()
ten = B()
ten.attr
data descriptor __get__
10
# But if we assign to it, it will be different
ten.__dict__
{}
ten.attr = 100
ten.attr
100
ten.__dict__
{'attr': 100}
class Overridden(object):
attr = DataDescriptor()
def __getattribute__(self, name):
print("no way, buddy")
o = Overridden()
o.attr
no way, buddy
get_attribute always gets called… it’s very strong. So if you tried to do self.* in get_attribute, you would get an infinite recursion error.
def func():
pass
func.__get__
<method-wrapper '__get__' of function object at 0x7f797d03cb70>
Methods are non-data descriptors
class C(object):
def meth(self):
pass
c = C()
c.meth
<bound method C.meth of <__main__.C object at 0x7f797d05a748>>
C.meth # This is unbound when accessed through the class
<function __main__.meth>
What happens when you want to accesss an unbound method?
type(c).__dict__['meth'].__get__(c, type(c))
<bound method C.meth of <__main__.C object at 0x7f797d05a748>>
get does the binding That’s the theory, let’s look at a few examples.
%load class_storage.py
"""A descriptor works only in a class.
Storing attribute data directly in a descriptor
means sharing between instances.
"""
from __future__ import print_function
class DescriptorClassStorage(object):
"""Descriptor storing data in class."""
def __init__(self, default=None):
self.value = default
def __get__(self, instance, owner):
return self.value
def __set__(self, instance, value):
self.value = value
if __name__ == '__main__':
class StoreClass(object):
"""All instances will share `attr`.
"""
attr = DescriptorClassStorage(10)
store1 = StoreClass()
store2 = StoreClass()
print('store1', store1.attr)
print('store2', store2.attr)
print('Setting store1 only.')
store1.attr = 100
print('store1', store1.attr)
print('store2', store2.attr)
"""A descriptor works only in a class.
Storing attribute data directly in a descriptor
means sharing between instances.
"""
from __future__ import print_function
class DescriptorClassStorage(object):
"""Descriptor storing data in class."""
def __init__(self, default=None):
self.value = default
def __get__(self, instance, owner):
return self.value
def __set__(self, instance, value):
self.value = value
if __name__ == '__main__':
class StoreClass(object):
"""All instances will share `attr`.
"""
attr = DescriptorClassStorage(10)
store1 = StoreClass()
store2 = StoreClass()
print('store1', store1.attr)
print('store2', store2.attr)
print('Setting store1 only.')
store1.attr = 100
print('store1', store1.attr)
print('store2', store2.attr)
store1 10
store2 10
Setting store1 only.
store1 100
store2 100
The descriptor belongs to the class not to the instance!
%load weakkeydict_storage.py
"""A descriptor works only in a class.
We can store a different value for each instance in a dictionary
in the descriptor.
"""
from __future__ import print_function
from weakref import WeakKeyDictionary
class DescriptorWeakKeyDictStorage(object):
"""Descriptor that stores attribute data in instances.
"""
_hidden = WeakKeyDictionary()
def __init__(self, default=None):
self.default = default
def __get__(self, instance, owner):
return DescriptorWeakKeyDictStorage._hidden.get(instance, self.default)
def __set__(self, instance, value):
DescriptorWeakKeyDictStorage._hidden[instance] = value
if __name__ == '__main__':
class StoreInstance(object):
"""All instances have own `attr`.
"""
attr = DescriptorWeakKeyDictStorage(10)
store1 = StoreInstance()
store2 = StoreInstance()
print('store1', store1.attr)
print('store2', store2.attr)
print('Setting store1 only.')
store1.attr = 100
print('store1', store1.attr)
print('store2', store2.attr)
print('_hidden:', DescriptorWeakKeyDictStorage._hidden.items())
del store1
print('Deleted store1')
print('_hidden:', DescriptorWeakKeyDictStorage._hidden.items())
"""A descriptor works only in a class.
We can store a different value for each instance in a dictionary
in the descriptor.
"""
from __future__ import print_function
from weakref import WeakKeyDictionary
class DescriptorWeakKeyDictStorage(object):
"""Descriptor that stores attribute data in instances.
"""
_hidden = WeakKeyDictionary()
def __init__(self, default=None):
self.default = default
def __get__(self, instance, owner):
return DescriptorWeakKeyDictStorage._hidden.get(instance, self.default)
def __set__(self, instance, value):
DescriptorWeakKeyDictStorage._hidden[instance] = value
if __name__ == '__main__':
class StoreInstance(object):
"""All instances have own `attr`.
"""
attr = DescriptorWeakKeyDictStorage(10)
store1 = StoreInstance()
store2 = StoreInstance()
print('store1', store1.attr)
print('store2', store2.attr)
print('Setting store1 only.')
store1.attr = 100
print('store1', store1.attr)
print('store2', store2.attr)
print('_hidden:', list(DescriptorWeakKeyDictStorage._hidden.items()))
del store1
print('Deleted store1')
print('_hidden:', list(DescriptorWeakKeyDictStorage._hidden.items()))
store1 10
store2 10
Setting store1 only.
store1 100
store2 10
_hidden: [(<__main__.StoreInstance object at 0x7f797d079390>, 100)]
Deleted store1
_hidden: []
have to use a special type of dictionary, beacuse a normal dictionary creates references and we don’t want that.
%load checked.py
"""Example for descriptor that checks conditions on attributes.
"""
from __future__ import print_function
from weakref import WeakKeyDictionary
class Checked(object):
"""Descriptor that checks with a user-supplied check function
if an attribute is valid.
"""
_hidden = WeakKeyDictionary()
def __init__(self, checker=None, default=None):
if checker:
# checker must be a callable
checker(default)
self.checker = checker
self.default = default
def __get__(self, instance, owner):
return Checked._hidden.get(instance, self.default)
def __set__(self, instance, value):
if self.checker:
self.checker(value)
Checked._hidden[instance] = value
if __name__ == '__main__':
def is_int(value):
"""Check if value is an integer.
"""
if not isinstance(value, int):
raise ValueError('Int required {} found'.format(type(value)))
class Restricted(object):
"""Use checked attributes.
"""
attr1 = Checked(checker=is_int, default=10)
attr2 = Checked(default=12.5)
# Setting the default to float, `is_int` raises a `ValueError`.
try:
attr3 = Checked(checker=is_int, default=12.5)
except ValueError:
print('Cannot set default to float, must be int.')
attr3 = Checked(checker=is_int, default=12)
restricted = Restricted()
print('attr1', restricted.attr1)
restricted.attr1 = 100
print('attr1', restricted.attr1)
try:
restricted.attr1 = 200.12
except ValueError:
print('Cannot set attr1 to float, must be int.')
restricted.attr1 = 200
Typically in python you don’t do type checking. But if you need to communicate with the outside world, then it’s better to make sure that your types are corrrect.
"""Example for descriptor that checks conditions on attributes.
"""
from __future__ import print_function
from weakref import WeakKeyDictionary
class Checked(object):
"""Descriptor that checks with a user-supplied check function
if an attribute is valid.
"""
_hidden = WeakKeyDictionary()
def __init__(self, checker=None, default=None):
if checker:
# checker must be a callable
checker(default)
self.checker = checker
self.default = default
def __get__(self, instance, owner):
return Checked._hidden.get(instance, self.default)
def __set__(self, instance, value):
if self.checker:
self.checker(value)
Checked._hidden[instance] = value
if __name__ == '__main__':
def is_int(value):
"""Check if value is an integer.
"""
if not isinstance(value, int):
raise ValueError('Int required {} found'.format(type(value)))
class Restricted(object):
"""Use checked attributes.
"""
attr1 = Checked(checker=is_int, default=10)
attr2 = Checked(default=12.5)
# Setting the default to float, `is_int` raises a `ValueError`.
try:
attr3 = Checked(checker=is_int, default=12.5)
except ValueError:
print('Cannot set default to float, must be int.')
attr3 = Checked(checker=is_int, default=12)
restricted = Restricted()
print('attr1', restricted.attr1)
restricted.attr1 = 100
print('attr1', restricted.attr1)
try:
restricted.attr1 = 200.12
except ValueError:
print('Cannot set attr1 to float, must be int.')
restricted.attr1 = 200
Cannot set default to float, must be int.
attr1 10
attr1 100
Cannot set attr1 to float, must be int.
You could certainly implement this checking another way, but this way is very concise and the logic is all contained in one place. It’s pretty declarative, and you don’t repeat yourself at all.
Excercises
Write a descriptor that only allows positive numbers to be set
class PositiveDescriptor(object):
"""A descriptor that only allows positive numbers to be set.
"""
def __init__(self):
self.value = 0
def __get__(self, instance, cls):
print('data descriptor __get__')
return self.value
def __set__(self, instance, value):
print('data descriptor __set__')
if value != abs(value):
raise ValueError("Only positive numbers are allowed")
class F(object):
x = PositiveDescriptor()
f = F()
f.x = 100
print(f.x)
f.x = -100
---------------------------------------------------------------------------
ValueError Traceback (most recent call last)
<ipython-input-42-94262ecaa394> in <module>()
18 f.x = 100
19 print(f.x)
---> 20 f.x = -100
<ipython-input-42-94262ecaa394> in __set__(self, instance, value)
10 print('data descriptor __set__')
11 if value != abs(value):
---> 12 raise ValueError("Only positive numbers are allowed")
13
14 class F(object):
ValueError: Only positive numbers are allowed
data descriptor __set__
data descriptor __get__
0
data descriptor __set__
Metaclasses
Very advanced topic. What are they used for?
- Metaclasses are to classes as what classes are to instances
- Let you control what happens when a class is defined
- Can be useful for tasks like ORMs (SQLAlchemy and Django both use metaclasses)
- Can be a great explorative tool for a code base. Helps you to search a codebase for things at runtime.
When you define a class the Metaclass is active already.
Let’s do some examples.
class SomeClass(object):
pass
SomeClass.__bases__
(builtins.object,)
object.__bases__
()
class A(metaclass=type):
pass
class B(object):
attr = 10
C = type('C', (object,) ,{'attr': 10})
Turns out that you are not limited to defining classes in your source code. You can define classes dynamically at runtime as well. You shouldn’t as a general rule, because it makes your programs much harder to read, but sometimes there are good reasons to do so.
def __init__(self, value):
self.value = value
def add(self, a, b):
return a + b
C = type('C', (object,), {'attr': 10, '__init__': __init__, 'add': add})
c = C(10)
c.value
10
c.add(4, 5)
9
class MyMeta(type):
def __str__(cls):
return 'hello'
class A(metaclass=MyMeta):
pass
str(A)
'hello'
This is pretty powerful, because you can redefine the behaviour of a class.
type(C)
builtins.type
type(A)
__main__.MyMeta
class MyMeta(type):
def __new__(mcl, name, bases, cdict):
print(mcl)
print(name)
print(bases)
print(cdict)
return super().__new__(mcl, name, bases, cdict)
class New(object, metaclass=MyMeta):
'''The stuff in the metaclass happens at *definition* time'''
pass
<class '__main__.MyMeta'>
New
(<class 'object'>,)
{'__module__': '__main__', '__qualname__': 'New', '__doc__': 'The stuff in the metaclass happens at *definition* time'}
In new you have access to all of the class dictionary and you could actually go in and change or delete methods. Be very, very careful with this.
class MyMeta(type):
def __init__(cls, name, bases, cdict):
print(cls)
print(name)
print(bases)
print(cdict)
super().__init__()
class New(object, metaclass=MyMeta):
def meth(self):
pass
---------------------------------------------------------------------------
TypeError Traceback (most recent call last)
<ipython-input-53-bd80dcd33f24> in <module>()
7 super().__init__()
8
----> 9 class New(object, metaclass=MyMeta):
10 def meth(self):
11 pass
<ipython-input-53-bd80dcd33f24> in __init__(cls, name, bases, cdict)
5 print(bases)
6 print(cdict)
----> 7 super().__init__()
8
9 class New(object, metaclass=MyMeta):
TypeError: type.__init__() takes 1 or 3 arguments
<class '__main__.New'>
New
(<class 'object'>,)
{'__module__': '__main__', 'meth': <function New.meth at 0x7f797d059a60>, '__qualname__': 'New'}
class NoBase:
pass
%load meta_2_3.py
# file: meta_2_3.py
"""
The code is a bit hard to understand. The basic idea is exploiting the idea
that metaclasses can customize class creation and are picked by by the parent
class. This particular implementation uses a metaclass to remove its own parent
from the inheritance tree on subclassing. The end result is that the function
creates a dummy class with a dummy metaclass. Once subclassed the dummy
classes metaclass is used which has a constructor that basically instances a
new class from the original parent and the actually intended metaclass.
That way the dummy class and dummy metaclass never show up.
From:
http://lucumr.pocoo.org/2013/5/21/porting-to-python-3-redux/#metaclass-syntax-changes
Used in:
* Jinja2
* SQLAlchemy
* future (python-future.org)
"""
from __future__ import print_function
import platform
# from jinja2/_compat.py
def with_metaclass(meta, *bases):
# This requires a bit of explanation: the basic idea is to make a
# dummy metaclass for one level of class instanciation that replaces
# itself with the actual metaclass. Because of internal type checks
# we also need to make sure that we downgrade the custom metaclass
# for one level to something closer to type (that's why __call__ and
# __init__ comes back from type etc.).
#
# This has the advantage over six.with_metaclass in that it does not
# introduce dummy classes into the final MRO.
class metaclass(meta):
__call__ = type.__call__
__init__ = type.__init__
def __new__(cls, name, this_bases, d):
if this_bases is None:
return type.__new__(cls, name, (), d)
return meta(name, bases, d)
return metaclass('temporary_class', None, {})
if __name__ == '__main__':
class BaseClass(object):
pass
class MetaClass(type):
"""Metaclass for Python 2 and 3.
"""
def __init__(cls, name, bases, cdict):
print('It works with {impl} version {ver}.'.format(
impl=platform.python_implementation(),
ver=platform.python_version()))
super(MetaClass, cls).__init__(name, bases, cdict)
class Class(with_metaclass(MetaClass, BaseClass)):
# BaseClass is optional.
pass
# file: meta_2_3.py
"""
The code is a bit hard to understand. The basic idea is exploiting the idea
that metaclasses can customize class creation and are picked by by the parent
class. This particular implementation uses a metaclass to remove its own parent
from the inheritance tree on subclassing. The end result is that the function
creates a dummy class with a dummy metaclass. Once subclassed the dummy
classes metaclass is used which has a constructor that basically instances a
new class from the original parent and the actually intended metaclass.
That way the dummy class and dummy metaclass never show up.
From:
http://lucumr.pocoo.org/2013/5/21/porting-to-python-3-redux/#metaclass-syntax-changes
Used in:
* Jinja2
* SQLAlchemy
* future (python-future.org)
"""
from __future__ import print_function
import platform
# from jinja2/_compat.py
def with_metaclass(meta, *bases):
# This requires a bit of explanation: the basic idea is to make a
# dummy metaclass for one level of class instanciation that replaces
# itself with the actual metaclass. Because of internal type checks
# we also need to make sure that we downgrade the custom metaclass
# for one level to something closer to type (that's why __call__ and
# __init__ comes back from type etc.).
#
# This has the advantage over six.with_metaclass in that it does not
# introduce dummy classes into the final MRO.
class metaclass(meta):
__call__ = type.__call__
__init__ = type.__init__
def __new__(cls, name, this_bases, d):
if this_bases is None:
return type.__new__(cls, name, (), d)
return meta(name, bases, d)
return metaclass('temporary_class', None, {})
if __name__ == '__main__':
class BaseClass(object):
pass
class MetaClass(type):
"""Metaclass for Python 2 and 3.
"""
def __init__(cls, name, bases, cdict):
print('It works with {impl} version {ver}.'.format(
impl=platform.python_implementation(),
ver=platform.python_version()))
super(MetaClass, cls).__init__(name, bases, cdict)
class Class(with_metaclass(MetaClass, BaseClass)):
# BaseClass is optional.
pass
It works with CPython version 3.4.0.
# file: meta_2_3.py
"""
The code is a bit hard to understand. The basic idea is exploiting the idea
that metaclasses can customize class creation and are picked by by the parent
class. This particular implementation uses a metaclass to remove its own parent
from the inheritance tree on subclassing. The end result is that the function
creates a dummy class with a dummy metaclass. Once subclassed the dummy
classes metaclass is used which has a constructor that basically instances a
new class from the original parent and the actually intended metaclass.
That way the dummy class and dummy metaclass never show up.
From:
http://lucumr.pocoo.org/2013/5/21/porting-to-python-3-redux/#metaclass-syntax-changes
Used in:
* Jinja2
* SQLAlchemy
* future (python-future.org)
"""
from __future__ import print_function
import platform
# from jinja2/_compat.py
def with_metaclass(meta, *bases):
# This requires a bit of explanation: the basic idea is to make a
# dummy metaclass for one level of class instanciation that replaces
# itself with the actual metaclass. Because of internal type checks
# we also need to make sure that we downgrade the custom metaclass
# for one level to something closer to type (that's why __call__ and
# __init__ comes back from type etc.).
#
# This has the advantage over six.with_metaclass in that it does not
# introduce dummy classes into the final MRO.
class metaclass(meta):
__call__ = type.__call__
__init__ = type.__init__
def __new__(cls, name, this_bases, d):
if this_bases is None:
return type.__new__(cls, name, (), d)
return meta(name, bases, d)
return metaclass('temporary_class', None, {})
if __name__ == '__main__':
class BaseClass(object):
pass
class MetaClass(type):
"""Metaclass for Python 2 and 3.
"""
def __init__(cls, name, bases, cdict):
print('It works with {impl} version {ver}.'.format(
impl=platform.python_implementation(),
ver=platform.python_version()))
super(MetaClass, cls).__init__(name, bases, cdict)
class Class(with_metaclass(MetaClass, BaseClass)):
# BaseClass is optional.
pass
It works with CPython version 3.4.0.
# file: meta_2_3.py
"""
The code is a bit hard to understand. The basic idea is exploiting the idea
that metaclasses can customize class creation and are picked by by the parent
class. This particular implementation uses a metaclass to remove its own parent
from the inheritance tree on subclassing. The end result is that the function
creates a dummy class with a dummy metaclass. Once subclassed the dummy
classes metaclass is used which has a constructor that basically instances a
new class from the original parent and the actually intended metaclass.
That way the dummy class and dummy metaclass never show up.
From:
http://lucumr.pocoo.org/2013/5/21/porting-to-python-3-redux/#metaclass-syntax-changes
Used in:
* Jinja2
* SQLAlchemy
* future (python-future.org)
"""
from __future__ import print_function
import platform
# from jinja2/_compat.py
def with_metaclass(meta, *bases):
# This requires a bit of explanation: the basic idea is to make a
# dummy metaclass for one level of class instanciation that replaces
# itself with the actual metaclass. Because of internal type checks
# we also need to make sure that we downgrade the custom metaclass
# for one level to something closer to type (that's why __call__ and
# __init__ comes back from type etc.).
#
# This has the advantage over six.with_metaclass in that it does not
# introduce dummy classes into the final MRO.
class metaclass(meta):
__call__ = type.__call__
__init__ = type.__init__
def __new__(cls, name, this_bases, d):
if this_bases is None:
return type.__new__(cls, name, (), d)
return meta(name, bases, d)
return metaclass('temporary_class', None, {})
if __name__ == '__main__':
class BaseClass(object):
pass
class MetaClass(type):
"""Metaclass for Python 2 and 3.
"""
def __init__(cls, name, bases, cdict):
print('It works with {impl} version {ver}.'.format(
impl=platform.python_implementation(),
ver=platform.python_version()))
super(MetaClass, cls).__init__(name, bases, cdict)
class Class(with_metaclass(MetaClass, BaseClass)):
# BaseClass is optional.
pass
It works with CPython version 3.4.0.
Let’s look at some examples, now that we have this powerful tool to override default behaviour
%load noclassattr.py
#file: noclassattr.py
"""Preventing non-callable class attributes with a metaclass.
"""
from __future__ import print_function
class NoClassAttributes(type):
"""No non-callable class attributes allowed
"""
def __init__(cls, name, bases, cdict):
allowed = set(['__module__', '__metaclass__', '__doc__',
'__qualname__'])
for key, value in cdict.items():
if (key not in allowed) and (not callable(value)):
msg = 'Found non-callable class attribute "%s". ' % key
msg += 'Only methods are allowed.'
raise Exception(msg)
super(NoClassAttributes, cls).__init__(name, bases, cdict)
if __name__ == '__main__':
from meta_2_3 import with_metaclass
class AttributeChecker(with_metaclass(NoClassAttributes)):
"""Base class for meta.
"""
pass
class AttributeLess(AttributeChecker):
"""Only methods work.
"""
def meth(self):
"""This is allowed'
"""
print('Hello from AttributeLess.')
attributeless = AttributeLess()
attributeless.meth()
class WithAttribute(AttributeChecker):
"""Has non-callable class attribute.
Will raise an exception.
"""
a = 10
def meth(self):
"""This is allowed'
"""
print('Hello from WithAttribute')
#file: noclassattr.py
"""Preventing non-callable class attributes with a metaclass.
"""
from __future__ import print_function
class NoClassAttributes(type):
"""No non-callable class attributes allowed
"""
def __init__(cls, name, bases, cdict):
allowed = set(['__module__', '__metaclass__', '__doc__',
'__qualname__'])
for key, value in cdict.items():
if (key not in allowed) and (not callable(value)):
msg = 'Found non-callable class attribute "%s". ' % key
msg += 'Only methods are allowed.'
raise Exception(msg)
super(NoClassAttributes, cls).__init__(name, bases, cdict)
if __name__ == '__main__':
from meta_2_3 import with_metaclass
class AttributeChecker(with_metaclass(NoClassAttributes)):
"""Base class for meta.
"""
pass
class AttributeLess(AttributeChecker):
"""Only methods work.
"""
def meth(self):
"""This is allowed'
"""
print('Hello from AttributeLess.')
attributeless = AttributeLess()
attributeless.meth()
class WithAttribute(AttributeChecker):
"""Has non-callable class attribute.
Will raise an exception.
"""
a = 10
def meth(self):
"""This is allowed'
"""
print('Hello from WithAttribute')
---------------------------------------------------------------------------
Exception Traceback (most recent call last)
<ipython-input-65-f15f69a2fcd7> in <module>()
41
42
---> 43 class WithAttribute(AttributeChecker):
44 """Has non-callable class attribute.
45 Will raise an exception.
<ipython-input-65-f15f69a2fcd7> in __init__(cls, name, bases, cdict)
16 msg = 'Found non-callable class attribute "%s". ' % key
17 msg += 'Only methods are allowed.'
---> 18 raise Exception(msg)
19 super(NoClassAttributes, cls).__init__(name, bases, cdict)
20
Exception: Found non-callable class attribute "a". Only methods are allowed.
Hello from AttributeLess.
#file: noclassattr.py
"""Preventing non-callable class attributes with a metaclass.
"""
from __future__ import print_function
class NoClassAttributes(type):
"""No non-callable class attributes allowed
"""
def __init__(cls, name, bases, cdict):
allowed = set(['__module__', '__metaclass__', '__doc__',
'__qualname__'])
for key, value in cdict.items():
if (key not in allowed) and (not callable(value)):
msg = 'Found non-callable class attribute "%s". ' % key
msg += 'Only methods are allowed.'
raise Exception(msg)
super(NoClassAttributes, cls).__init__(name, bases, cdict)
if __name__ == '__main__':
from meta_2_3 import with_metaclass
class AttributeChecker(with_metaclass(NoClassAttributes)):
"""Base class for meta.
"""
pass
class AttributeLess(AttributeChecker):
"""Only methods work.
"""
def meth(self):
"""This is allowed'
"""
print('Hello from AttributeLess.')
attributeless = AttributeLess()
attributeless.meth()
Hello from AttributeLess.
if __name__ == '__main__':
class WithAttribute(AttributeChecker):
"""Has non-callable class attribute.
Will raise an exception.
"""
a = 10
def meth(self):
"""This is allowed'
"""
print('Hello from WithAttribute')
---------------------------------------------------------------------------
Exception Traceback (most recent call last)
<ipython-input-67-7a0d1a162bdd> in <module>()
1 if __name__ == '__main__':
----> 2 class WithAttribute(AttributeChecker):
3 """Has non-callable class attribute.
4 Will raise an exception.
5 """
<ipython-input-66-8613178937f0> in __init__(cls, name, bases, cdict)
16 msg = 'Found non-callable class attribute "%s". ' % key
17 msg += 'Only methods are allowed.'
---> 18 raise Exception(msg)
19 super(NoClassAttributes, cls).__init__(name, bases, cdict)
20
Exception: Found non-callable class attribute "a". Only methods are allowed.
ls
autometa_python2.py class_deco.py meta_2_3.py prepare.py [0m[01;34m__pycache__[0m/ submeta.py use_classwatcher.py
base_conflict.py classwatcher.py [01;32mnoclassattr.py[0m* problem_autometa_python2.py [01;32mslotstyped.py[0m* use_autometa_python2.py use_classwatcher.py~
It turns out that decorators are newer than Metaclasses and you can achieve some of the same results by using a class decorator.
def deco(cls):
cls.attr = 10
return cls
@deco
class A(object):
pass
A.attr
10
%load class_deco.py
# file: class_deco.py
def noclassattr_deco(cls):
"""Class decorator to allow only callable attributes.
"""
allowed = set(['__module__', '__metaclass__', '__doc__', '__qualname__',
'__weakref__', '__dict__'])
for key, value in cls.__dict__.items():
if (key not in allowed) and (not callable(value)):
msg = 'Found non-callable class attribute "%s". ' % key
msg += 'Only methods are allowed.'
raise Exception(msg)
return cls
if __name__ == '__main__':
@noclassattr_deco
class AttributeLess(object):
"""Only methods work.
"""
def meth(self):
"""This is allowed'
"""
print('Hello from AttributeLess.')
attributeless = AttributeLess()
attributeless.meth()
@noclassattr_deco
class WithAttribute(object):
"""Has non-callable class attribute.
Will raise an exception.
"""
a = 10
def meth(self):
"""This is allowed'
"""
print('Hello from WithAttribute')
---------------------------------------------------------------------------
Exception Traceback (most recent call last)
<ipython-input-72-124bdc771eca> in <module>()
29
30 @noclassattr_deco
---> 31 class WithAttribute(object):
32 """Has non-callable class attribute.
33 Will raise an exception.
<ipython-input-72-124bdc771eca> in noclassattr_deco(cls)
10 msg = 'Found non-callable class attribute "%s". ' % key
11 msg += 'Only methods are allowed.'
---> 12 raise Exception(msg)
13 return cls
14
Exception: Found non-callable class attribute "a". Only methods are allowed.
Hello from AttributeLess.
%load slotstyped.py
slots are a way of having class attributes without a dictionary. If you know which attributes you are going to want, you can specify them ahead of time and save some memory (if you’re going to be instantiating millions of objects).
# file: slotstyped.py
"""Use of descriptor and metaclass to get slots with
given types.
"""
from __future__ import print_function
class TypDescriptor(object):
"""Descriptor with type.
"""
def __init__(self, data_type, default_value=None):
self.name = None
self.data_type = data_type
if default_value:
self.default_value = default_value
else:
self.default_value = data_type()
def __get__(self, instance, cls):
return getattr(instance, self.name, self.default_value)
def __set__(self, instance, value):
if not isinstance(value, self.data_type):
raise TypeError('Required data type is %s. Got %s' % (
self.data_type, type(value)))
setattr(instance, self.name, value)
def __delete__(self, instance):
raise AttributeError('Cannot delete %r' % instance)
class TypeProtected(type):
"""Metaclass to save descriptor values in slots.
"""
def __new__(mcl, name, bases, cdict):
slots = []
for key, value in cdict.items():
if isinstance(value, TypDescriptor):
value.name = '__' + key
slots.append(value.name)
cdict['__slots__'] = slots
return super(TypeProtected, mcl).__new__(mcl, name, bases, cdict)
if __name__ == '__main__':
from meta_2_3 import with_metaclass
class Typed(with_metaclass(TypeProtected)):
pass
class MyClass(Typed):
"""Test class."""
attr1 = TypDescriptor(int)
attr2 = TypDescriptor(float, 5.5)
def main():
"""Test it.
"""
my_inst = MyClass()
print(my_inst.attr1)
print(my_inst.attr2)
print(dir(my_inst))
print(my_inst.__slots__)
my_inst.attr1 = 100.0
print(my_inst.attr1)
# this will fail
my_inst.unknown = 100
main()
---------------------------------------------------------------------------
TypeError Traceback (most recent call last)
<ipython-input-73-f167ef5081ac> in <module>()
73 my_inst.unknown = 100
74
---> 75 main()
<ipython-input-73-f167ef5081ac> in main()
68 print(dir(my_inst))
69 print(my_inst.__slots__)
---> 70 my_inst.attr1 = 100.0
71 print(my_inst.attr1)
72 # this will fail
<ipython-input-73-f167ef5081ac> in __set__(self, instance, value)
25 if not isinstance(value, self.data_type):
26 raise TypeError('Required data type is %s. Got %s' % (
---> 27 self.data_type, type(value)))
28 setattr(instance, self.name, value)
29
TypeError: Required data type is <class 'int'>. Got <class 'float'>
0
5.5
['_MyClass__attr1', '_MyClass__attr2', '__class__', '__delattr__', '__dir__', '__doc__', '__eq__', '__format__', '__ge__', '__getattribute__', '__gt__', '__hash__', '__init__', '__le__', '__lt__', '__module__', '__ne__', '__new__', '__reduce__', '__reduce_ex__', '__repr__', '__setattr__', '__sizeof__', '__slots__', '__str__', '__subclasshook__', 'attr1', 'attr2']
['__attr1', '__attr2']
%load autometa_python2.py
# file: autometa_python2.py
"""Example usage of a metaclass.
We change the metaclass of classes that inherit form `object`.
"""
from __future__ import print_function
import __builtin__
class DebugMeta(type):
"""Metaclass to be used for debugging.
"""
names = []
counter = -1 # Do not count definition of new_object`.
def __init__(cls, name, bases, cdict):
"""Store all class names and count how many classes are defined.
"""
if DebugMeta.counter >= 0:
DebugMeta.names.append('%s.%s' % (cls.__module__, name))
super(DebugMeta, cls).__init__(name, bases, cdict)
DebugMeta.counter += 1
def report(cls):
print('Defined %d classes.' % DebugMeta.counter)
print(DebugMeta.names)
class new_object(object):
"""Replacement for the built-in `object`.
"""
__metaclass__ = DebugMeta
def set_new_meta():
"""We actually change a built-in. This is a very strong measure.
"""
__builtin__.object = new_object
%load use_autometa_python2.py
# file: use_autometa_python2.py
from autometa_python2 import set_new_meta
set_new_meta()
class SomeClass1(object):
"""Test class.
"""
pass
class SomeClass2(object):
"""Test class.
"""
def __init__(self, arg1):
self.arg1 = arg1
def compute(self, arg2):
return self.arg1 + arg2
class SomeClass3():
"""Test class. Does NOT inherit from object.
"""
pass
if __name__ == '__main__':
def test():
"""Make an instance and write the report.
"""
inst = SomeClass2(10)
assert inst.compute(10) == 20
object.report()
test()
%load base_conflict.py
# file: base_conflict.py
from meta_2_3 import with_metaclass
class MetaClass1(type):
pass
class MetaClass2(type):
pass
class BaseClass1(with_metaclass(MetaClass1)):
pass
class BaseClass2(with_metaclass(MetaClass2)):
pass
class DoesNotWorkClass(BaseClass1, BaseClass2):
pass
# file: base_conflict.py
from meta_2_3 import with_metaclass
class MetaClass1(type):
pass
class MetaClass2(type):
pass
class BaseClass1(with_metaclass(MetaClass1)):
pass
class BaseClass2(with_metaclass(MetaClass2)):
pass
class DoesNotWorkClass(BaseClass1, BaseClass2):
pass
---------------------------------------------------------------------------
TypeError Traceback (most recent call last)
<ipython-input-78-cf37774691e1> in <module>()
15 pass
16
---> 17 class DoesNotWorkClass(BaseClass1, BaseClass2):
18 pass
TypeError: metaclass conflict: the metaclass of a derived class must be a (non-strict) subclass of the metaclasses of all its bases
%load submeta.py
# file: submeta.py
from meta_2_3 import with_metaclass
class BaseMetaClass(type):
pass
class SubMetaClass(BaseMetaClass):
pass
class BaseClass1(with_metaclass(BaseMetaClass)):
pass
class BaseClass2(with_metaclass(SubMetaClass)):
pass
class WorkingClass(BaseClass1, BaseClass2):
pass
# file: submeta.py
from meta_2_3 import with_metaclass
class BaseMetaClass(type):
pass
class SubMetaClass(BaseMetaClass):
pass
class BaseClass1(with_metaclass(BaseMetaClass)):
pass
class BaseClass2(with_metaclass(SubMetaClass)):
pass
class WorkingClass(BaseClass1, BaseClass2):
pass
%load classwatcher.py
# file: classwatcher.py
"""Find all defined classes.
Needs Python 3.
"""
import builtins
from collections import Counter
class MultipleInstancesError(Exception):
"""Allow only one instance.
"""
pass
class ClassWatcher(object):
"""After instantiation of this class, all newly defined classes will
be counted.
Only one instance of this class is allowed.
"""
def __new__(cls, only_packages=frozenset(), ignore_packages=frozenset()):
"""
only_packages: positive list of package names
Only these packages will be used.
ignore_packages: negative list of package names
These packages will not be considered.
The names in both sets are checked with `.startwith()'.
This allows to filter for `package` or `package.subpackage` and
so on.
For example, you can include `package` with `only_packages` and
and then exclude `package.subpackage` with ignore_packages`.
"""
if hasattr(cls, '_instance_exists'):
msg = 'Only one instance of ClassWatcher allowed.'
raise MultipleInstancesError(msg)
cls._instance_exists = True
cls.defined_classes = Counter()
cls.activate(only_packages, ignore_packages)
return super().__new__(cls)
@staticmethod
def __build_class__(func, name, *bases, metaclass=type, **kwds):
"""Replacement for the the built-in `__build_class__`.
Use on your own risk.
"""
name = '{}.{}'.format(func.__module__, func.__qualname__)
if not ClassWatcher.only_packages:
add_name = True
else:
add_name = False
for p_name in ClassWatcher.only_packages:
if name.startswith(p_name):
add_name = True
for p_name in ClassWatcher.ignore_packages:
if name.startswith(p_name):
add_name = False
if add_name:
ClassWatcher.defined_classes[name] += 1
cls = ClassWatcher.orig__build_class__(func, name, *bases,
metaclass=metaclass, **kwds)
return cls
@classmethod
def activate(cls, only_packages=frozenset(), ignore_packages=frozenset()):
"""Replace the built-in `__build_class__` with a customer version.
"""
cls.orig__build_class__ = builtins.__build_class__
builtins.__build_class__ = cls.__build_class__
cls.only_packages = frozenset(only_packages)
cls.ignore_packages = frozenset(ignore_packages)
@classmethod
def deactivate(cls):
"""Set built-in `__build_class__` back to real built-in.
"""
builtins.__build_class__ = cls.orig__build_class__
def report(self, limit=20):
"""Show results.
"""
print('total defined classes:', sum(self.defined_classes.values()))
print('total unique classes: ', len(self.defined_classes))
all_names = self.defined_classes.most_common()
width = max(len(name[0]) for name in all_names[:limit])
count_width = 10
print('{:{width}}{:>{count_width}}'.format('Name', 'Count',
width=width, count_width=count_width))
print('#' * (width + count_width))
for counter, (cls_name, count) in enumerate(all_names, 1):
print('{:{width}s}{:{count_width}d}'.format(cls_name, count,
width=width, count_width=count_width))
if counter >= limit:
print('...')
print('Skipped', len(all_names) - counter, 'additional lines.')
break
prepare comes in even earlier than new, and allows you to do a few more things.
%load prepare.py
# file: pepare.py
"""Using `__prepare__` to preserve definition order of attributes.
Needs Python 3.
"""
class AttributeOrderDict(dict):
"""Dict-like object used for recording attribute definition order.
"""
def __init__(self, no_special_methods=True, no_callables=True):
self.member_order = []
self.no_special_methods = no_special_methods
self.no_callables = no_callables
super().__init__()
def __setitem__(self, key, value):
skip = False
# Don't allow setting more than once.
if key in self:
raise AttributeError(
'Attribute {} defined more than once.'.format(key))
# Skip callables if not wanted.
if self.no_callables:
if callable(value):
skip = True
# Skip special methods if not wanted.
if self.no_special_methods:
if key.startswith('__') and key.endswith('__'):
skip = True
if not skip:
self.member_order.append(key)
super().__setitem__(key, value)
class OrderedMeta(type):
"""Meta class that helps to record attribute definition order.
"""
@classmethod
def __prepare__(mcs, name, bases, **kwargs):
return AttributeOrderDict(**kwargs)
def __new__(mcs, name, bases, cdict, **kwargs):
cls = type.__new__(mcs, name, bases, cdict)
cls.member_order = cdict.member_order
cls._closed = True
return cls
# Needed to use up kwargs.
def __init__(cls, name, bases, cdict, **kwargs):
super().__init__(name, bases, cdict)
def __setattr__(cls, name, value):
# Later attribute additions go through here.
if getattr(cls, '_closed', False):
raise AttributeError(
'Cannot set attribute after class definition.')
super().__setattr__(name, value)
if __name__ == '__main__':
class MyClass(metaclass=OrderedMeta, no_callables=False):
"""Test class with extra attribute `member_order`.
"""
attr1 = 1
attr2 = 2
def method1(self):
pass
def method2(self):
pass
attr3 = 3
# attr3 = 3 # uncomment to trigger exception
print(MyClass.member_order)
# MyClass.attr4 = 4 # uncomment to trigger exception
['attr1', 'attr2', 'method1', 'method2', 'attr3']
Contact
blog comments powered by Disqus