__slots__

[flori]

i try to greate somthing like this

class ca(object): __slots__ = ("a",)
class cb(ca): __slots__ = ("a","b")
class cc(ca): __slots__ = ("a","c")
class cd(cb,cc): __slots__ = ("a","b","c","d ")

but i didn't find a simple solution
so i'm using a metaclass that generate a __slots__-free and a
__slots__-version
the __slots__ -free is returned
the __new__ of the base class look for the sloted verion
the classes defines a (class)-attribute __object_slots_ _ with their
new attribute names.

killer? someone a good idea?

my soure (something not important i'm cutting away)

import sys
import types
import inspect
import weakref
#import
#
class const(object):
__slots__=()
cs_classid = intern("_classi d")
cs__all_persite nt_slots__ = intern("__all_p ersitent_slots_ _")
cs__all_nonpers itent_slots__ =
intern("__all_n onpersitent_slo ts__")
cs__all_slots__ = intern("__all_s lots__")
cs__cls_unslote d__ = intern("__cls_u nsloted__")
cs__nonpersiten t_slots__ = intern("__nonpe rsitent_slots__ ")
cs__object_slot s__ = intern("__objec t_slots__")
cs__object_slot s_nomore__ = intern("__objec t_slots_nomore_ _")
cs__slots__ = intern("__slots __")
cs__slots_fixed __ = intern("__slots _fixed__")
cs_lstobjProper ty = intern("_lstobj Property")
cslstSlots = intern("lstSlot s")
cs_sloted_prefi x = intern("_sloted _")
cslstNonPersite ntSlots = intern("lstNonP ersitentSlots")
#
#
class ClCustomBase(ob ject):
""" no really need for this but i like it
may be i need for debugging, if..
"""
pass
#
class ClMetaClass(ClC ustomBase):
#
def __new__(cls, sName, tupBases, dctClassDefinit ion):
""" calculates the class-attribute.
calcs
__all_slots__ := __object_slots_ _ + (for all
tupBases).__all _slots__
__all_nonpersit ent_slots__ := __nonpersitent_ slots__ +
(for all tupBases).__all _nonpersitent_s lots__
__all_persitent _slots__ := __all_slots__ -
__all_nonpersit ent_slots__
"""
# init
objClsUnsloted= None
dctDefinitionLo cals = sys._getframe(1 ).f_locals
# create a (hopefully) unique name
# first i try to use the id(objClsUnslot ed) but it's not known
before creation
# - and i should know it before
dctClassDefinit ion[const.cs_classi d] = sName +
hex(id(tupBases ) ^ id(sys._getfram e(0)) ) ^
id(dctClassDefi nition.get(cons t.cs__object_sl ots__, None))
dctAddInfo = {
const.cslstSlot s: list(),
const.cslstNonP ersitentSlots: list(),
}
# prepare definition of unsloted
cls.calculateSl ots(sName, tupBases, dctClassDefinit ion,
dctAddInfo, dctDefinitionLo cals)
objClsUnsloted = type(sName, tupBases, dctClassDefinit ion)
#
# prepare definition of sloted
dctClassDefinit ion[const.cs__slots __] =
tuple(dctAddInf o[const.cslstSlot s])
dctClassDefinit ion[const.cs__cls_u nsloted__] = objClsUnsloted
# definition of the sloted class
objClsSloted = type(const.cs_s loted_prefix+sN ame,
(objClsUnsloted ,), dctClassDefinit ion)
objClsUnsloted. __cls_sloted__ = objClsSloted
dctDefinitionLo cals[const.cs_sloted _prefix+sName] =
objClsSloted
# done
dctDefinitionLo cals = None # help gc
return objClsUnsloted
#
def __init__(self, sName, tupBases, dctClassDefinit ion):
super(ClMetaBas e,self).__init_ _(sName, tupBases,
dctClassDefinit ion)
#
def calculateSlots( cls, sName, tupBases, dctClassDefinit ion,
dctAddInfo, dctDefinitionLo cals):
""" calculates all slot things needed fo the sloted version -
ah you geuess it
"""
lstSlots=dctAdd Info[const.cslstSlot s]
tupClassSlots =
dctClassDefinit ion.get(const.c s__object_slots __, None)
if isinstance(tupC lassSlots, basestring):
raise TypeError, "in class "+sName+":__obj ect_slots__ is
defined as string (may be you forgotten a ,?)"
if tupClassSlots is None:
# or should we simply pass ?
raise TypeError, "The class "+sName+" (with ClBase as
superclass) must have a __object_slots_ _-attribute like
\n__object_slot s__ = "+str(tuple(lst Slots))+".\n"+" , but
-"+str(tupClassS lots)+"- found \n"
else:
lstSlots.extend ( tupClassSlots )
#
lstNonPersitent Slots=dctAddInf o[const.cslstNonP ersitentSlots]
lstNonPersitent Slots.extend(
dctClassDefinit ion.get(const.c s__nonpersitent _slots__, ()) )
#
for objCls in tupBases:
tupSubSlots = getattr(objCls, const.cs__all_s lots__, None)
if tupSubSlots is not None:
lstSlots.extend (tupSubSlots)
#
tupSubSlots = getattr(objCls,
const.cs__nonpe rsitent_slots__ , None)
if tupSubSlots is not None:
lstNonPersitent Slots.extend(tu pSubSlots)
#no more needed
#lstBasesUnslot ed.append(getat tr(objCls,
cs__cls_unslote d__, objCls))
#
#no more needed
#dctAddInfo["tupBasesUnslot ed"] = tuple( lstBasesUnslote d )
#add nonpersitent slots to the all slots
#
lstNonPersitent Slots.sort()
for iIndex in xrange(len(lstN onPersitentSlot s)-2, -1, -1):
if lstNonPersitent Slots[iIndex] ==
lstNonPersitent Slots[iIndex+1]:
del lstNonPersitent Slots[iIndex+1]
#
#
tupObjectSlotsN oMore =
dctClassDefinit ion.get(const.c s__object_slots _nomore__, None)
if tupObjectSlotsN oMore is not None:
for sObjectSlotsNoM ore in tupObjectSlotsN oMore:
try:
lstNonPersitent Slots.remove(sO bjectSlotsNoMor e)
except ValueError:
pass
#
#
#
lstSlots.extend (lstNonPersiten tSlots)
#sort slots and remove doubles
lstSlots.sort()
for iIndex in xrange(len(lstS lots)-2, -1, -1):
s=intern(lstSlo ts[iIndex])
if s == lstSlots[iIndex+1]:
del lstSlots[iIndex+1]
else:
lstSlots[iIndex]=s
#
#
if tupObjectSlotsN oMore is not None:
for sObjectSlotsNoM ore in tupObjectSlotsN oMore:
try:
lstSlots.remove (sObjectSlotsNo More)
except ValueError:
pass
#
#
#
# non persitent and persitent are disjunct ==> persitent :=
slots - non persitenten
lstPersitentSlo ts=lstSlots[:]
for sKey in lstNonPersitent Slots:
lstPersitentSlo ts.remove(sKey)
#
# update class definition
dctClassDefinit ion[const.cs__all_s lots__]=tuple(lstSlots )
dctClassDefinit ion[const.cs__all_n onpersitent_slo ts__]=tuple(lstNonPe rsitentSlots)
dctClassDefinit ion[const.cs__all_p ersitent_slots_ _]=tuple(lstPersi tentSlots)
#
# normally there isn't a __slots__ - definition, but if we
cannot trust it
try:
dctClassDefinit ion.remove(cons t.cs__slots__)
except:
pass
dctDefinitionLo cals = None # help gc
#
calculateSlots= classmethod(cal culateSlots)
#
#
class ClBase(ClCustom Base):
""" base of all
"""
#
__metaclass__ = ClMetaClass
__object_slots_ _ = ()
#__object_slots __ = ("gna", "__weakref_ _", )
#__weakref__ = 1
#
def __new__(cls, *lstArgs, **kwArgs):
self = super(ClBase,
cls).__new__(ge tattr(cls,"__cl s_sloted__",cls ))
return self
#now __init__ is called
# / __new__
def __init__(self, *lstArgs, **kwArgs):
super(ClBase, self).__init__( )
#
#

[Alex Martelli]

flori wrote:

i try to greate somthing like this

class ca(object): __slots__ = ("a",)
class cb(ca): __slots__ = ("a","b")
class cc(ca): __slots__ = ("a","c")
class cd(cb,cc): __slots__ = ("a","b","c","d ")

but i didn't find a simple solution

....for WHAT problem...?

class ca(object): __slots__ = 'a', .... class cb(ca): __slots__ = 'b', .... xx=cb()
xx.a=23

slots are inherited... is this perhaps what was unclear to you?
Alex

[Ulrich Petri]

"flori" <fg****@inneo.d e> schrieb im Newsbeitrag
news:9d******** *************** ***@posting.goo gle.com...

i try to greate somthing like this

class ca(object): __slots__ = ("a",)
class cb(ca): __slots__ = ("a","b")
class cc(ca): __slots__ = ("a","c")
class cd(cb,cc): __slots__ = ("a","b","c","d ")

but i didn't find a simple solution
so i'm using a metaclass that generate a __slots__-free and a
__slots__-version
the __slots__ -free is returned
the __new__ of the base class look for the sloted verion
the classes defines a (class)-attribute __object_slots_ _ with their
new attribute names.

killer? someone a good idea?

my soure (something not important i'm cutting away)

<snip lots of code that looks like java in python>

What exactly are you trying to accomplish?

Ciao Ulrich

[flori]

fg****@inneo.de (flori) wrote in message news:<9d******* *************** ****@posting.go ogle.com>...

i try to greate somthing like this

sorry for silly question
my prob is when i try to do this
class ca(object): __slots__ = ("a",)
class cb(ca): __slots__ = ("a","b")
class cc(ca): __slots__ = ("a","c")
class cd(cb,cc): __slots__ = ("a","b","c","d ")

in line "class cd(.." thorws a exception
TypeError: multiple bases have instance lay-out conflict

(if cb (or cc) doesn't define __slots__ it works.)

My only solution is to generate a "unsloted" class (_ca)
and genererate a "sloted" class (ca) with only the __slots__ attribute.
when i had to inherit the class i use the "unsloted" class.

like that:
class _ca(object): pass
class ca(object): __slots__ = ("a",)
class _cb(_ca): pass
class cb(_cb): __slots__ = ("a","b")
class _cc(_ca): pass
class cc(_cc): __slots__ = ("a","c")
class _cd(_cb,_cc): pass
class cd(_cd): __slots__ = ("a","b","c","d ")

my source does this a metaclass

my question is is their a nicer way to do this?

[Francis Avila]

"flori" <fg****@inneo.d e> wrote in message
news:9d******** *************** ***@posting.goo gle.com...

fg****@inneo.de (flori) wrote in message

news:<9d******* *************** ****@posting.go ogle.com>...

i try to greate somthing like this

my question is is their a nicer way to do this?

Yes. Don't use slots! Why are you using slots?!
--
Francis Avila

[Alex Martelli]

flori wrote:
...

class _ca(object): pass
class ca(object): __slots__ = ("a",)
class _cb(_ca): pass
class cb(_cb): __slots__ = ("a","b")
class _cc(_ca): pass
class cc(_cc): __slots__ = ("a","c")
class _cd(_cb,_cc): pass
class cd(_cd): __slots__ = ("a","b","c","d ")

my source does this a metaclass

my question is is their a nicer way to do this?

Yes: and the nicer way to get exactly the same result is,
remove all those useless __slots__ declarations. What do
you think they're buying you...?

Maybe you haven't tried the elementary test...:

x = cd()
x.pippo = 'ma va!'
print x.pippo

....?

Once _any_ base class allows instances to have a per-instance
dictionary (which is the case for all classes _except_ the very
special ones where we insert the very-special-purpose and often-
misunderstood *optimization* '__slots__'.... ), then of course
*all* instances of that class (and an instance of a subclass is
an instance of the base class too) will have a per-instance
dictionary. And once you have a per-instance dictionary anyway,
adding __slots__ is just silly -- can't make the per-inst dict
disappear, so no memory is actually saved, no attribute setting
is prohibited, etc, etc.

And yes, you can't multiply inherit from several classes that
define __slots__ (not even identical slots, if I recall correctly),
just like you can't multiply inherit from different built-in
types (with non-null instance-layout constraints), nor even from
one built-in type and a separate class with __slots__. Defining
__slots__ does constrain the instance layout, and generally speaking
Python doesn't even _try_ to sort out any case of multiple separate
constraints on instance layout.

__slots__ is meant as a special-purpose memory optimization to
save the per-instance memory cost of a dictionary when you're
defining small classes of which a huge number of instances are
meant to exist at the same time, so that the several tens of bytes
per instance that a dictionary might cost is an unacceptable price
to pay compared to the "actual" footprint of each of the huge
number of instances. For this use case, which is really the only
one that __slots__ is intended for, multiple inheritance isn't
really indispensable. All in all, I doubt, therefore, that these
limitations will go away.
Alex

[Brad Clements]

_
"Alex Martelli" <al***@aleax.it > wrote in message
news:Z4******** *********@news2 .tin.it...

__slots__ is meant as a special-purpose memory optimization to
save the per-instance memory cost of a dictionary when you're
defining small classes of which a huge number of instances are
meant to exist at the same time, so that the several tens of bytes
per instance that a dictionary might cost is an unacceptable price
to pay compared to the "actual" footprint of each of the huge
number of instances.

And I'd like to add.. I was starting a new embedded Python project when
__slots__ came out, the documentation at the time didn't make clear their
intended use of storage optimization. Rather what *I* read into it was a
way to reduce the possibility of error by mistyping an instance attribute.
Now, why I thought this was neat is beyond me, since in my vast 4 years of
Python programming I've never really had that problem anyway.

But nonetheless I slotted myself into a corner anyway. Now I have to undo
all those nasty slots statements.

My advice.. forget __slots__!

[Patrick Maupin]

Brad Clements wrote:

But nonetheless I slotted myself into a corner anyway. Now I have to undo
all those nasty slots statements.

My advice.. forget __slots__!

In general, I agree. However, if one is contemplating
performance optimizations by "Pyrexcizin g" Python code
and creating new embedded types, one might find that the
use of __slots__ is a viable small step on the path to
"cdef object." (If one is the sort who wants to keep testing
and debugging in actual Python instead of Pyrex until the
very last minute.)

One might also find, while taking this path, that the
use of __slots__ by itself makes SOME applications run
fast enough without going all the way to Pyrex. At this
point, one might reasonably conclude (despite the
oft-repeated claim that __slots__ are an un-Pythonic
"premature optimization") that the ability to avoid writing
a C extension by merely adding a single line of (basically
non-executable) code to a class or two is unalloyed goodness.

And in the spare time one creates by not writing a C extension,
one may be forgiven for contemplating whether the original
"premature optimization" claim referred to the code which USES
__slots__ (as some seem to interpret), or to the language feature
of __slots__ itself, which is admittedly not as nice as something
like Psyco has the potential to be, but which nonetheless has the
same advantage over some future optimizations that list
comprehensions have over generator expressions, namely that it
is available NOW, in a working and seemingly stable interpreter.

Regards,
Pat

[Mirko Zeibig]

On Wed, 12 Nov 2003 at 04:52 GMT, Patrick Maupin <pm*****@speake asy.net> wrote:

Brad Clements wrote:

But nonetheless I slotted myself into a corner anyway. Now I have to undo
all those nasty slots statements.

My advice.. forget __slots__!

One might also find, while taking this path, that the
use of __slots__ by itself makes SOME applications run
fast enough without going all the way to Pyrex. At this
point, one might reasonably conclude (despite the
oft-repeated claim that __slots__ are an un-Pythonic
"premature optimization") that the ability to avoid writing
a C extension by merely adding a single line of (basically
non-executable) code to a class or two is unalloyed goodness.

Just as an info: I ran some tests using slots vs. "normal classes" vs. tuples,
here are the definitions for the classes:

class NoSlots(object) :
def __init__(self, *args):
self.id, self.weight = args

class Slots(object):
__slots__ = ('id', 'weight')
def __init__(self, *args):
self.id, self.weight = args

def createtuple(id, weight):
return (id, weight)

def create(func, count):
w = timer.create("c reating %s" % func)
l = [ func(str(id), id*2) for id in xrange(count) ]
w.stop()
showMemUsage(fu nc)
return l

def usetupledirectl y(func, count):
w = timer.create("c reating %s" % func)
l = [ (str(id), id*2) for id in xrange(count) ]
w.stop()
showMemUsage(fu nc)
return l
created lot's of these and hold them in a list. I measured memory-usage very
roughly using /proc/self/status (VmRSS):

[mize@lxmize python]$ for i in 1000 10000 100000 ; do ./memusage_speed_ tuple_vs_simple _classes.py $i; done
*************** *************** *************** *************** *************** *****
Creating 1000 objects
MemUsage <class '__main__.NoSlo ts'>: 2780 kB
MemUsage <class '__main__.Slots '>: 80 kB
MemUsage <function createtuple at 0x8242214>: 88 kB
MemUsage <function usetupledirectl y at 0x8214a84>: 88 kB
Runtime UsrTim SysTim Calls NestC PendC TimerName
0.216 0.130 0.000 1 0 0 total
0.039 0.040 0.000 1 0 0 creating <class '__main__.NoSlo ts'>
0.035 0.040 0.000 1 0 0 creating <class '__main__.Slots '>
0.019 0.020 0.000 1 0 0 creating <function createtuple at 0x8242214>
0.011 0.010 0.000 1 0 0 creating <function usetupledirectl y at 0x8214a84>
0.007 0.010 0.000 1 0 0 deleting stuff, disablegc=0
*************** *************** *************** *************** *************** *****
Creating 10000 objects
MemUsage <class '__main__.NoSlo ts'>: 4752 kB
MemUsage <class '__main__.Slots '>: 800 kB
MemUsage <function createtuple at 0x8242214>: 860 kB
MemUsage <function usetupledirectl y at 0x8214a84>: 864 kB
Runtime UsrTim SysTim Calls NestC PendC TimerName
1.351 1.230 0.020 1 0 0 total
0.438 0.440 0.000 1 0 0 creating <class '__main__.NoSlo ts'>
0.365 0.370 0.000 1 0 0 creating <class '__main__.Slots '>
0.186 0.180 0.000 1 0 0 creating <function createtuple at 0x8242214>
0.148 0.130 0.020 1 0 0 creating <function usetupledirectl y at 0x8214a84>
0.100 0.100 0.000 1 0 0 deleting stuff, disablegc=0
*************** *************** *************** *************** *************** *****
Creating 100000 objects
MemUsage <class '__main__.NoSlo ts'>: 25160 kB
MemUsage <class '__main__.Slots '>: 8528 kB
MemUsage <function createtuple at 0x8242214>: 9308 kB
MemUsage <function usetupledirectl y at 0x8214a84>: 9308 kB
Runtime UsrTim SysTim Calls NestC PendC TimerName
16.979 16.230 0.610 1 0 0 total
5.448 5.230 0.200 1 0 0 creating <class '__main__.NoSlo ts'>
4.792 4.640 0.140 1 0 0 creating <class '__main__.Slots '>
3.111 2.970 0.140 1 0 0 creating <function createtuple at 0x8242214>
2.805 2.670 0.130 1 0 0 creating <function usetupledirectl y at 0x8214a84>
0.716 0.710 0.000 1 0 0 deleting stuff, disablegc=0
[mize@lxmize python]$

So while there is a big memory saving in using __slots__, the time savings I
observed are relatively small. Test system is a Pentium Celeron 500MHz with 256
MB running the Python 2.2 on Linux. Of course never trust measurements you
didn't manipulate yourself ;-).

Regards
Mirko

[Patrick Maupin]

Mirko Zeibig wrote:

So while there is a big memory saving in using __slots__, the time savings I
observed are relatively small. Test system is a Pentium Celeron 500MHz with 256
MB running the Python 2.2 on Linux. Of course never trust measurements you
didn't manipulate yourself ;-).

Always excellent advice, which I usually attempt to follow. I think
I tried to make it fairly clear in the post you were responding to that
__slots__ would not always give great results. However, some empirical
evidence I have (aka measurements I manipulated myself :) shows that in
_some_ cases the time savings can be substantial. In the cases I have
seen, the time savings actually follow the memory savings. In other
words, if __slots__ can make the difference between paging or not, or
even, possibly, lots of RAM cache misses or not, then it could be
worthwhile. (As your detailed post makes clear, saving 70% of the RAM
by using __slots__ on simple objects is not out of the question, although
I do find it somewhat surprising that on your small objects the __slots__
requirement was even smaller than a tuple!)

Also, if I am not misremembering (it was quite a few months back),
I think the speed benefit was more pronounced in 2.3 than in 2.2,
so maybe there are some additional interpreter optimizations there.

FWIW, my use case for __slots__ involved creating at least tens of
millions of objects, and I was not exaggerating when I wrote that
I was on the way to writing a C extension (in Pyrex).

Regards,
Pat