1 |
Author: zmedico |
2 |
Date: 2008-06-25 22:36:19 +0000 (Wed, 25 Jun 2008) |
3 |
New Revision: 10790 |
4 |
|
5 |
Modified: |
6 |
main/trunk/pym/_emerge/__init__.py |
7 |
main/trunk/pym/portage/cache/mappings.py |
8 |
main/trunk/pym/portage/dbapi/bintree.py |
9 |
main/trunk/pym/portage/dbapi/porttree.py |
10 |
Log: |
11 |
Add a generic portage.cache.mappings.slot_dict_class() function which |
12 |
generates mapping classes that behave similar to a dict but store |
13 |
values as object attributes that are allocated via __slots__. Instances |
14 |
of these objects have a smaller memory footprint than a normal dict object. |
15 |
These classes are used to reduce the memory footprint of the dbapi.aux_get() |
16 |
caches and the Package.metadata attribute. |
17 |
|
18 |
|
19 |
Modified: main/trunk/pym/_emerge/__init__.py |
20 |
=================================================================== |
21 |
--- main/trunk/pym/_emerge/__init__.py 2008-06-25 20:47:38 UTC (rev 10789) |
22 |
+++ main/trunk/pym/_emerge/__init__.py 2008-06-25 22:36:19 UTC (rev 10790) |
23 |
@@ -1371,108 +1371,33 @@ |
24 |
return True |
25 |
return False |
26 |
|
27 |
-class _PackageMetadataWrapper(object): |
28 |
+_all_metadata_keys = set(x for x in portage.auxdbkeys \ |
29 |
+ if not x.startswith("UNUSED_")) |
30 |
+_all_metadata_keys.discard("CDEPEND") |
31 |
+_all_metadata_keys.update(Package.metadata_keys) |
32 |
+ |
33 |
+from portage.cache.mappings import slot_dict_class |
34 |
+_PackageMetadataWrapperBase = slot_dict_class(_all_metadata_keys) |
35 |
+ |
36 |
+class _PackageMetadataWrapper(_PackageMetadataWrapperBase): |
37 |
""" |
38 |
Detect metadata updates and synchronize Package attributes. |
39 |
""" |
40 |
- _keys = set(x for x in portage.auxdbkeys \ |
41 |
- if not x.startswith("UNUSED_")) |
42 |
- _keys.discard("CDEPEND") |
43 |
- _keys.update(Package.metadata_keys) |
44 |
- _keys = tuple(sorted(_keys)) |
45 |
- __slots__ = ("__weakref__", "_pkg") + tuple("_val_" + k for k in _keys) |
46 |
+ |
47 |
+ __slots__ = ("_pkg",) |
48 |
_wrapped_keys = frozenset( |
49 |
["COUNTER", "INHERITED", "IUSE", "SLOT", "USE", "_mtime_"]) |
50 |
|
51 |
def __init__(self, pkg, metadata): |
52 |
+ _PackageMetadataWrapperBase.__init__(self) |
53 |
self._pkg = pkg |
54 |
self.update(metadata) |
55 |
|
56 |
- def __iter__(self): |
57 |
- for k, v in self.iteritems(): |
58 |
- yield k |
59 |
- |
60 |
- def __len__(self): |
61 |
- l = 0 |
62 |
- for i in self.iteritems(): |
63 |
- l += 1 |
64 |
- return l |
65 |
- |
66 |
- def keys(self): |
67 |
- return list(self) |
68 |
- |
69 |
- def iteritems(self): |
70 |
- for k in self._keys: |
71 |
- try: |
72 |
- yield (k, getattr(self, "_val_" + k)) |
73 |
- except AttributeError: |
74 |
- pass |
75 |
- |
76 |
- def items(self): |
77 |
- return list(self.iteritems()) |
78 |
- |
79 |
- def itervalues(self): |
80 |
- for k, v in self.itervalues(): |
81 |
- yield v |
82 |
- |
83 |
- def values(self): |
84 |
- return list(self.itervalues()) |
85 |
- |
86 |
- def __delitem__(self, k): |
87 |
- try: |
88 |
- delattr(self, "_val_" + k) |
89 |
- except AttributeError: |
90 |
- raise KeyError(k) |
91 |
- |
92 |
def __setitem__(self, k, v): |
93 |
- setattr(self, "_val_" + k, v) |
94 |
+ _PackageMetadataWrapperBase.__setitem__(self, k, v) |
95 |
if k in self._wrapped_keys: |
96 |
getattr(self, "_set_" + k.lower())(k, v) |
97 |
|
98 |
- def update(self, d): |
99 |
- i = getattr(d, "iteritems", None) |
100 |
- if i is None: |
101 |
- i = d |
102 |
- else: |
103 |
- i = i() |
104 |
- for k, v in i: |
105 |
- self[k] = v |
106 |
- |
107 |
- def __getitem__(self, k): |
108 |
- try: |
109 |
- return getattr(self, "_val_" + k) |
110 |
- except AttributeError: |
111 |
- raise KeyError(k) |
112 |
- |
113 |
- def get(self, key, default=None): |
114 |
- try: |
115 |
- return self[key] |
116 |
- except KeyError: |
117 |
- return default |
118 |
- |
119 |
- def __contains__(self, k): |
120 |
- return hasattr(self, "_val_" + k) |
121 |
- |
122 |
- def pop(self, key, *args): |
123 |
- if len(args) > 1: |
124 |
- raise TypeError("pop expected at most 2 arguments, got " + \ |
125 |
- repr(1 + len(args))) |
126 |
- try: |
127 |
- value = self[key] |
128 |
- except KeyError: |
129 |
- if args: |
130 |
- return args[0] |
131 |
- raise |
132 |
- del self[key] |
133 |
- return value |
134 |
- |
135 |
- def clear(self): |
136 |
- for k in self._keys: |
137 |
- try: |
138 |
- delattr(self, "_val_" + k) |
139 |
- except AttributError: |
140 |
- pass |
141 |
- |
142 |
def _set_inherited(self, k, v): |
143 |
if isinstance(v, basestring): |
144 |
v = frozenset(v.split()) |
145 |
|
146 |
Modified: main/trunk/pym/portage/cache/mappings.py |
147 |
=================================================================== |
148 |
--- main/trunk/pym/portage/cache/mappings.py 2008-06-25 20:47:38 UTC (rev 10789) |
149 |
+++ main/trunk/pym/portage/cache/mappings.py 2008-06-25 22:36:19 UTC (rev 10790) |
150 |
@@ -4,6 +4,7 @@ |
151 |
# $Id$ |
152 |
|
153 |
import UserDict |
154 |
+import weakref |
155 |
|
156 |
class ProtectedDict(UserDict.DictMixin): |
157 |
""" |
158 |
@@ -101,3 +102,128 @@ |
159 |
self.pull = None |
160 |
return key in self.d |
161 |
|
162 |
+_slot_dict_classes = weakref.WeakValueDictionary() |
163 |
+ |
164 |
+def slot_dict_class(keys): |
165 |
+ if isinstance(keys, frozenset): |
166 |
+ keys_set = keys |
167 |
+ else: |
168 |
+ keys_set = frozenset(keys) |
169 |
+ v = _slot_dict_classes.get(keys_set) |
170 |
+ if v is None: |
171 |
+ |
172 |
+ class SlotDict(object): |
173 |
+ |
174 |
+ _keys = keys_set |
175 |
+ __slots__ = ("__weakref__",) + tuple("_val_" + k for k in _keys) |
176 |
+ |
177 |
+ def __iter__(self): |
178 |
+ for k, v in self.iteritems(): |
179 |
+ yield k |
180 |
+ |
181 |
+ def __len__(self): |
182 |
+ l = 0 |
183 |
+ for i in self.iteritems(): |
184 |
+ l += 1 |
185 |
+ return l |
186 |
+ |
187 |
+ def keys(self): |
188 |
+ return list(self) |
189 |
+ |
190 |
+ def iteritems(self): |
191 |
+ for k in self._keys: |
192 |
+ try: |
193 |
+ yield (k, getattr(self, "_val_" + k)) |
194 |
+ except AttributeError: |
195 |
+ pass |
196 |
+ |
197 |
+ def items(self): |
198 |
+ return list(self.iteritems()) |
199 |
+ |
200 |
+ def itervalues(self): |
201 |
+ for k, v in self.itervalues(): |
202 |
+ yield v |
203 |
+ |
204 |
+ def values(self): |
205 |
+ return list(self.itervalues()) |
206 |
+ |
207 |
+ def __delitem__(self, k): |
208 |
+ try: |
209 |
+ delattr(self, "_val_" + k) |
210 |
+ except AttributeError: |
211 |
+ raise KeyError(k) |
212 |
+ |
213 |
+ def __setitem__(self, k, v): |
214 |
+ setattr(self, "_val_" + k, v) |
215 |
+ |
216 |
+ def setdefault(self, key, default=None): |
217 |
+ try: |
218 |
+ return self[key] |
219 |
+ except KeyError: |
220 |
+ self[key] = default |
221 |
+ return default |
222 |
+ |
223 |
+ def update(self, d): |
224 |
+ i = getattr(d, "iteritems", None) |
225 |
+ if i is None: |
226 |
+ i = d |
227 |
+ else: |
228 |
+ i = i() |
229 |
+ for k, v in i: |
230 |
+ self[k] = v |
231 |
+ |
232 |
+ def __getitem__(self, k): |
233 |
+ try: |
234 |
+ return getattr(self, "_val_" + k) |
235 |
+ except AttributeError: |
236 |
+ raise KeyError(k) |
237 |
+ |
238 |
+ def get(self, key, default=None): |
239 |
+ try: |
240 |
+ return self[key] |
241 |
+ except KeyError: |
242 |
+ return default |
243 |
+ |
244 |
+ def __contains__(self, k): |
245 |
+ return hasattr(self, "_val_" + k) |
246 |
+ |
247 |
+ def has_key(self, k): |
248 |
+ return k in self |
249 |
+ |
250 |
+ def pop(self, key, *args): |
251 |
+ if len(args) > 1: |
252 |
+ raise TypeError( |
253 |
+ "pop expected at most 2 arguments, got " + \ |
254 |
+ repr(1 + len(args))) |
255 |
+ try: |
256 |
+ value = self[key] |
257 |
+ except KeyError: |
258 |
+ if args: |
259 |
+ return args[0] |
260 |
+ raise |
261 |
+ del self[key] |
262 |
+ return value |
263 |
+ |
264 |
+ def popitem(self): |
265 |
+ try: |
266 |
+ k, v = self.iteritems().next() |
267 |
+ except StopIteration: |
268 |
+ raise KeyError, 'container is empty' |
269 |
+ del self[k] |
270 |
+ return (k, v) |
271 |
+ |
272 |
+ def copy(self): |
273 |
+ c = self.__class__() |
274 |
+ c.update(self) |
275 |
+ return c |
276 |
+ |
277 |
+ def clear(self): |
278 |
+ for k in self._keys: |
279 |
+ try: |
280 |
+ delattr(self, "_val_" + k) |
281 |
+ except AttributError: |
282 |
+ pass |
283 |
+ |
284 |
+ v = SlotDict |
285 |
+ _slot_dict_classes[keys_set] = v |
286 |
+ return v |
287 |
|
288 |
Modified: main/trunk/pym/portage/dbapi/bintree.py |
289 |
=================================================================== |
290 |
--- main/trunk/pym/portage/dbapi/bintree.py 2008-06-25 20:47:38 UTC (rev 10789) |
291 |
+++ main/trunk/pym/portage/dbapi/bintree.py 2008-06-25 22:36:19 UTC (rev 10790) |
292 |
@@ -2,6 +2,7 @@ |
293 |
# Distributed under the terms of the GNU General Public License v2 |
294 |
# $Id$ |
295 |
|
296 |
+from portage.cache.mappings import slot_dict_class |
297 |
from portage.dep import isvalidatom, isjustname, dep_getkey, match_from_list |
298 |
from portage.dbapi.virtual import fakedbapi |
299 |
from portage.exception import InvalidPackageName, InvalidAtom, \ |
300 |
@@ -33,6 +34,7 @@ |
301 |
["CHOST", "DEPEND", "EAPI", "IUSE", "KEYWORDS", |
302 |
"LICENSE", "PDEPEND", "PROVIDE", |
303 |
"RDEPEND", "repository", "RESTRICT", "SLOT", "USE"]) |
304 |
+ self._aux_cache_slot_dict = slot_dict_class(self._aux_cache_keys) |
305 |
self._aux_cache = {} |
306 |
|
307 |
def match(self, *pargs, **kwargs): |
308 |
@@ -75,7 +77,7 @@ |
309 |
if not mydata.setdefault("EAPI", "0"): |
310 |
mydata["EAPI"] = "0" |
311 |
if cache_me: |
312 |
- aux_cache = {} |
313 |
+ aux_cache = self._aux_cache_slot_dict() |
314 |
for x in self._aux_cache_keys: |
315 |
aux_cache[x] = mydata.get(x, "") |
316 |
self._aux_cache[mycpv] = aux_cache |
317 |
|
318 |
Modified: main/trunk/pym/portage/dbapi/porttree.py |
319 |
=================================================================== |
320 |
--- main/trunk/pym/portage/dbapi/porttree.py 2008-06-25 20:47:38 UTC (rev 10789) |
321 |
+++ main/trunk/pym/portage/dbapi/porttree.py 2008-06-25 22:36:19 UTC (rev 10790) |
322 |
@@ -3,6 +3,7 @@ |
323 |
# $Id$ |
324 |
|
325 |
from portage.cache.cache_errors import CacheError |
326 |
+from portage.cache.mappings import slot_dict_class |
327 |
from portage.const import REPO_NAME_LOC |
328 |
from portage.data import portage_gid, secpass |
329 |
from portage.dbapi import dbapi |
330 |
@@ -138,6 +139,10 @@ |
331 |
["DEPEND", "EAPI", "IUSE", "KEYWORDS", "LICENSE", |
332 |
"PDEPEND", "PROVIDE", "RDEPEND", "repository", |
333 |
"RESTRICT", "SLOT"]) |
334 |
+ |
335 |
+ # Repoman modifies _aux_cache_keys, so delay _aux_cache_slot_dict |
336 |
+ # initialization until the first aux_get call. |
337 |
+ self._aux_cache_slot_dict = None |
338 |
self._aux_cache = {} |
339 |
self._broken_ebuilds = set() |
340 |
|
341 |
@@ -386,7 +391,10 @@ |
342 |
returnme.append(mydata.get(x,"")) |
343 |
|
344 |
if cache_me: |
345 |
- aux_cache = {} |
346 |
+ if self._aux_cache_slot_dict is None: |
347 |
+ self._aux_cache_slot_dict = \ |
348 |
+ slot_dict_class(self._aux_cache_keys) |
349 |
+ aux_cache = self._aux_cache_slot_dict() |
350 |
for x in self._aux_cache_keys: |
351 |
aux_cache[x] = mydata.get(x, "") |
352 |
self._aux_cache[mycpv] = aux_cache |
353 |
@@ -866,4 +874,3 @@ |
354 |
except Exception, e: |
355 |
pass |
356 |
return myslot |
357 |
- |
358 |
|
359 |
-- |
360 |
gentoo-commits@l.g.o mailing list |