[gentoo-commits] portage r10781 - in main/branches/prefix/pym: _emerge portage portage/dbapi

[Fabian Groffen (grobian)]

Author: grobian
Date: 2008-06-25 09:00:23 +0000 (Wed, 25 Jun 2008)
New Revision: 10781

Modified:
   main/branches/prefix/pym/_emerge/__init__.py
   main/branches/prefix/pym/portage/__init__.py
   main/branches/prefix/pym/portage/dbapi/vartree.py
   main/branches/prefix/pym/portage/dep.py
Log:
   Merged from trunk 10770:10780

   | 10771   | Bug #229233 - Fix Atom -> str breakage in the                |
   | zmedico | depgraph.display_problems() package.provided warning         |
   |         | message.                                                     |
   
   | 10772   | Use weakref.WeakValueDictionary to make cached Atom          |
   | zmedico | instances eligible for garbage collection when no strong     |
   |         | references remain.                                           |
   
   | 10773   | If dep calculation time exceeds 20 seconds then              |
   | zmedico | automatically enable "complete" mode since any performance   |
   |         | difference is not as likely to be noticed by the user after  |
   |         | this much time has passed.                                   |
   
   | 10774   | Fix spelling of depgraph._complete_threshold.                |
   | zmedico |                                                              |
   
   | 10775   | Bug #229069 - Before deleting the depgraph, break references |
   | zmedico | pointing to the depgraph from Package instances in the merge |
   |         | list. This helps reduce the heap size a lot.                 |
   
   | 10776   | For the Package.metadata attribute, only implement the dict  |
   | zmedico | interface instead of actually inheriting from dict. This     |
   |         | slightly decreases the memory footprint by defining          |
   |         | __slots__ and storing items as object attributes.            |
   
   | 10777   | Remove the BlockerDB._blocker_cache attribute and just       |
   | zmedico | create new instances on demand instead of taking space on    |
   |         | the heap.                                                    |
   
   | 10778   | Make clear_caches() call portage.dircache.clear().           |
   | zmedico |                                                              |
   
   | 10779   | Use os.listdir() instead of portage.listdir() to avoid       |
   | zmedico | needless caching of directories that only need to be listed  |
   |         | once.                                                        |
   
   | 10780   | Implement __contains__, pop() and clear() for                |
   | zmedico | Package.metadata attributes.                                 |


Modified: main/branches/prefix/pym/_emerge/__init__.py
===================================================================
--- main/branches/prefix/pym/_emerge/__init__.py	2008-06-25 08:42:02 UTC (rev 10780)
+++ main/branches/prefix/pym/_emerge/__init__.py	2008-06-25 09:00:23 UTC (rev 10781)
@@ -1312,7 +1312,7 @@
 	def __init__(self, **kwargs):
 		Task.__init__(self, **kwargs)
 		self.root = self.root_config.root
-		self.metadata = self._metadata_wrapper(self, self.metadata)
+		self.metadata = _PackageMetadataWrapper(self, self.metadata)
 		self.cp = portage.cpv_getkey(self.cpv)
 		self.slot_atom = portage.dep.Atom("%s:%s" % (self.cp, self.slot))
 		self.category, self.pf = portage.catsplit(self.cpv)
@@ -1359,60 +1359,6 @@
 						chain((re.escape(x) for x in all), iuse_implicit)))
 			return object.__getattribute__(self, name)
 
-	class _metadata_wrapper(dict):
-		"""
-		Detect metadata updates and synchronize Package attributes.
-		"""
-		_wrapped_keys = frozenset(
-			["COUNTER", "INHERITED", "IUSE", "SLOT", "USE", "_mtime_"])
-
-		def __init__(self, pkg, metadata):
-			dict.__init__(self)
-			self._pkg = pkg
-			i = getattr(metadata, "iteritems", None)
-			if i is None:
-				i = metadata
-			else:
-				i = i()
-			for k, v in i:
-				self[k] = v
-
-		def __setitem__(self, k, v):
-			dict.__setitem__(self, k, v)
-			if k in self._wrapped_keys:
-				getattr(self, "_set_" + k.lower())(k, v)
-
-		def _set_inherited(self, k, v):
-			if isinstance(v, basestring):
-				v = frozenset(v.split())
-			self._pkg.inherited = v
-
-		def _set_iuse(self, k, v):
-			self._pkg.iuse = self._pkg._iuse(
-				v.split(), self._pkg.root_config.iuse_implicit)
-
-		def _set_slot(self, k, v):
-			self._pkg.slot = v
-
-		def _set_use(self, k, v):
-			self._pkg.use = self._pkg._use(v.split())
-
-		def _set_counter(self, k, v):
-			if isinstance(v, basestring):
-				try:
-					v = int(v.strip())
-				except ValueError:
-					v = 0
-			self._pkg.counter = v
-
-		def _set__mtime_(self, k, v):
-			if isinstance(v, basestring):
-				try:
-					v = float(v.strip())
-				except ValueError:
-					v = 0
-			self._pkg.mtime = v
-
 	def _get_hash_key(self):
 		hash_key = getattr(self, "_hash_key", None)
 		if hash_key is None:
@@ -1452,6 +1398,135 @@
 			return True
 		return False
 
+class _PackageMetadataWrapper(object):
+	"""
+	Detect metadata updates and synchronize Package attributes.
+	"""
+	_keys = Package.metadata_keys
+	__slots__ = ("__weakref__", "_pkg") + tuple("_val_" + k for k in _keys)
+	_wrapped_keys = frozenset(
+		["COUNTER", "INHERITED", "IUSE", "SLOT", "USE", "_mtime_"])
+
+	def __init__(self, pkg, metadata):
+		self._pkg = pkg
+		self.update(metadata)
+
+	def __iter__(self):
+		for k, v in self.iteritems():
+			yield k
+
+	def __len__(self):
+		l = 0
+		for i in self.iteritems():
+			l += 1
+		return l
+
+	def keys(self):
+		return list(self)
+
+	def iteritems(self):
+		for k in self._keys:
+			try:
+				yield (k, getattr(self, "_val_" + k))
+			except AttributeError:
+				pass
+
+	def items(self):
+		return list(self.iteritems())
+
+	def itervalues(self):
+		for k, v in self.itervalues():
+			yield v
+
+	def values(self):
+		return list(self.itervalues())
+
+	def __delitem__(self, k):
+		try:
+			delattr(self, "_val_" + k)
+		except AttributeError:
+			raise KeyError(k)
+
+	def __setitem__(self, k, v):
+		setattr(self, "_val_" + k, v)
+		if k in self._wrapped_keys:
+			getattr(self, "_set_" + k.lower())(k, v)
+
+	def update(self, d):
+		i = getattr(d, "iteritems", None)
+		if i is None:
+			i = d
+		else:
+			i = i()
+		for k, v in i:
+			self[k] = v
+
+	def __getitem__(self, k):
+		try:
+			return getattr(self, "_val_" + k)
+		except AttributeError:
+			raise KeyError(k)
+
+	def get(self, key, default=None):
+		try:
+			return self[key]
+		except KeyError:
+			return default
+
+	def __contains__(self, k):
+		return hasattr(self, "_val_" + k)
+
+	def pop(self, key, *args):
+		if len(args) > 1:
+			raise TypeError("pop expected at most 2 arguments, got " + \
+				repr(1 + len(args)))
+		try:
+			value = self[key]
+		except KeyError:
+			if args:
+				return args[0]
+			raise
+		del self[key]
+		return value
+
+	def clear(self):
+		for k in self._keys:
+			try:
+				delattr(self, "_val_" + k)
+			except AttributError:
+				pass
+
+	def _set_inherited(self, k, v):
+		if isinstance(v, basestring):
+			v = frozenset(v.split())
+		self._pkg.inherited = v
+
+	def _set_iuse(self, k, v):
+		self._pkg.iuse = self._pkg._iuse(
+			v.split(), self._pkg.root_config.iuse_implicit)
+
+	def _set_slot(self, k, v):
+		self._pkg.slot = v
+
+	def _set_use(self, k, v):
+		self._pkg.use = self._pkg._use(v.split())
+
+	def _set_counter(self, k, v):
+		if isinstance(v, basestring):
+			try:
+				v = int(v.strip())
+			except ValueError:
+				v = 0
+		self._pkg.counter = v
+
+	def _set__mtime_(self, k, v):
+		if isinstance(v, basestring):
+			try:
+				v = float(v.strip())
+			except ValueError:
+				v = 0
+		self._pkg.mtime = v
+
 class DependencyArg(object):
 	def __init__(self, arg=None, root_config=None):
 		self.arg = arg
@@ -1657,15 +1732,14 @@
 		self._root_config = root_config
 		self._vartree = root_config.trees["vartree"]
 		self._portdb = root_config.trees["porttree"].dbapi
-		self._blocker_cache = \
-			BlockerCache(self._vartree.root, self._vartree.dbapi)
+			
 		self._dep_check_trees = { self._vartree.root : {
 			"porttree"    :  self._vartree,
 			"vartree"     :  self._vartree,
 		}}
 
 	def findInstalledBlockers(self, new_pkg, acquire_lock=0):
-		blocker_cache = self._blocker_cache
+		blocker_cache = BlockerCache(self._vartree.root, self._vartree.dbapi)
 		dep_keys = ["DEPEND", "RDEPEND", "PDEPEND"]
 		dep_check_trees = self._dep_check_trees
 		settings = self._vartree.settings
@@ -1714,7 +1788,7 @@
 		blocker_parents = digraph()
 		blocker_atoms = []
 		for pkg in installed_pkgs:
-			for blocker_atom in self._blocker_cache[pkg.cpv].atoms:
+			for blocker_atom in blocker_cache[pkg.cpv].atoms:
 				blocker_atom = blocker_atom[1:]
 				blocker_atoms.append(blocker_atom)
 				blocker_parents.add(blocker_atom, pkg)
@@ -1801,6 +1875,15 @@
 		self._cp_map = {}
 		self._cpv_map = {}
 
+	def clear(self):
+		"""
+		Remove all packages.
+		"""
+		if self._cpv_map:
+			self._clear_cache()
+			self._cp_map.clear()
+			self._cpv_map.clear()
+
 	def copy(self):
 		obj = PackageVirtualDbapi(self.settings)
 		obj._match_cache = self._match_cache.copy()
@@ -1908,7 +1991,14 @@
 
 	_dep_keys = ["DEPEND", "RDEPEND", "PDEPEND"]
 
+	# If dep calculation time exceeds this value then automatically
+	# enable "complete" mode since any performance difference is
+	# not as likely to be noticed by the user after this much time
+	# has passed.
+	_complete_threshold = 20
+
 	def __init__(self, settings, trees, myopts, myparams, spinner):
+		self._creation_time = time.time()
 		self.settings = settings
 		self.target_root = settings["ROOT"]
 		self.myopts = myopts
@@ -3409,16 +3499,20 @@
 		intially satisfied.
 
 		Since this method can consume enough time to disturb users, it is
-		currently only enabled by the --complete-graph option.
+		currently only enabled by the --complete-graph option, or when
+		dep calculation time exceeds self._complete_threshold.
 		"""
-		if "complete" not in self.myparams:
-			# Skip this to avoid consuming enough time to disturb users.
-			return 1
-
 		if "--buildpkgonly" in self.myopts or \
 			"recurse" not in self.myparams:
 			return 1
 
+		if "complete" not in self.myparams:
+			if time.time() - self._creation_time > self._complete_threshold:
+				self.myparams.add("complete")
+			else:
+				# Skip this to avoid consuming enough time to disturb users.
+				return 1
+
 		# Put the depgraph into a mode that causes it to only
 		# select packages that have already been added to the
 		# graph or those that are installed and have not been
@@ -3860,6 +3954,29 @@
 			retlist.reverse()
 		return retlist
 
+	def break_refs(self, mergelist):
+		"""
+		Take a mergelist like that returned from self.altlist() and
+		break any references that lead back to the depgraph. This is
+		useful if you want to hold references to packages without
+		also holding the depgraph on the heap.
+		"""
+		for node in mergelist:
+			if not isinstance(node, Package):
+				continue
+
+			# The visible packages cache has fullfilled it's purpose
+			# and it's no longer needed, so free the memory.
+			node.root_config.visible_pkgs.clear()
+
+			if isinstance(node.root_config.trees["vartree"], FakeVartree):
+				# The FakeVartree references the _package_cache which
+				# references the depgraph. So that Package instances don't
+				# hold the depgraph and FakeVartree on the heap, replace
+				# the FakeVartree reference with the real vartree.
+				node.root_config.trees["vartree"] = \
+					self._trees_orig[node.root]["vartree"]
+
 	def _resolve_conflicts(self):
 		if not self._complete_graph():
 			raise self._unknown_internal_error()
@@ -5249,7 +5366,7 @@
 					refs.sort()
 					ref_string = ", ".join(["'%s'" % name for name in refs])
 					ref_string = " pulled in by " + ref_string
-				msg.append("  %s%s\n" % (colorize("INFORM", arg), ref_string))
+				msg.append("  %s%s\n" % (colorize("INFORM", str(arg)), ref_string))
 			msg.append("\n")
 			if "world" in problems_sets:
 				msg.append("This problem can be solved in one of the following ways:\n\n")
@@ -8829,6 +8946,7 @@
 				mtimedb.filename = None
 				time.sleep(3) # allow the parent to have first fetch
 			mymergelist = mydepgraph.altlist()
+			mydepgraph.break_refs(mymergelist)
 			del mydepgraph
 			clear_caches(trees)
 
@@ -8872,6 +8990,7 @@
 
 			pkglist = mydepgraph.altlist()
 			mydepgraph.saveNomergeFavorites()
+			mydepgraph.break_refs(pkglist)
 			del mydepgraph
 			clear_caches(trees)
 
@@ -9004,6 +9123,7 @@
 		d["porttree"].dbapi._aux_cache.clear()
 		d["bintree"].dbapi._aux_cache.clear()
 		d["bintree"].dbapi._clear_cache()
+	portage.dircache.clear()
 	gc.collect()
 
 def load_emerge_config(trees=None):

Modified: main/branches/prefix/pym/portage/__init__.py
===================================================================
--- main/branches/prefix/pym/portage/__init__.py	2008-06-25 08:42:02 UTC (rev 10780)
+++ main/branches/prefix/pym/portage/__init__.py	2008-06-25 09:00:23 UTC (rev 10781)
@@ -1956,7 +1956,7 @@
 		pkginternaluse = ""
 		iuse = ""
 		if mydb:
-			if isinstance(mydb, dict):
+			if not hasattr(mydb, "aux_get"):
 				slot = mydb["SLOT"]
 				iuse = mydb["IUSE"]
 			else:
@@ -2434,7 +2434,7 @@
 		if len(self.virtuals) == 0:
 			self.getvirtuals()
 		# Grab the virtuals this package provides and add them into the tree virtuals.
-		if isinstance(mydbapi, dict):
+		if not hasattr(mydbapi, "aux_get"):
 			provides = mydbapi["PROVIDE"]
 		else:
 			provides = mydbapi.aux_get(mycpv, ["PROVIDE"])[0]
@@ -2443,7 +2443,7 @@
 		if isinstance(mydbapi, portdbapi):
 			self.setcpv(mycpv, mydb=mydbapi)
 			myuse = self["PORTAGE_USE"]
-		elif isinstance(mydbapi, dict):
+		elif not hasattr(mydbapi, "aux_get"):
 			myuse = mydbapi["USE"]
 		else:
 			myuse = mydbapi.aux_get(mycpv, ["USE"])[0]

Modified: main/branches/prefix/pym/portage/dbapi/vartree.py
===================================================================
--- main/branches/prefix/pym/portage/dbapi/vartree.py	2008-06-25 08:42:02 UTC (rev 10780)
+++ main/branches/prefix/pym/portage/dbapi/vartree.py	2008-06-25 09:00:23 UTC (rev 10781)
@@ -1630,7 +1630,7 @@
 		# name of the dir; the package may have been moved.
 		myebuildpath = None
 		ebuild_phase = "prerm"
-		mystuff = listdir(self.dbdir, EmptyOnError=1)
+		mystuff = os.listdir(self.dbdir)
 		for x in mystuff:
 			if x.endswith(".ebuild"):
 				myebuildpath = os.path.join(self.dbdir, self.pkg + ".ebuild")
@@ -2665,7 +2665,7 @@
 			return a
 
 		# copy "info" files (like SLOT, CFLAGS, etc.) into the database
-		for x in listdir(inforoot):
+		for x in os.listdir(inforoot):
 			self.copyfile(inforoot+"/"+x)
 
 		# write local package counter for recording

Modified: main/branches/prefix/pym/portage/dep.py
===================================================================
--- main/branches/prefix/pym/portage/dep.py	2008-06-25 08:42:02 UTC (rev 10780)
+++ main/branches/prefix/pym/portage/dep.py	2008-06-25 09:00:23 UTC (rev 10781)
@@ -19,6 +19,7 @@
 #
 
 import re, sys, types
+import weakref
 from itertools import chain
 import portage.exception
 from portage.exception import InvalidData, InvalidAtom
@@ -412,7 +413,7 @@
 	"""
 
 	__metaclass__ = _AtomCache
-	_atoms = {}
+	_atoms = weakref.WeakValueDictionary()
 
 	_str_methods = ("endswith", "find", "index", "lstrip", "replace",
 		"startswith", "strip", "rindex", "rfind", "rstrip", "__getitem__",

-- 
gentoo-commits@l.g.o mailing list