[gentoo-commits] portage r10457 - in main/branches/prefix: bin pym/_emerge pym/portage pym/portage/dbapi pym/portage/sets

[Fabian Groffen (grobian)]

Author: grobian
Date: 2008-05-27 15:30:44 +0000 (Tue, 27 May 2008)
New Revision: 10457

Modified:
   main/branches/prefix/bin/ebuild.sh
   main/branches/prefix/pym/_emerge/__init__.py
   main/branches/prefix/pym/portage/__init__.py
   main/branches/prefix/pym/portage/dbapi/vartree.py
   main/branches/prefix/pym/portage/dep.py
   main/branches/prefix/pym/portage/sets/base.py
Log:
   Merged from trunk 10418:10438

   | 10419   | * Add support for Package instances and USE deps in          |
   | zmedico | match_from_list(). * Add USE dep matching support to         |
   |         | depgraph._iter_atoms_for_pkg().                              |
   
   | 10420   | Remove the Package.cpv_slot attribute.                       |
   | zmedico |                                                              |
   
   | 10421   | Use the metadata wrapper to initialize Package slot and use  |
   | zmedico | attributes.                                                  |
   
   | 10422   | Remove unused code.                                          |
   | zmedico |                                                              |
   
   | 10423   | Fix breakage in Package constructor attribute                |
   | zmedico | initialization.                                              |
   
   | 10425   | Fix a broken reference to the Package.cpv_slot attribute.    |
   | zmedico |                                                              |
   
   | 10427   | * Enable config.setcpv() to use a Package instance in place  |
   | zmedico | of a cpv. * Make depgraph._select_package() pass Package     |
   |         | instances into setcpv() calls. * Enable the Package          |
   |         | constructor to use an iterable for metadata items.           |
   
   | 10429   | Skip source_all_bashrcs() when $EBUILD_PHASE is not set.     |
   | zmedico |                                                              |
   
   | 10431   | Avoid triggering "cPickle.UnpicklingError: Global and        |
   | zmedico | instance pickles are not supported." errors that are         |
   |         | triggered when the pickle contains custom classes.           |
   
   | 10433   | Don't silently swallow instances of UnpicklingError since    |
   | zmedico | they are easily triggered by storing instances of custom     |
   |         | classes in a pickle.                                         |
   
   | 10434   | Don't silently swallow instances of UnpicklingError since    |
   | zmedico | they are easily triggered by storing instances of custom     |
   |         | classes in a pickle.                                         |
   
   | 10436   | Add some debug output for arguments and their associated     |
   | zmedico | atoms, to help in debugging problems similar to bug #223735. |
   
   | 10438   | Bug #223685 - Use the finally clause to collect elog         |
   | zmedico | messages just before releasing the build dir lock, so they   |
   |         | never get missed.                                            |


Modified: main/branches/prefix/bin/ebuild.sh
===================================================================
--- main/branches/prefix/bin/ebuild.sh	2008-05-27 10:20:38 UTC (rev 10456)
+++ main/branches/prefix/bin/ebuild.sh	2008-05-27 15:30:44 UTC (rev 10457)
@@ -1356,6 +1356,7 @@
 }
 
 source_all_bashrcs() {
+	[ -n "$EBUILD_PHASE" ] || return
 	local OCC="${CC}" OCXX="${CXX}"
 	# source the existing profile.bashrc's.
 	save_IFS

Modified: main/branches/prefix/pym/_emerge/__init__.py
===================================================================
--- main/branches/prefix/pym/_emerge/__init__.py	2008-05-27 10:20:38 UTC (rev 10456)
+++ main/branches/prefix/pym/_emerge/__init__.py	2008-05-27 15:30:44 UTC (rev 10457)
@@ -1295,7 +1295,8 @@
 	__slots__ = ("built", "cpv", "depth",
 		"installed", "metadata", "onlydeps", "operation",
 		"root", "type_name",
-		"category", "cp", "cpv_slot", "pf", "pv_split", "slot_atom")
+		"category", "cp", "cpv_split",
+		"pf", "pv_split", "slot", "slot_atom", "use")
 
 	metadata_keys = [
 		"CHOST", "COUNTER", "DEPEND", "EAPI", "IUSE", "KEYWORDS",
@@ -1304,12 +1305,39 @@
 
 	def __init__(self, **kwargs):
 		Task.__init__(self, **kwargs)
+		self.metadata = self._metadata_wrapper(self, self.metadata)
 		self.cp = portage.cpv_getkey(self.cpv)
-		self.slot_atom = "%s:%s" % (self.cp, self.metadata["SLOT"])
-		self.cpv_slot = "%s:%s" % (self.cpv, self.metadata["SLOT"])
+		self.slot_atom = portage.dep.Atom("%s:%s" % (self.cp, self.slot))
 		self.category, self.pf = portage.catsplit(self.cpv)
-		self.pv_split = portage.catpkgsplit(self.cpv)[1:]
+		self.cpv_split = portage.catpkgsplit(self.cpv)
+		self.pv_split = self.cpv_split[1:]
 
+	class _use(object):
+		def __init__(self, use):
+			self.enabled = frozenset(use)
+
+	class _metadata_wrapper(dict):
+		"""
+		Detect metadata updates and synchronize Package attributes.
+		"""
+		def __init__(self, pkg, metadata):
+			dict.__init__(self)
+			self._pkg = pkg
+			i = getattr(metadata, "iteritems", None)
+			if i is None:
+				i = metadata
+			else:
+				i = i()
+			for k, v in i:
+				self[k] = v
+
+		def __setitem__(self, k, v):
+			dict.__setitem__(self, k, v)
+			if k == "USE":
+				self._pkg.use = self._pkg._use(v.split())
+			elif k == "SLOT":
+				self._pkg.slot = v
+
 	def _get_hash_key(self):
 		hash_key = getattr(self, "_hash_key", None)
 		if hash_key is None:
@@ -1422,8 +1450,12 @@
 			self._cache_data = mypickle.load()
 			f.close()
 			del f
-		except (IOError, OSError, EOFError, cPickle.UnpicklingError):
-			pass
+		except (IOError, OSError, EOFError, cPickle.UnpicklingError), e:
+			if isinstance(e, cPickle.UnpicklingError):
+				writemsg("!!! Error loading '%s': %s\n" % \
+					(self._cache_filename, str(e)), noiselevel=-1)
+			del e
+
 		cache_valid = self._cache_data and \
 			isinstance(self._cache_data, dict) and \
 			self._cache_data.get("version") == self._cache_version and \
@@ -1451,7 +1483,7 @@
 				if not isinstance(counter, (int, long)):
 					invalid_items.add(k)
 					continue
-				if not isinstance(atoms, list):
+				if not isinstance(atoms, (list, tuple)):
 					invalid_items.add(k)
 					continue
 				invalid_atom = False
@@ -1518,7 +1550,7 @@
 		@type blocker_data: BlockerData
 		"""
 		self._cache_data["blockers"][cpv] = \
-			(blocker_data.counter, blocker_data.atoms)
+			(blocker_data.counter, tuple(str(x) for x in blocker_data.atoms))
 		self._modified = True
 
 	def __iter__(self):
@@ -2466,6 +2498,7 @@
 	def select_files(self, myfiles):
 		"""Given a list of .tbz2s, .ebuilds sets, and deps, create the
 		appropriate depgraph and return a favorite list."""
+		debug = "--debug" in self.myopts
 		root_config = self.roots[self.target_root]
 		sets = root_config.sets
 		getSetAtoms = root_config.setconfig.getSetAtoms
@@ -2692,6 +2725,8 @@
 					if arg not in refs:
 						refs.append(arg)
 		pprovideddict = pkgsettings.pprovideddict
+		if debug:
+			portage.writemsg("\n", noiselevel=-1)
 		# Order needs to be preserved since a feature of --nodeps
 		# is to allow the user to force a specific merge order.
 		args.reverse()
@@ -2713,6 +2748,9 @@
 								"dependencies for %s\n") % arg.arg)
 							return 0, myfavorites
 						continue
+					if debug:
+						portage.writemsg("      Arg: %s\n     Atom: %s\n" % \
+							(arg, atom), noiselevel=-1)
 					pkg, existing_node = self._select_package(
 						myroot, atom, onlydeps=onlydeps)
 					if not pkg:
@@ -3050,34 +3088,24 @@
 					if pkg is None:
 						calculated_use = False
 						try:
-							metadata = dict(izip(self._mydbapi_keys,
-								db.aux_get(cpv, self._mydbapi_keys)))
+							metadata = zip(self._mydbapi_keys,
+								db.aux_get(cpv, self._mydbapi_keys))
 						except KeyError:
 							continue
+						pkg = Package(built=built, cpv=cpv,
+							installed=installed, metadata=metadata,
+							onlydeps=onlydeps, root=root, type_name=pkg_type)
+						metadata = pkg.metadata
 						if not built and ("?" in metadata["LICENSE"] or \
 							"?" in metadata["PROVIDE"]):
 							# This is avoided whenever possible because
 							# it's expensive. It only needs to be done here
 							# if it has an effect on visibility.
-							pkgsettings.setcpv(cpv, mydb=metadata)
+							pkgsettings.setcpv(pkg)
 							metadata["USE"] = pkgsettings["PORTAGE_USE"]
 							calculated_use = True
-						pkg = Package(built=built, cpv=cpv,
-							installed=installed, metadata=metadata,
-							onlydeps=onlydeps, root=root, type_name=pkg_type)
 						self._pkg_cache[pkg] = pkg
-					myarg = None
-					if root == self.target_root:
-						try:
-							myarg = self._iter_atoms_for_pkg(pkg).next()
-						except StopIteration:
-							pass
-						except portage.exception.InvalidDependString:
-							if not installed:
-								# masked by corruption
-								continue
-					if not installed and myarg:
-						found_available_arg = True
+
 					if not installed or (installed and matched_packages):
 						# Only enforce visibility on installed packages
 						# if there is at least one other visible package
@@ -3114,8 +3142,24 @@
 					if not pkg.built and not calculated_use:
 						# This is avoided whenever possible because
 						# it's expensive.
-						pkgsettings.setcpv(cpv, mydb=pkg.metadata)
+						pkgsettings.setcpv(pkg)
 						pkg.metadata["USE"] = pkgsettings["PORTAGE_USE"]
+
+					myarg = None
+					if root == self.target_root:
+						try:
+							# Ebuild USE must have been calculated prior
+							# to this point, in case atoms have USE deps.
+							myarg = self._iter_atoms_for_pkg(pkg).next()
+						except StopIteration:
+							pass
+						except portage.exception.InvalidDependString:
+							if not installed:
+								# masked by corruption
+								continue
+					if not installed and myarg:
+						found_available_arg = True
+
 					if atom.use and not pkg.built:
 						use = pkg.metadata["USE"].split()
 						if atom.use.enabled.difference(use):
@@ -3136,9 +3180,7 @@
 						e_pkg = self._slot_pkg_map[root].get(pkg.slot_atom)
 						if not e_pkg:
 							break
-						cpv_slot = "%s:%s" % \
-							(e_pkg.cpv, e_pkg.metadata["SLOT"])
-						if portage.dep.match_from_list(atom, [cpv_slot]):
+						if portage.dep.match_from_list(atom, [e_pkg]):
 							if highest_version and \
 								e_pkg.cp == atom_cp and \
 								e_pkg < highest_version and \
@@ -3158,14 +3200,11 @@
 						"--reinstall" in self.myopts):
 						iuses = set(filter_iuse_defaults(
 							pkg.metadata["IUSE"].split()))
-						old_use = pkg.metadata["USE"].split()
-						mydb = pkg.metadata
-						if myeb and not usepkgonly:
-							mydb = portdb
+						old_use = pkg.use.enabled
 						if myeb:
-							pkgsettings.setcpv(myeb, mydb=mydb)
+							pkgsettings.setcpv(myeb)
 						else:
-							pkgsettings.setcpv(cpv, mydb=mydb)
+							pkgsettings.setcpv(pkg)
 						now_use = pkgsettings["PORTAGE_USE"].split()
 						forced_flags = set()
 						forced_flags.update(pkgsettings.useforce)
@@ -3173,8 +3212,7 @@
 						cur_iuse = iuses
 						if myeb and not usepkgonly:
 							cur_iuse = set(filter_iuse_defaults(
-								portdb.aux_get(myeb,
-								["IUSE"])[0].split()))
+								myeb.metadata["IUSE"].split()))
 						if self._reinstall_for_flags(forced_flags,
 							old_use, iuses,
 							now_use, cur_iuse):
@@ -3185,7 +3223,7 @@
 						("--newuse" in self.myopts or \
 						"--reinstall" in self.myopts) and \
 						cpv in vardb.match(atom):
-						pkgsettings.setcpv(cpv, mydb=pkg.metadata)
+						pkgsettings.setcpv(pkg)
 						forced_flags = set()
 						forced_flags.update(pkgsettings.useforce)
 						forced_flags.update(pkgsettings.usemask)
@@ -3202,7 +3240,7 @@
 						if reinstall_for_flags:
 							reinstall = True
 					if not built:
-						myeb = cpv
+						myeb = pkg
 					matched_packages.append(pkg)
 					if reinstall_for_flags:
 						self._reinstall_nodes[pkg] = \
@@ -3214,7 +3252,8 @@
 
 		if "--debug" in self.myopts:
 			for pkg in matched_packages:
-				print (pkg.type_name + ":").rjust(10), pkg.cpv
+				portage.writemsg("%s %s\n" % \
+					((pkg.type_name + ":").rjust(10), pkg.cpv), noiselevel=-1)
 
 		# Filter out any old-style virtual matches if they are
 		# mixed with new-style virtual matches.
@@ -5195,23 +5234,23 @@
 				continue
 			mydb = trees[myroot][self.pkg_tree_map[pkg_type]].dbapi
 			try:
-				metadata = dict(izip(self._mydbapi_keys,
-					mydb.aux_get(pkg_key, self._mydbapi_keys)))
+				metadata = zip(self._mydbapi_keys,
+					mydb.aux_get(pkg_key, self._mydbapi_keys))
 			except KeyError:
 				# It does no exist or it is corrupt.
 				if action == "uninstall":
 					continue
 				raise portage.exception.PackageNotFound(pkg_key)
-			if pkg_type == "ebuild":
-				pkgsettings = self.pkgsettings[myroot]
-				pkgsettings.setcpv(pkg_key, mydb=metadata)
-				metadata["USE"] = pkgsettings["PORTAGE_USE"]
 			installed = action == "uninstall"
 			built = pkg_type != "ebuild"
 			pkg = Package(built=built, cpv=pkg_key,
 				installed=installed, metadata=metadata,
 				operation=action, root=myroot,
 				type_name=pkg_type)
+			if pkg_type == "ebuild":
+				pkgsettings = self.pkgsettings[myroot]
+				pkgsettings.setcpv(pkg)
+				pkg.metadata["USE"] = pkgsettings["PORTAGE_USE"]
 			self._pkg_cache[pkg] = pkg
 
 			root_config = self.roots[pkg.root]
@@ -5957,9 +5996,6 @@
 							pkgsettings, self.edebug, mydbapi=portdb,
 							tree="porttree")
 						del pkgsettings["PORTAGE_BINPKG_TMPFILE"]
-						if retval != os.EX_OK or \
-							"--buildpkgonly" in self.myopts:
-							elog_process(pkg_key, pkgsettings, phasefilter=filter_mergephases)
 						if retval != os.EX_OK:
 							return retval
 						bintree = self.trees[myroot]["bintree"]
@@ -6011,6 +6047,8 @@
 							return retval
 				finally:
 					if builddir_lock:
+						elog_process(pkg.cpv, pkgsettings,
+							phasefilter=filter_mergephases)
 						portage.locks.unlockdir(builddir_lock)
 					try:
 						if not catdir_lock:
@@ -8534,7 +8572,17 @@
 			# XXX: Stored as a list for backward compatibility.
 			mtimedb["resume"]["myopts"] = \
 				[k for k in myopts if myopts[k] is True]
-			mtimedb["resume"]["favorites"]=favorites
+
+			# Convert Atom instances to plain str since the mtimedb loader
+			# sets unpickler.find_global = None which causes unpickler.load()
+			# to raise the following exception:
+			#
+			# cPickle.UnpicklingError: Global and instance pickles are not supported.
+			#
+			# TODO: Maybe stop setting find_global = None, or find some other
+			# way to avoid accidental triggering of the above UnpicklingError.
+			mtimedb["resume"]["favorites"] = [str(x) for x in favorites]
+
 			if ("--digest" in myopts) and not ("--fetchonly" in myopts or "--fetch-all-uri" in myopts):
 				for pkgline in mydepgraph.altlist():
 					if pkgline[0]=="ebuild" and pkgline[3]=="merge":

Modified: main/branches/prefix/pym/portage/__init__.py
===================================================================
--- main/branches/prefix/pym/portage/__init__.py	2008-05-27 10:20:38 UTC (rev 10456)
+++ main/branches/prefix/pym/portage/__init__.py	2008-05-27 15:30:44 UTC (rev 10457)
@@ -1924,6 +1924,13 @@
 		"""
 
 		self.modifying()
+
+		pkg = None
+		if not isinstance(mycpv, basestring):
+			pkg = mycpv
+			mycpv = pkg.cpv
+			mydb = pkg.metadata
+
 		if self.mycpv == mycpv:
 			return
 		ebuild_phase = self.get("EBUILD_PHASE")
@@ -1939,7 +1946,10 @@
 				iuse = mydb["IUSE"]
 			else:
 				slot, iuse = mydb.aux_get(self.mycpv, ["SLOT", "IUSE"])
-			cpv_slot = "%s:%s" % (self.mycpv, slot)
+			if pkg is None:
+				cpv_slot = "%s:%s" % (self.mycpv, slot)
+			else:
+				cpv_slot = pkg
 			pkginternaluse = []
 			for x in iuse.split():
 				if x.startswith("+"):
@@ -5696,7 +5706,7 @@
 						# Check if the atom would result in a direct circular
 						# dependency and try to avoid that if it seems likely
 						# to be unresolvable.
-						cpv_slot_list = [parent.cpv_slot]
+						cpv_slot_list = [parent]
 						circular_atom = None
 						for atom in atoms:
 							if "!" == atom[:1]:
@@ -6625,7 +6635,11 @@
 			d = mypickle.load()
 			f.close()
 			del f
-		except (IOError, OSError, EOFError, cPickle.UnpicklingError):
+		except (IOError, OSError, EOFError, cPickle.UnpicklingError), e:
+			if isinstance(e, cPickle.UnpicklingError):
+				writemsg("!!! Error loading '%s': %s\n" % \
+					(filename, str(e)), noiselevel=-1)
+			del e
 			d = {}
 
 		if "old" in d:

Modified: main/branches/prefix/pym/portage/dbapi/vartree.py
===================================================================
--- main/branches/prefix/pym/portage/dbapi/vartree.py	2008-05-27 10:20:38 UTC (rev 10456)
+++ main/branches/prefix/pym/portage/dbapi/vartree.py	2008-05-27 15:30:44 UTC (rev 10457)
@@ -705,8 +705,11 @@
 				self._aux_cache = mypickle.load()
 				f.close()
 				del f
-			except (IOError, OSError, EOFError, cPickle.UnpicklingError):
-				pass
+			except (IOError, OSError, EOFError, cPickle.UnpicklingError), e:
+				if isinstance(e, cPickle.UnpicklingError):
+					writemsg("!!! Error loading '%s': %s\n" % \
+						(self._aux_cache_filename, str(e)), noiselevel=-1)
+				del e
 			if not self._aux_cache or \
 				not isinstance(self._aux_cache, dict) or \
 				self._aux_cache.get("version") != self._aux_cache_version or \

Modified: main/branches/prefix/pym/portage/dep.py
===================================================================
--- main/branches/prefix/pym/portage/dep.py	2008-05-27 10:20:38 UTC (rev 10456)
+++ main/branches/prefix/pym/portage/dep.py	2008-05-27 15:30:44 UTC (rev 10457)
@@ -759,6 +759,9 @@
 	@return: A list of package atoms that match the given package atom
 	"""
 
+	if not candidate_list:
+		return []
+
 	from portage.util import writemsg
 	if "!" == mydep[:1]:
 		mydep = mydep[1:]
@@ -791,13 +794,21 @@
 
 	if operator is None:
 		for x in candidate_list:
-			if dep_getkey(x) != mycpv:
+			cp = getattr(x, "cp", None)
+			if cp is None:
+				cp = dep_getkey(x)
+			if cp != mycpv:
 				continue
 			mylist.append(x)
 
 	elif operator == "=": # Exact match
-		mylist = [cpv for cpv in candidate_list if \
-			cpvequal(remove_slot(cpv), mycpv)]
+		for x in candidate_list:
+			xcpv = getattr(x, "cpv", None)
+			if xcpv is None:
+				xcpv = dep_getcpv(x)
+			if not cpvequal(xcpv, mycpv):
+				continue
+			mylist.append(x)
 
 	elif operator == "=*": # glob match
 		# XXX: Nasty special casing for leading zeros
@@ -809,7 +820,9 @@
 			myver = "0"+myver
 		mycpv = mysplit[0]+"/"+mysplit[1]+"-"+myver
 		for x in candidate_list:
-			xs = catpkgsplit(remove_slot(x))
+			xs = getattr(x, "cpv_split", None)
+			if xs is None:
+				xs = catpkgsplit(remove_slot(x))
 			myver = xs[2].lstrip("0")
 			if not myver or not myver[0].isdigit():
 				myver = "0"+myver
@@ -819,8 +832,10 @@
 
 	elif operator == "~": # version, any revision, match
 		for x in candidate_list:
-			xs = catpkgsplit(remove_slot(x))
+			xs = getattr(x, "cpv_split", None)
 			if xs is None:
+				xs = catpkgsplit(remove_slot(x))
+			if xs is None:
 				raise InvalidData(x)
 			if not cpvequal(xs[0]+"/"+xs[1]+"-"+xs[2], mycpv_cps[0]+"/"+mycpv_cps[1]+"-"+mycpv_cps[2]):
 				continue
@@ -831,8 +846,13 @@
 	elif operator in [">", ">=", "<", "<="]:
 		mysplit = ["%s/%s" % (cat, pkg), ver, rev]
 		for x in candidate_list:
+			xs = getattr(x, "cpv_split", None)
+			if xs is None:
+				xs = catpkgsplit(remove_slot(x))
+			xcat, xpkg, xver, xrev = xs
+			xs = ["%s/%s" % (xcat, xpkg), xver, xrev]
 			try:
-				result = pkgcmp(pkgsplit(remove_slot(x)), mysplit)
+				result = pkgcmp(xs, mysplit)
 			except ValueError: # pkgcmp may return ValueError during int() conversion
 				writemsg("\nInvalid package name: %s\n" % x, noiselevel=-1)
 				raise
@@ -859,9 +879,26 @@
 		candidate_list = mylist
 		mylist = []
 		for x in candidate_list:
-			xslot = dep_getslot(x)
+			xslot = getattr(x, "slot", None)
+			if xslot is None and isinstance(x, basestring):
+				xslot = dep_getslot(x)
 			if xslot is not None and xslot != slot:
 				continue
 			mylist.append(x)
 
+	if mydep.use:
+		candidate_list = mylist
+		mylist = []
+		for x in candidate_list:
+			# Note: IUSE intersection is neglected here since there
+			# is currently no way to access implicit IUSE. However, IUSE
+			# filtering can be added elsewhere in the chain.
+			use = getattr(x, "use", None)
+			if use is not None:
+				if mydep.use.enabled.difference(use.enabled):
+					continue
+				if mydep.use.disabled.intersection(use.enabled):
+					continue
+			mylist.append(x)
+
 	return mylist

Modified: main/branches/prefix/pym/portage/sets/base.py
===================================================================
--- main/branches/prefix/pym/portage/sets/base.py	2008-05-27 10:20:38 UTC (rev 10456)
+++ main/branches/prefix/pym/portage/sets/base.py	2008-05-27 15:30:44 UTC (rev 10457)
@@ -108,7 +108,7 @@
 		atoms = list(self.iterAtomsForPackage(pkg))
 		if not atoms:
 			return None
-		return best_match_to_list(pkg.cpv_slot, atoms)
+		return best_match_to_list(pkg, atoms)
 
 	def iterAtomsForPackage(self, pkg):
 		"""
@@ -116,7 +116,7 @@
 		arguments against the PROVIDE metadata.  This will raise an
 		InvalidDependString exception if PROVIDE is invalid.
 		"""
-		cpv_slot_list = ["%s:%s" % (pkg.cpv, pkg.metadata["SLOT"])]
+		cpv_slot_list = [pkg]
 		cp = cpv_getkey(pkg.cpv)
 		self._load() # make sure the atoms are loaded
 		atoms = self._atommap.get(cp)

-- 
gentoo-commits@l.g.o mailing list