[gentoo-commits] portage r10815 - in main/branches/prefix: bin cnf pym/_emerge pym/portage pym/portage/dbapi

[Fabian Groffen (grobian)]

Author: grobian
Date: 2008-06-27 14:53:20 +0000 (Fri, 27 Jun 2008)
New Revision: 10815

Modified:
   main/branches/prefix/bin/portageq
   main/branches/prefix/bin/repoman
   main/branches/prefix/cnf/sets.conf
   main/branches/prefix/pym/_emerge/__init__.py
   main/branches/prefix/pym/portage/__init__.py
   main/branches/prefix/pym/portage/dbapi/__init__.py
   main/branches/prefix/pym/portage/dbapi/bintree.py
   main/branches/prefix/pym/portage/output.py
   main/branches/prefix/pym/portage/util.py
Log:
   Merged from trunk 10801:10812

   | 10802   | Handle KeyError from aux_get() inside                        |
   | zmedico | dbapi._iter_match_slot() and _iter_match_use(). Thanks to    |
   |         | grobian for reporting.                                       |
   
   | 10803   | Use settings["ROOT"] at the beginning of env_update() to     |
   | zmedico | avoid breakage due to "root" being an ObjectProxy instance.  |
   
   | 10804   | * Remove PORTAGE_LEGACY_GLOBALS hack for portage import      |
   | zmedico | since late initialization of portage.settings (via           |
   |         | ObjectProxy) allows us to rely on being able to access the   |
   |         | portage.exception namespace before portage.settings is       |
   |         | initialized. * Use portage.settings["ROOT"] instead of       |
   |         | portage.root to avoid potential ObjectProxy compatibility    |
   |         | issues.                                                      |
   
   | 10805   | Fix the PermissionDenied message for color.map.              |
   | zmedico |                                                              |
   
   | 10806   | Add "PATH" to the set of allowed pacakge metadata keys for   |
   | zmedico | the PackageIndex.                                            |
   
   | 10807   | Call use_reduce() with matchall=1 when checking depstrings.  |
   | zmedico | Otherwise it's possible for invalid deps to be ignored.      |
   |         | Thanks to grobian for reporting.                             |
   
   | 10808   | For brevity, and consistency with pkgcore, rename            |
   | zmedico | "all-installed" to just "installed". Thanks to ferringb for  |
   |         | the suggestion.                                              |
   
   | 10809   | Split the --skipfirst code into a resume_depgraph() function |
   | zmedico | that will be useful for implementing a --keep-going option   |
   |         | (bug #12768).                                                |
   
   | 10810   | Use bool() to simplify ObjectProxy.__nonzero__().            |
   | zmedico |                                                              |
   
   | 10811   | Bug #12768 - Add a --keep-going option. This uses the same   |
   | zmedico | resume_depgraph() function that's called for emerge --resume |
   |         | operations. Given the remaining tasks, depedencies are       |
   |         | recalculated and any tasks with unsatisfied dependencies are |
   |         | automatically dropped.                                       |
   
   | 10812   | Call clear_caches() after the depgraph is out of scope, in   |
   | zmedico | MergeTask.merge().                                           |


Modified: main/branches/prefix/bin/portageq
===================================================================
--- main/branches/prefix/bin/portageq	2008-06-27 14:47:19 UTC (rev 10814)
+++ main/branches/prefix/bin/portageq	2008-06-27 14:53:20 UTC (rev 10815)
@@ -462,25 +462,19 @@
 
 	global portage
 
-	# First import the main portage module without legacy globals since it
-	# is almost certain to succeed in that case. This provides access to
-	# the portage.exception namespace which is needed for later exception
-	# handling, like if portage.exception.PermissionDenied is raised when
-	# constructing the legacy global config instance.
-	os.environ["PORTAGE_LEGACY_GLOBALS"] = "false"
-
-	# for an explanation on this logic, see pym/_emerge/__init__.py
-	if os.environ.__contains__("PORTAGE_PYTHONPATH"):
-		sys.path.insert(0, os.environ["PORTAGE_PYTHONPATH"])
-	else:
-		sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(os.path.realpath(__file__))), "pym"))
-	import portage
-
-	del os.environ["PORTAGE_LEGACY_GLOBALS"]
 	try:
-		reload(portage)
+		try:
+			import portage
+		except ImportError:
+			from os import path as osp
+			# for an explanation on this logic, see pym/_emerge/__init__.py
+			if os.environ.__contains__("PORTAGE_PYTHONPATH"):
+				sys.path.insert(0, os.environ["PORTAGE_PYTHONPATH"])
+			else:
+				sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(os.path.realpath(__file__))), "pym"))
+			import portage
 		if uses_root:
-			sys.argv[2] = portage.root
+			sys.argv[2] = portage.settings["ROOT"]
 		retval = function(sys.argv[2:])
 		if retval:
 			sys.exit(retval)

Modified: main/branches/prefix/bin/repoman
===================================================================
--- main/branches/prefix/bin/repoman	2008-06-27 14:47:19 UTC (rev 10814)
+++ main/branches/prefix/bin/repoman	2008-06-27 14:53:20 UTC (rev 10815)
@@ -1247,7 +1247,7 @@
 				mydeplist = []
 
 			try:
-				portage.dep.use_reduce(mydeplist, excludeall=myiuse)
+				portage.dep.use_reduce(mydeplist, matchall=1)
 			except portage.exception.InvalidDependString, e:
 				badsyntax.append(str(e))
 

Modified: main/branches/prefix/cnf/sets.conf
===================================================================
--- main/branches/prefix/cnf/sets.conf	2008-06-27 14:47:19 UTC (rev 10814)
+++ main/branches/prefix/cnf/sets.conf	2008-06-27 14:53:20 UTC (rev 10815)
@@ -25,7 +25,7 @@
 world-candidate = False
 
 # Again, not much to change here, though people might prefer a different name
-[all-installed]
+[installed]
 class = portage.sets.dbapi.EverythingSet
 world-candidate = False
 

Modified: main/branches/prefix/pym/_emerge/__init__.py
===================================================================
--- main/branches/prefix/pym/_emerge/__init__.py	2008-06-27 14:47:19 UTC (rev 10814)
+++ main/branches/prefix/pym/_emerge/__init__.py	2008-06-27 14:53:20 UTC (rev 10815)
@@ -212,6 +212,7 @@
 "--fetchonly",    "--fetch-all-uri",
 "--getbinpkg",    "--getbinpkgonly",
 "--help",         "--ignore-default-opts",
+"--keep-going",
 "--noconfmem",
 "--newuse",       "--nocolor",
 "--nodeps",       "--noreplace",
@@ -5428,7 +5429,7 @@
 			world_set.update(all_added)
 		world_set.unlock()
 
-	def loadResumeCommand(self, resume_data):
+	def loadResumeCommand(self, resume_data, skip_masked=False):
 		"""
 		Add a resume command to the graph and validate it in the process.  This
 		will raise a PackageNotFound exception if a package is not available.
@@ -5441,7 +5442,6 @@
 		if not isinstance(mergelist, list):
 			mergelist = []
 
-		skipfirst = "--skipfirst" in self.myopts
 		fakedb = self.mydbapi
 		trees = self.trees
 		serialized_tasks = []
@@ -5479,7 +5479,7 @@
 			root_config = self.roots[pkg.root]
 			if "merge" == pkg.operation and \
 				not visible(root_config.settings, pkg):
-				if skipfirst:
+				if skip_masked:
 					masked_tasks.append(Dependency(root=pkg.root, parent=pkg))
 				else:
 					self._unsatisfied_deps_for_display.append(
@@ -5533,7 +5533,7 @@
 				# was dropped via --skipfirst. It makes the
 				# resume list invalid, so convert it to a
 				# UnsatisfiedResumeDep exception.
-				raise self.UnsatisfiedResumeDep(
+				raise self.UnsatisfiedResumeDep(self,
 					masked_tasks + self._unsatisfied_deps)
 			self._serialized_tasks_cache = None
 			try:
@@ -5610,6 +5610,9 @@
 		can occur when a required package is dropped from the
 		merge list via --skipfirst.
 		"""
+		def __init__(self, depgraph, value):
+			portage.exception.PortageException.__init__(self, value)
+			self.depgraph = depgraph
 
 	class _internal_exception(portage.exception.PortageException):
 		def __init__(self, value=""):
@@ -5910,11 +5913,16 @@
 		"--fetchonly", "--fetch-all-uri",
 		"--nodeps", "--pretend"])
 
-	def __init__(self, settings, trees, myopts):
+	def __init__(self, settings, trees, mtimedb, myopts,
+		spinner, mergelist, favorites):
 		self.settings = settings
 		self.target_root = settings["ROOT"]
 		self.trees = trees
 		self.myopts = myopts
+		self._spinner = spinner
+		self._mtimedb = mtimedb
+		self._mergelist = mergelist
+		self._favorites = favorites
 		self.edebug = 0
 		if settings.get("PORTAGE_DEBUG", "") == "1":
 			self.edebug = 1
@@ -5963,15 +5971,85 @@
 
 		return blocker_dblinks
 
-	def merge(self, mylist, favorites, mtimedb):
-		try:
-			return self._merge(mylist, favorites, mtimedb)
-		finally:
-			if self._spawned_pids:
-				from portage import process
-				process.spawned_pids.extend(self._spawned_pids)
-				self._spawned_pids = []
+	def merge(self):
 
+		keep_going = "--keep-going" in self.myopts
+
+		while True:
+			try:
+				rval = self._merge()
+			finally:
+				spawned_pids = self._spawned_pids
+				while spawned_pids:
+					pid = spawned_pids.pop()
+					try:
+						if os.waitpid(pid, os.WNOHANG) == (0, 0):
+							os.kill(pid, signal.SIGTERM)
+							os.waitpid(pid, 0)
+					except OSError:
+						pass # cleaned up elsewhere.
+
+			if rval == os.EX_OK or not keep_going:
+				break
+			mtimedb = self._mtimedb
+			if "resume" not in mtimedb:
+				break
+			mergelist = self._mtimedb["resume"].get("mergelist")
+			if not mergelist:
+				break
+			if mergelist[0][-1] != "merge":
+				break
+			# Skip the first one because it failed to build or install.
+			del mergelist[0]
+			if not mergelist:
+				break
+			mylist = self._calc_resume_list()
+			clear_caches(self.trees)
+			if not mylist:
+				break
+			self.curval += 1
+			self._mergelist = mylist
+
+		return rval
+
+	def _calc_resume_list(self):
+		"""
+		Use the current resume list to calculate a new one,
+		dropping any packages with unsatisfied deps.
+		"""
+		print colorize("GOOD", "*** Resuming merge...")
+
+		show_spinner = "--quiet" not in self.myopts and \
+			"--nodeps" not in self.myopts
+
+		if show_spinner:
+			print "Calculating dependencies  ",
+
+		myparams = create_depgraph_params(self.myopts, None)
+		success, mydepgraph, dropped_tasks = resume_depgraph(
+			self.settings, self.trees, self._mtimedb, self.myopts,
+			myparams, self._spinner, skip_unsatisfied=True)
+
+		if show_spinner:
+			print "\b\b... done!"
+
+		if not success:
+			mydepgraph.display_problems()
+			return None
+
+		if dropped_tasks:
+			portage.writemsg("!!! One or more packages have been " + \
+				"dropped due to\n" + \
+				"!!! masking or unsatisfied dependencies:\n\n",
+				noiselevel=-1)
+			for task in dropped_tasks:
+				portage.writemsg("  " + str(task) + "\n", noiselevel=-1)
+			portage.writemsg("\n", noiselevel=-1)
+
+		mylist = mydepgraph.altlist()
+		mydepgraph.break_refs(mylist)
+		return mylist
+
 	def _poll_child_processes(self):
 		"""
 		After each merge, collect status from child processes
@@ -5991,7 +6069,10 @@
 				pass
 			spawned_pids.remove(pid)
 
-	def _merge(self, mylist, favorites, mtimedb):
+	def _merge(self):
+		mylist = self._mergelist
+		favorites = self._favorites
+		mtimedb = self._mtimedb
 		from portage.elog import elog_process
 		from portage.elog.filtering import filter_mergephases
 		buildpkgonly = "--buildpkgonly" in self.myopts
@@ -8483,6 +8564,73 @@
 	else:
 		print "Number removed:       "+str(len(cleanlist))
 
+def resume_depgraph(settings, trees, mtimedb, myopts, myparams, spinner,
+	skip_masked=False, skip_unsatisfied=False):
+	"""
+	Construct a depgraph for the given resume list. This will raise
+	PackageNotFound or depgraph.UnsatisfiedResumeDep when necessary.
+	@rtype: tuple
+	@returns: (success, depgraph, dropped_tasks)
+	"""
+	mergelist = mtimedb["resume"]["mergelist"]
+	dropped_tasks = set()
+	while True:
+		mydepgraph = depgraph(settings, trees,
+			myopts, myparams, spinner)
+		try:
+			success = mydepgraph.loadResumeCommand(mtimedb["resume"],
+				skip_masked=skip_masked)
+		except depgraph.UnsatisfiedResumeDep, e:
+			if not skip_unsatisfied:
+				raise
+
+			graph = mydepgraph.digraph
+			unsatisfied_parents = dict((dep.parent, dep.parent) \
+				for dep in e.value)
+			traversed_nodes = set()
+			unsatisfied_stack = list(unsatisfied_parents)
+			while unsatisfied_stack:
+				pkg = unsatisfied_stack.pop()
+				if pkg in traversed_nodes:
+					continue
+				traversed_nodes.add(pkg)
+
+				# If this package was pulled in by a parent
+				# package scheduled for merge, removing this
+				# package may cause the the parent package's
+				# dependency to become unsatisfied.
+				for parent_node in graph.parent_nodes(pkg):
+					if not isinstance(parent_node, Package) \
+						or parent_node.operation != "merge":
+						continue
+					unsatisfied = \
+						graph.child_nodes(parent_node,
+						ignore_priority=DepPriority.SOFT)
+					if pkg in unsatisfied:
+						unsatisfied_parents[parent_node] = parent_node
+						unsatisfied_stack.append(parent_node)
+
+			pruned_mergelist = [x for x in mergelist \
+				if isinstance(x, list) and \
+				tuple(x) not in unsatisfied_parents]
+
+			# It shouldn't happen, but if the size of mergelist
+			# does not decrease for some reason then the loop
+			# will be infinite. Therefore, if that case ever
+			# occurs for some reason, raise the exception to
+			# break out of the loop.
+			if not pruned_mergelist or \
+				len(pruned_mergelist) == len(mergelist):
+				raise
+			mergelist[:] = pruned_mergelist
+			dropped_tasks.update(unsatisfied_parents)
+			del e, graph, traversed_nodes, \
+				unsatisfied_parents, unsatisfied_stack
+			continue
+		else:
+			break
+	return (success, mydepgraph, dropped_tasks)
+
 def action_build(settings, trees, mtimedb,
 	myopts, myaction, myfiles, spinner):
 
@@ -8606,66 +8754,18 @@
 					del mergelist[i]
 					break
 
-		dropped_tasks = set()
-
+		skip_masked      = "--skipfirst" in myopts
+		skip_unsatisfied = "--skipfirst" in myopts
 		success = False
+		mydepgraph = None
 		try:
-			while True:
-				mydepgraph = depgraph(settings, trees,
-					myopts, myparams, spinner)
-				try:
-					success = mydepgraph.loadResumeCommand(mtimedb["resume"])
-				except depgraph.UnsatisfiedResumeDep, e:
-					if "--skipfirst" not in myopts:
-						raise
-
-					graph = mydepgraph.digraph
-					unsatisfied_parents = dict((dep.parent, dep.parent) \
-						for dep in e.value)
-					traversed_nodes = set()
-					unsatisfied_stack = list(unsatisfied_parents)
-					while unsatisfied_stack:
-						pkg = unsatisfied_stack.pop()
-						if pkg in traversed_nodes:
-							continue
-						traversed_nodes.add(pkg)
-
-						# If this package was pulled in by a parent
-						# package scheduled for merge, removing this
-						# package may cause the the parent package's
-						# dependency to become unsatisfied.
-						for parent_node in graph.parent_nodes(pkg):
-							if not isinstance(parent_node, Package) \
-								or parent_node.operation != "merge":
-								continue
-							unsatisfied = \
-								graph.child_nodes(parent_node,
-								ignore_priority=DepPriority.SOFT)
-							if pkg in unsatisfied:
-								unsatisfied_parents[parent_node] = parent_node
-								unsatisfied_stack.append(parent_node)
-
-					pruned_mergelist = [x for x in mergelist \
-						if isinstance(x, list) and \
-						tuple(x) not in unsatisfied_parents]
-
-					# It shouldn't happen, but if the size of mergelist
-					# does not decrease for some reason then the loop
-					# will be infinite. Therefore, if that case ever
-					# occurs for some reason, raise the exception to
-					# break out of the loop.
-					if not pruned_mergelist or \
-						len(pruned_mergelist) == len(mergelist):
-						raise
-					mergelist[:] = pruned_mergelist
-					dropped_tasks.update(unsatisfied_parents)
-					del e, graph, traversed_nodes, \
-						unsatisfied_parents, unsatisfied_stack
-					continue
-				else:
-					break
+			success, mydepgraph, dropped_tasks = resume_depgraph(
+				settings, trees, mtimedb, myopts, myparams, spinner,
+				skip_masked=skip_masked, skip_unsatisfied=skip_unsatisfied)
 		except (portage.exception.PackageNotFound,
 			depgraph.UnsatisfiedResumeDep), e:
+			if isinstance(e, depgraph.UnsatisfiedResumeDep):
+				mydepgraph = e.depgraph
 			if show_spinner:
 				print
 			from textwrap import wrap
@@ -8734,7 +8834,8 @@
 				portage.writemsg("\n", noiselevel=-1)
 			del dropped_tasks
 		else:
-			mydepgraph.display_problems()
+			if mydepgraph is not None:
+				mydepgraph.display_problems()
 			if not (ask or pretend):
 				# delete the current list and also the backup
 				# since it's probably stale too.
@@ -8873,7 +8974,6 @@
 
 		if ("--resume" in myopts):
 			favorites=mtimedb["resume"]["favorites"]
-			mergetask = MergeTask(settings, trees, myopts)
 			if "PORTAGE_PARALLEL_FETCHONLY" in settings:
 				""" parallel-fetch uses --resume --fetchonly and we don't want
 				it to write the mtimedb"""
@@ -8884,7 +8984,9 @@
 			del mydepgraph
 			clear_caches(trees)
 
-			retval = mergetask.merge(mymergelist, favorites, mtimedb)
+			mergetask = MergeTask(settings, trees, mtimedb, myopts,
+				spinner, mymergelist, favorites)
+			retval = mergetask.merge()
 			merge_count = mergetask.curval
 		else:
 			if "resume" in mtimedb and \
@@ -8928,8 +9030,9 @@
 			del mydepgraph
 			clear_caches(trees)
 
-			mergetask = MergeTask(settings, trees, myopts)
-			retval = mergetask.merge(pkglist, favorites, mtimedb)
+			mergetask = MergeTask(settings, trees, mtimedb, myopts,
+				spinner, pkglist, favorites)
+			retval = mergetask.merge()
 			merge_count = mergetask.curval
 
 		if retval == os.EX_OK and not (buildpkgonly or fetchonly or pretend):

Modified: main/branches/prefix/pym/portage/__init__.py
===================================================================
--- main/branches/prefix/pym/portage/__init__.py	2008-06-27 14:47:19 UTC (rev 10814)
+++ main/branches/prefix/pym/portage/__init__.py	2008-06-27 14:53:20 UTC (rev 10815)
@@ -542,8 +542,8 @@
 def env_update(makelinks=1, target_root=None, prev_mtimes=None, contents=None,
 	env=None):
 	if target_root is None:
-		global root
-		target_root = root
+		global settings
+		target_root = settings["ROOT"]
 	if prev_mtimes is None:
 		global mtimedb
 		prev_mtimes = mtimedb["ldpath"]

Modified: main/branches/prefix/pym/portage/dbapi/__init__.py
===================================================================
--- main/branches/prefix/pym/portage/dbapi/__init__.py	2008-06-27 14:47:19 UTC (rev 10814)
+++ main/branches/prefix/pym/portage/dbapi/__init__.py	2008-06-27 14:53:20 UTC (rev 10815)
@@ -135,8 +135,11 @@
 
 	def _iter_match_slot(self, atom, cpv_iter):
 		for cpv in cpv_iter:
-			if self.aux_get(cpv, ["SLOT"])[0] == atom.slot:
-				yield cpv
+			try:
+				if self.aux_get(cpv, ["SLOT"])[0] == atom.slot:
+					yield cpv
+			except KeyError:
+				continue
 
 	def _iter_match_use(self, atom, cpv_iter):
 		"""
@@ -146,7 +149,10 @@
 		if self._iuse_implicit is None:
 			self._iuse_implicit = self.settings._get_implicit_iuse()
 		for cpv in cpv_iter:
-			iuse, use = self.aux_get(cpv, ["IUSE", "USE"])
+			try:
+				iuse, use = self.aux_get(cpv, ["IUSE", "USE"])
+			except KeyError:
+				continue
 			use = use.split()
 			iuse = self._iuse_implicit.union(
 				re.escape(x.lstrip("+-")) for x in iuse.split())

Modified: main/branches/prefix/pym/portage/dbapi/bintree.py
===================================================================
--- main/branches/prefix/pym/portage/dbapi/bintree.py	2008-06-27 14:47:19 UTC (rev 10814)
+++ main/branches/prefix/pym/portage/dbapi/bintree.py	2008-06-27 14:53:20 UTC (rev 10815)
@@ -173,6 +173,7 @@
 				"IUSE"    : "",
 				"KEYWORDS": "",
 				"LICENSE" : "",
+				"PATH"    : "",
 				"PDEPEND" : "",
 				"PROVIDE" : "",
 				"RDEPEND" : "",

Modified: main/branches/prefix/pym/portage/output.py
===================================================================
--- main/branches/prefix/pym/portage/output.py	2008-06-27 14:47:19 UTC (rev 10814)
+++ main/branches/prefix/pym/portage/output.py	2008-06-27 14:53:20 UTC (rev 10815)
@@ -231,8 +231,11 @@
 	parse_color_map(onerror=lambda e: writemsg("%s\n" % str(e), noiselevel=-1))
 except FileNotFound:
 	pass
+except PermissionDenied, e:
+	writemsg("Permission denied: '%s'\n" % str(e), noiselevel=-1)
+	del e
 except PortageException, e:
-	writemsg("%s\n" % str(e))
+	writemsg("%s\n" % str(e), noiselevel=-1)
 	del e
 
 def nc_len(mystr):

Modified: main/branches/prefix/pym/portage/util.py
===================================================================
--- main/branches/prefix/pym/portage/util.py	2008-06-27 14:47:19 UTC (rev 10814)
+++ main/branches/prefix/pym/portage/util.py	2008-06-27 14:53:20 UTC (rev 10815)
@@ -970,9 +970,7 @@
 		return object.__getattribute__(self, '_get_target')() != other
 
 	def __nonzero__(self):
-		if object.__getattribute__(self, '_get_target')():
-			return True
-		return False
+		return bool(object.__getattribute__(self, '_get_target')())
 
 class LazyItemsDict(dict):
 	"""A mapping object that behaves like a standard dict except that it allows

-- 
gentoo-commits@l.g.o mailing list