[gentoo-commits] portage r10876 - in main/branches/prefix: . man pym/_emerge

[Fabian Groffen (grobian)]

Author: grobian
Date: 2008-07-01 16:48:46 +0000 (Tue, 01 Jul 2008)
New Revision: 10876

Modified:
   main/branches/prefix/NEWS
   main/branches/prefix/man/emerge.1
   main/branches/prefix/pym/_emerge/__init__.py
   main/branches/prefix/pym/_emerge/help.py
Log:
   Merged from trunk 10816:10824

   | 10817   | Generate and log eerror messages for any packages that need  |
   | zmedico | to be dropped due to unsatisfied dependencies when           |
   |         | --keep-going is enabled.                                     |
   
   | 10818   | Add documentation for the new --keep-going option.           |
   | zmedico |                                                              |
   
   | 10819   | Fix typo.                                                    |
   | zmedico |                                                              |
   
   | 10820   | Pass the digraph into the MergeTask constructor. It's        |
   | zmedico | currently unused but it will be useful for parallel          |
   |         | scheduling.                                                  |
   
   | 10821   | Break references from Package instances in the digraph       |
   | zmedico | before passing it into the MergeTask constructor.            |
   
   | 10822   | Remove the MergeTask._blocker_db attribute since it can just |
   | zmedico | be a local variable.                                         |
   
   | 10823   | Fix depgraph.break_refs() to work with DependencyArg         |
   | zmedico | instances (or anything else that may have a "root_config"    |
   |         | attribute).                                                  |
   
   | 10824   | Bug #226307 - Copy come code from                            |
   | zmedico | depgraph._iter_atoms_for_pkg() that was used to solve bug    |
   |         | #218854, and use it inside unmerge() when matching sets to   |
   |         | packages.                                                    |


Modified: main/branches/prefix/NEWS
===================================================================
--- main/branches/prefix/NEWS	2008-07-01 12:55:49 UTC (rev 10875)
+++ main/branches/prefix/NEWS	2008-07-01 16:48:46 UTC (rev 10876)
@@ -3,6 +3,11 @@
 portage-2.2
 -------------
 
+* Add emerge --keep-going option to continue as much as possible after
+  an error. When an error occurs, dependencies are recalculated for
+  remaining packages and any with unsatisfied dependencies are
+  automatically dropped. The --skipfirst option automatically drops
+  packages in the same way, and also drops any masked packages.
 * Add subversion support for repoman.
 * It is now possible to use `emerge <file>` to reinstall the package that
   installed a particular file. Package contents entries are indexed to

Modified: main/branches/prefix/man/emerge.1
===================================================================
--- main/branches/prefix/man/emerge.1	2008-07-01 12:55:49 UTC (rev 10875)
+++ main/branches/prefix/man/emerge.1	2008-07-01 16:48:46 UTC (rev 10876)
@@ -321,6 +321,12 @@
 .BR "\-\-ignore-default-opts"
 Causes \fIEMERGE_DEFAULT_OPTS\fR (see \fBmake.conf\fR(5)) to be ignored.
 .TP
+.BR "\-\-keep\-going"
+Continue as much as possible after an error. When an error occurs,
+dependencies are recalculated for remaining packages and any with
+unsatisfied dependencies are automatically dropped. Also see
+the related \fB\-\-skipfirst\fR option.
+.TP
 .BR "\-\-newuse " (\fB\-N\fR)
 Tells emerge to include installed packages where USE flags have changed since 
 compilation.  USE flag changes include:
@@ -399,9 +405,10 @@
 .TP
 .BR "\-\-skipfirst"
 This option is only valid when used with \fB\-\-resume\fR.  It removes the 
-first package in the resume list so that a merge may continue in the presence 
-of an uncorrectable or inconsequential error.  This should only be used in 
-cases where skipping the package will not result in failed dependencies.
+first package in the resume list. Dependencies are recalculated for
+remaining packages and any that have unsatisfied dependencies or are
+masked will be automatically dropped. Also see the related
+\fB\-\-keep\-going\fR option.
 .TP
 .BR "\-\-tree " (\fB\-t\fR)
 Shows the dependency tree for the given target by indenting dependencies.

Modified: main/branches/prefix/pym/_emerge/__init__.py
===================================================================
--- main/branches/prefix/pym/_emerge/__init__.py	2008-07-01 12:55:49 UTC (rev 10875)
+++ main/branches/prefix/pym/_emerge/__init__.py	2008-07-01 16:48:46 UTC (rev 10876)
@@ -67,6 +67,7 @@
 import portage.exception
 from portage.const import EPREFIX, BPREFIX
 from portage.data import secpass
+from portage.elog.messages import eerror
 from portage.util import normalize_path as normpath
 from portage.util import writemsg
 from portage.sets import load_default_config, SETPREFIX
@@ -3892,26 +3893,24 @@
 			retlist.reverse()
 		return retlist
 
-	def break_refs(self, mergelist):
+	def break_refs(self, nodes):
 		"""
 		Take a mergelist like that returned from self.altlist() and
 		break any references that lead back to the depgraph. This is
 		useful if you want to hold references to packages without
 		also holding the depgraph on the heap.
 		"""
-		for node in mergelist:
-			if not isinstance(node, Package):
-				continue
+		for node in nodes:
+			if hasattr(node, "root_config"):
+				# The FakeVartree references the _package_cache which
+				# references the depgraph. So that Package instances don't
+				# hold the depgraph and FakeVartree on the heap, replace
+				# the RootConfig that references the FakeVartree with the
+				# original RootConfig instance which references the actual
+				# vartree.
+				node.root_config = \
+					self._trees_orig[node.root_config.root]["root_config"]
 
-			# The FakeVartree references the _package_cache which
-			# references the depgraph. So that Package instances don't
-			# hold the depgraph and FakeVartree on the heap, replace
-			# the RootConfig that references the FakeVartree with the
-			# original RootConfig instance which references the actual
-			# vartree.
-			node.root_config = \
-				self._trees_orig[node.root]["root_config"]
-
 	def _resolve_conflicts(self):
 		if not self._complete_graph():
 			raise self._unknown_internal_error()
@@ -5914,7 +5913,7 @@
 		"--nodeps", "--pretend"])
 
 	def __init__(self, settings, trees, mtimedb, myopts,
-		spinner, mergelist, favorites):
+		spinner, mergelist, favorites, digraph):
 		self.settings = settings
 		self.target_root = settings["ROOT"]
 		self.trees = trees
@@ -5927,11 +5926,9 @@
 		if settings.get("PORTAGE_DEBUG", "") == "1":
 			self.edebug = 1
 		self.pkgsettings = {}
-		self._blocker_db = {}
 		for root in trees:
 			self.pkgsettings[root] = portage.config(
 				clone=trees[root]["vartree"].settings)
-			self._blocker_db[root] = BlockerDB(trees[root]["root_config"])
 		self.curval = 0
 		self._spawned_pids = []
 
@@ -5954,10 +5951,11 @@
 		import gc
 		gc.collect()
 
+		blocker_db = BlockerDB(self.trees[new_pkg.root]["root_config"])
+
 		blocker_dblinks = []
-		for blocking_pkg in self._blocker_db[
-			new_pkg.root].findInstalledBlockers(new_pkg,
-			acquire_lock=acquire_lock):
+		for blocking_pkg in blocker_db.findInstalledBlockers(
+			new_pkg, acquire_lock=acquire_lock):
 			if new_pkg.slot_atom == blocking_pkg.slot_atom:
 				continue
 			if new_pkg.cpv == blocking_pkg.cpv:
@@ -5999,15 +5997,40 @@
 				break
 			if mergelist[0][-1] != "merge":
 				break
+
 			# Skip the first one because it failed to build or install.
+			pkg_key = tuple(mergelist[0])
 			del mergelist[0]
+			failed_pkg = None
+			for task in self._mergelist:
+				if task == pkg_key:
+					failed_pkg = task
+					break
+			if failed_pkg is None:
+				break
 			if not mergelist:
 				break
-			mylist = self._calc_resume_list()
+
+			mylist, dropped_tasks = self._calc_resume_list()
 			clear_caches(self.trees)
 			if not mylist:
 				break
-			self.curval += 1
+
+			if dropped_tasks:
+
+				def _eerror(lines):
+					for l in lines:
+						eerror(l, phase="other", key=failed_pkg.cpv)
+
+				msg = []
+				msg.append("One or more packages have been " + \
+					"dropped due to unsatisfied dependencies:")
+				msg.append("")
+				msg.extend("  " + str(task) for task in dropped_tasks)
+				msg.append("")
+				_eerror(msg)
+				del _eerror, msg
+			del dropped_tasks
 			self._mergelist = mylist
 
 		return rval
@@ -6035,20 +6058,12 @@
 
 		if not success:
 			mydepgraph.display_problems()
-			return None
+			return (None, None)
 
-		if dropped_tasks:
-			portage.writemsg("!!! One or more packages have been " + \
-				"dropped due to\n" + \
-				"!!! masking or unsatisfied dependencies:\n\n",
-				noiselevel=-1)
-			for task in dropped_tasks:
-				portage.writemsg("  " + str(task) + "\n", noiselevel=-1)
-			portage.writemsg("\n", noiselevel=-1)
-
 		mylist = mydepgraph.altlist()
 		mydepgraph.break_refs(mylist)
-		return mylist
+		mydepgraph.break_refs(dropped_tasks)
+		return (mylist, dropped_tasks)
 
 	def _poll_child_processes(self):
 		"""
@@ -6562,6 +6577,19 @@
 	global_unmerge=0
 	xterm_titles = "notitles" not in settings.features
 
+	pkg_cache = {}
+
+	def _pkg(cpv):
+		pkg = pkg_cache.get(cpv)
+		if pkg is None:
+			pkg = Package(cpv=cpv, installed=True,
+				metadata=izip(Package.metadata_keys,
+					vartree.dbapi.aux_get(cpv, Package.metadata_keys)),
+				root_config=root_config,
+				type_name="installed")
+			pkg_cache[cpv] = pkg
+		return pkg
+
 	vdb_path = os.path.join(settings["ROOT"], portage.VDB_PATH)
 	try:
 		# At least the parent needs to exist for the lock file.
@@ -6804,6 +6832,12 @@
 	# relevant package sets.
 	for cp in xrange(len(pkgmap)):
 		for cpv in pkgmap[cp]["selected"].copy():
+			try:
+				pkg = _pkg(cpv)
+			except KeyError:
+				# It could have been uninstalled
+				# by a concurrent process.
+				continue
 			parents = []
 			for s in installed_sets:
 				# skip sets that the user requested to unmerge, and skip world 
@@ -6814,9 +6848,34 @@
 				# only check instances of EditablePackageSet as other classes are generally used for
 				# special purposes and can be ignored here (and are usually generated dynamically, so the
 				# user can't do much about them anyway)
-				elif sets[s].containsCPV(cpv) \
-					and isinstance(sets[s], EditablePackageSet):
-					parents.append(s)
+				if isinstance(sets[s], EditablePackageSet):
+
+					# This is derived from a snippet of code in the
+					# depgraph._iter_atoms_for_pkg() method.
+					for atom in sets[s].iterAtomsForPackage(pkg):
+						inst_matches = vartree.dbapi.match(atom)
+						inst_matches.reverse() # descending order
+						higher_slot = None
+						for inst_cpv in inst_matches:
+							try:
+								inst_pkg = _pkg(inst_cpv)
+							except KeyError:
+								# It could have been uninstalled
+								# by a concurrent process.
+								continue
+
+							if inst_pkg.cp != atom.cp:
+								continue
+							if pkg >= inst_pkg:
+								# This is descending order, and we're not
+								# interested in any versions <= pkg given.
+								break
+							if pkg.slot_atom != inst_pkg.slot_atom:
+								higher_slot = inst_pkg
+								break
+						if higher_slot is None:
+							parents.append(s)
+							break
 			if parents:
 				#print colorize("WARN", "Package %s is going to be unmerged," % cpv)
 				#print colorize("WARN", "but still listed in the following package sets:")
@@ -8981,11 +9040,12 @@
 				time.sleep(3) # allow the parent to have first fetch
 			mymergelist = mydepgraph.altlist()
 			mydepgraph.break_refs(mymergelist)
+			mydepgraph.break_refs(mydepgraph.digraph.order)
+			mergetask = MergeTask(settings, trees, mtimedb, myopts,
+				spinner, mymergelist, favorites, mydepgraph.digraph)
 			del mydepgraph
 			clear_caches(trees)
 
-			mergetask = MergeTask(settings, trees, mtimedb, myopts,
-				spinner, mymergelist, favorites)
 			retval = mergetask.merge()
 			merge_count = mergetask.curval
 		else:
@@ -9027,11 +9087,12 @@
 			pkglist = mydepgraph.altlist()
 			mydepgraph.saveNomergeFavorites()
 			mydepgraph.break_refs(pkglist)
+			mydepgraph.break_refs(mydepgraph.digraph.order)
+			mergetask = MergeTask(settings, trees, mtimedb, myopts,
+				spinner, pkglist, favorites, mydepgraph.digraph)
 			del mydepgraph
 			clear_caches(trees)
 
-			mergetask = MergeTask(settings, trees, mtimedb, myopts,
-				spinner, pkglist, favorites)
 			retval = mergetask.merge()
 			merge_count = mergetask.curval
 

Modified: main/branches/prefix/pym/_emerge/help.py
===================================================================
--- main/branches/prefix/pym/_emerge/help.py	2008-07-01 12:55:49 UTC (rev 10875)
+++ main/branches/prefix/pym/_emerge/help.py	2008-07-01 16:48:46 UTC (rev 10876)
@@ -15,8 +15,8 @@
 	print "   "+turquoise("emerge")+" "+turquoise("--resume")+" [ "+green("--pretend")+" | "+green("--ask")+" | "+green("--skipfirst")+" ]"
 	print "   "+turquoise("emerge")+" "+turquoise("--help")+" [ "+green("system")+" | "+green("world")+" | "+green("--sync")+" ] "
 	print bold("Options:")+" "+green("-")+"["+green("abBcCdDefgGhkKlnNoOpqPsStuvV")+"]"
-	print "          [ "+green("--columns")+"   ] [ "+green("--deep")+"      ] [ "+green("--newuse")+"    ]"
-	print "          [ "+green("--noconfmem")+" ] [ "+green("--nospinner")+" ] [ "+green("--oneshot")+"   ]"
+	print "          [ "+green("--columns")+"   ] [ "+green("--deep")+"      ] [ "+green("--keep-going")+" ] [ "+green("--newuse")+"   ]"
+	print "          [ "+green("--noconfmem")+" ] [ "+green("--nospinner")+" ] [ "+green("--oneshot")+"    ]"
 	print "          [ " + green("--color")+" < " + turquoise("y") + " | "+ turquoise("n")+" >           ] [ "+green("--complete-graph")+"            ]"
 	print "          [ "+green("--reinstall ")+turquoise("changed-use")+"     ] [ " + green("--with-bdeps")+" < " + turquoise("y") + " | "+ turquoise("n")+" >      ]"
 	print bold("Actions:")+"  [ "+green("--clean")+" | "+green("--depclean")+" | "+green("--prune")+" | "+green("--regen")+" | "+green("--search")+" | "+green("--unmerge")+" ]"
@@ -292,6 +292,17 @@
 		print "              downloaded from the remote server without consulting packages"
 		print "              existing in the packages directory."
 		print
+		print "       "+green("--keep-going")
+		desc = "Continue as much as possible after " + \
+			"an error. When an error occurs, " + \
+			"dependencies are recalculated for " + \
+			"remaining packages and any with " + \
+			"unsatisfied dependencies are " + \
+			"automatically dropped. Also see " + \
+			"the related --skipfirst option."
+		for line in wrap(desc, desc_width):
+			print desc_indent + line
+		print
 		print "       "+green("--newuse")+" ("+green("-N")+" short option)"
 		print "              Tells emerge to include installed packages where USE flags have "
 		print "              changed since installation."
@@ -349,11 +360,17 @@
 		print "              enabled are added or removed."
 		print
 		print "       "+green("--skipfirst")
-		print "              This option is only valid in a resume situation. It removes the"
-		print "              first package in the resume list so that a merge may continue in"
-		print "              the presence of an uncorrectable or inconsequential error. This"
-		print "              should only be used in cases where skipping the package will not"
-		print "              result in failed dependencies."
+		desc = "This option is only valid when " + \
+			"used with --resume.  It removes the " + \
+			"first package in the resume list. " + \
+			"Dependencies are recalculated for " + \
+			"remaining packages and any that " + \
+			"have unsatisfied dependencies or are " + \
+			"masked will be automatically dropped. " + \
+			"Also see the related " + \
+			"--keep-going option."
+		for line in wrap(desc, desc_width):
+			print desc_indent + line
 		print
 		print "       "+green("--tree")+" ("+green("-t")+" short option)"
 		print "              Shows the dependency tree using indentation for dependencies."

-- 
gentoo-commits@l.g.o mailing list