[gentoo-portage-dev] [PATCH] emerge: add --changed-deps/--binpkg-changed-deps (282927)

[Zac Medico]

The @changed-deps set is useful, but it has limitations similar to the
@installed set (see bug #387059), which can make it unsuitable for use
when updating the whole system. Therefore, implement two new options
that are analogous to --newuse and --binpkg-respect-use, called
--changed-deps and --binpkg-changed-deps.

The rationale for having a separate --binpkg-* option is the same in
both cases: depending on the situation, people may want different
behavior for binary packages. For example, just like
---binpkg-respect-use is automatically enabled if the user has not
specified --usepkgonly, so is --binpkg-changed-deps (though the user
can explicitly override the automatic behavior). In both cases,
inconsistencies in dependencies are automatically avoided, increasing
the probability of a successful dependency calculation.

X-Gentoo-Bug: 282927
X-Gentoo-Bug-URL: https://bugs.gentoo.org/show_bug.cgi?id=282927
---
 man/emerge.1                                    |  22 +++-
 pym/_emerge/create_depgraph_params.py           |  16 +++
 pym/_emerge/depgraph.py                         | 138 ++++++++++++++++++++++--
 pym/_emerge/main.py                             |  26 +++++
 pym/portage/dep/_slot_operator.py               |  13 +++
 pym/portage/tests/resolver/test_changed_deps.py | 120 +++++++++++++++++++++
 6 files changed, 323 insertions(+), 12 deletions(-)
 create mode 100644 pym/portage/tests/resolver/test_changed_deps.py

diff --git a/man/emerge.1 b/man/emerge.1
index faa1f33..7eb30a5 100644
--- a/man/emerge.1
+++ b/man/emerge.1
@@ -386,9 +386,20 @@ Specifies an integer number of times to backtrack if
 dependency calculation fails due to a conflict or an
 unsatisfied dependency (default: \'10\').
 .TP
+.BR "\-\-binpkg\-changed\-deps [ y | n ]"
+Tells emerge to ignore binary packages for which the corresponding
+ebuild dependencies have changed since the packages were built.
+In order to help avoid issues with resolving inconsistent dependencies,
+this option is automatically enabled unless the \fB\-\-usepkgonly\fR
+option is enabled. Behavior with respect to changed build\-time
+dependencies is controlled by the \fB\-\-with\-bdeps\fR option.
+.TP
 .BR "\-\-binpkg\-respect\-use [ y | n ]"
-Tells emerge to ignore binary packages if their use flags
-don't match the current configuration. (default: \'n\')
+Tells emerge to ignore binary packages if their USE flags
+don't match the current configuration. In order to help avoid issues
+with resolving inconsistent USE flag settings, this option is
+automatically enabled unless the \fB\-\-usepkgonly\fR option
+is enabled.
 .TP
 .BR "\-\-buildpkg [ y | n ] (\-b short option)"
 Tells emerge to build binary packages for all ebuilds processed in
@@ -410,6 +421,13 @@ Creates binary packages for all ebuilds processed without actually
 merging the packages.  This comes with the caveat that all build-time
 dependencies must already be emerged on the system.
 .TP
+.BR "\-\-changed\-deps [ y | n ]"
+Tells emerge to replace installed packages for which the corresponding
+ebuild dependencies have changed since the packages were built. This
+option also implies the \fB\-\-selective\fR option. Behavior with
+respect to changed build\-time dependencies is controlled by the
+\fB\-\-with\-bdeps\fR option.
+.TP
 .BR "\-\-changed\-use " (\fB\-U\fR)
 Tells emerge to include installed packages where USE flags have
 changed since installation. This option also implies the
diff --git a/pym/_emerge/create_depgraph_params.py b/pym/_emerge/create_depgraph_params.py
index 6f74de7..11e20f4 100644
--- a/pym/_emerge/create_depgraph_params.py
+++ b/pym/_emerge/create_depgraph_params.py
@@ -22,6 +22,8 @@ def create_depgraph_params(myopts, myaction):
 	# ignore_built_slot_operator_deps: ignore the slot/sub-slot := operator parts
 	#	of dependencies that have been recorded when packages where built
 	# with_test_deps: pull in test deps for packages matched by arguments
+	# changed_deps: rebuild installed packages with outdated deps
+	# binpkg_changed_deps: reject binary packages with outdated deps
 	myparams = {"recurse" : True}
 
 	bdeps = myopts.get("--with-bdeps")
@@ -51,6 +53,7 @@ def create_depgraph_params(myopts, myaction):
 		"--newuse" in myopts or \
 		"--reinstall" in myopts or \
 		"--noreplace" in myopts or \
+		myopts.get("--changed-deps", "n") != "n" or \
 		myopts.get("--selective", "n") != "n":
 		myparams["selective"] = True
 
@@ -99,6 +102,19 @@ def create_depgraph_params(myopts, myaction):
 		# have been specified.
 		myparams['binpkg_respect_use'] = 'auto'
 
+	binpkg_changed_deps = myopts.get('--binpkg-changed-deps')
+	if binpkg_changed_deps is not None:
+		myparams['binpkg_changed_deps'] = binpkg_changed_deps
+	elif '--usepkgonly' not in myopts:
+		# In order to avoid dependency resolution issues due to changed
+		# dependencies, enable this automatically, as long as it doesn't
+		# strongly conflict with other options that have been specified.
+		myparams['binpkg_changed_deps'] = 'auto'
+
+	changed_deps = myopts.get('--changed-deps')
+	if changed_deps is not None:
+		myparams['changed_deps'] = changed_deps
+
 	if myopts.get("--selective") == "n":
 		# --selective=n can be used to remove selective
 		# behavior that may have been implied by some
diff --git a/pym/_emerge/depgraph.py b/pym/_emerge/depgraph.py
index 28abea4..fae117d 100644
--- a/pym/_emerge/depgraph.py
+++ b/pym/_emerge/depgraph.py
@@ -24,7 +24,8 @@ from portage.dbapi._similar_name_search import similar_name_search
 from portage.dep import Atom, best_match_to_list, extract_affecting_use, \
 	check_required_use, human_readable_required_use, match_from_list, \
 	_repo_separator
-from portage.dep._slot_operator import ignore_built_slot_operator_deps
+from portage.dep._slot_operator import (ignore_built_slot_operator_deps,
+	strip_slots)
 from portage.eapi import eapi_has_strong_blocks, eapi_has_required_use, \
 	_get_eapi_attrs
 from portage.exception import (InvalidAtom, InvalidData, InvalidDependString,
@@ -794,14 +795,12 @@ class depgraph(object):
 		match the user's config.
 		"""
 		if not self._dynamic_config.ignored_binaries \
-			or '--quiet' in self._frozen_config.myopts \
-			or self._dynamic_config.myparams.get(
-			"binpkg_respect_use") in ("y", "n"):
+			or '--quiet' in self._frozen_config.myopts:
 			return
 
-		for pkg in list(self._dynamic_config.ignored_binaries):
+		ignored_binaries = {}
 
-			selected_pkg = list()
+		for pkg in list(self._dynamic_config.ignored_binaries):
 
 			for selected_pkg in self._dynamic_config._package_tracker.match(
 				pkg.root, pkg.slot_atom):
@@ -819,15 +818,38 @@ class depgraph(object):
 					self._dynamic_config.ignored_binaries.pop(pkg)
 					break
 
-		if not self._dynamic_config.ignored_binaries:
+			else:
+				for reason, info in self._dynamic_config.\
+					ignored_binaries[pkg].items():
+					ignored_binaries.setdefault(reason, {})[pkg] = info
+
+		if self._dynamic_config.myparams.get(
+			"binpkg_respect_use") in ("y", "n"):
+			ignored_binaries.pop("respect_use", None)
+
+		if self._dynamic_config.myparams.get(
+			"binpkg_changed_deps") in ("y", "n"):
+			ignored_binaries.pop("changed_deps", None)
+
+		if not ignored_binaries:
 			return
 
 		self._show_merge_list()
 
+		if ignored_binaries.get("respect_use"):
+			self._show_ignored_binaries_respect_use(
+				ignored_binaries["respect_use"])
+
+		if ignored_binaries.get("changed_deps"):
+			self._show_ignored_binaries_changed_deps(
+				ignored_binaries["changed_deps"])
+
+	def _show_ignored_binaries_respect_use(self, respect_use):
+
 		writemsg("\n!!! The following binary packages have been ignored " + \
 				"due to non matching USE:\n\n", noiselevel=-1)
 
-		for pkg, flags in self._dynamic_config.ignored_binaries.items():
+		for pkg, flags in respect_use.items():
 			flag_display = []
 			for flag in sorted(flags):
 				if flag not in pkg.use.enabled:
@@ -852,6 +874,30 @@ class depgraph(object):
 				line = colorize("INFORM", line)
 			writemsg(line + "\n", noiselevel=-1)
 
+	def _show_ignored_binaries_changed_deps(self, changed_deps):
+
+		writemsg("\n!!! The following binary packages have been "
+			"ignored due to changed dependencies:\n\n",
+			noiselevel=-1)
+
+		for pkg in changed_deps:
+			msg = "     %s%s%s" % (pkg.cpv, _repo_separator, pkg.repo)
+			if pkg.root_config.settings["ROOT"] != "/":
+				msg += " for %s" % pkg.root
+			writemsg("%s\n" % msg, noiselevel=-1)
+
+		msg = [
+			"",
+			"NOTE: The --binpkg-changed-deps=n option will prevent emerge",
+			"      from ignoring these binary packages if possible.",
+			"      Using --binpkg-changed-deps=y will silence this warning."
+		]
+
+		for line in msg:
+			if line:
+				line = colorize("INFORM", line)
+			writemsg(line + "\n", noiselevel=-1)
+
 	def _get_missed_updates(self):
 
 		# In order to minimize noise, show only the highest
@@ -2173,6 +2219,52 @@ class depgraph(object):
 				return flags
 		return None
 
+	def _changed_deps(self, pkg):
+
+		ebuild = None
+		try:
+			ebuild = self._pkg(pkg.cpv, "ebuild",
+				pkg.root_config, myrepo=pkg.repo)
+		except PackageNotFound:
+			# Use first available instance of the same version.
+			for ebuild in self._iter_match_pkgs(
+				pkg.root_config, "ebuild", Atom("=" + pkg.cpv)):
+				break
+
+		if ebuild is None:
+			changed = False
+		else:
+			if self._dynamic_config.myparams.get("bdeps", "n") == "y":
+				depvars = Package._dep_keys
+			else:
+				depvars = Package._runtime_keys
+
+			# Use _raw_metadata, in order to avoid interaction
+			# with --dynamic-deps.
+			try:
+				built_deps = []
+				for k in depvars:
+					dep_struct = portage.dep.use_reduce(
+						pkg._raw_metadata[k], uselist=pkg.use.enabled,
+						eapi=pkg.eapi, token_class=Atom)
+					strip_slots(dep_struct)
+					built_deps.append(dep_struct)
+			except InvalidDependString:
+				changed = True
+			else:
+				unbuilt_deps = []
+				for k in depvars:
+					dep_struct = portage.dep.use_reduce(
+						ebuild._raw_metadata[k],
+						uselist=pkg.use.enabled,
+						eapi=ebuild.eapi, token_class=Atom)
+					strip_slots(dep_struct)
+					unbuilt_deps.append(dep_struct)
+
+				changed = built_deps != unbuilt_deps
+
+		return changed
+
 	def _create_graph(self, allow_unsatisfied=False):
 		dep_stack = self._dynamic_config._dep_stack
 		dep_disjunctive_stack = self._dynamic_config._dep_disjunctive_stack
@@ -4595,8 +4687,14 @@ class depgraph(object):
 							mreasons = ["need to rebuild from source"]
 						elif pkg.installed and root_slot in self._rebuild.reinstall_list:
 							mreasons = ["need to rebuild from source"]
-						elif pkg.built and not mreasons:
+						elif (pkg.built and not mreasons and
+							self._dynamic_config.ignored_binaries.get(
+							pkg, {}).get("respect_use")):
 							mreasons = ["use flag configuration mismatch"]
+						elif (pkg.built and not mreasons and
+							self._dynamic_config.ignored_binaries.get(
+							pkg, {}).get("changed_deps")):
+							mreasons = ["changed deps"]
 					masked_packages.append(
 						(root_config, pkgsettings, cpv, repo, metadata, mreasons))
 
@@ -5693,6 +5791,12 @@ class depgraph(object):
 					# reject the built package if necessary.
 					reinstall_use = ("--newuse" in self._frozen_config.myopts or \
 						"--reinstall" in self._frozen_config.myopts)
+					changed_deps = (
+						self._dynamic_config.myparams.get(
+						"changed_deps", "n") != "n")
+					binpkg_changed_deps = (
+						self._dynamic_config.myparams.get(
+						"binpkg_changed_deps", "n") != "n")
 					respect_use = self._dynamic_config.myparams.get("binpkg_respect_use") in ("y", "auto")
 					if built and not useoldpkg and \
 						(not installed or matched_packages) and \
@@ -5719,8 +5823,22 @@ class depgraph(object):
 								forced_flags, old_use, iuses, now_use, cur_iuse)
 							if reinstall_for_flags:
 								if not pkg.installed:
-									self._dynamic_config.ignored_binaries.setdefault(pkg, set()).update(reinstall_for_flags)
+									self._dynamic_config.\
+										ignored_binaries.setdefault(
+										pkg, {}).setdefault(
+										"respect_use", set()).update(
+										reinstall_for_flags)
 								break
+
+						if (((installed and changed_deps) or
+							(not installed and binpkg_changed_deps)) and
+							self._changed_deps(pkg)):
+							if not installed:
+								self._dynamic_config.\
+									ignored_binaries.setdefault(
+									pkg, {})["changed_deps"] = True
+							break
+
 					# Compare current config to installed package
 					# and do not reinstall if possible.
 					if not installed and not useoldpkg and cpv in vardb.match(atom):
diff --git a/pym/_emerge/main.py b/pym/_emerge/main.py
index 7c707f9..ecbbdb0 100644
--- a/pym/_emerge/main.py
+++ b/pym/_emerge/main.py
@@ -129,7 +129,9 @@ def insert_optional_args(args):
 		'--autounmask-keep-masks': y_or_n,
 		'--autounmask-unrestricted-atoms' : y_or_n,
 		'--autounmask-write'     : y_or_n,
+		'--binpkg-changed-deps'  : y_or_n,
 		'--buildpkg'             : y_or_n,
+		'--changed-deps'         : y_or_n,
 		'--complete-graph'       : y_or_n,
 		'--deep'       : valid_integers,
 		'--depclean-lib-check'   : y_or_n,
@@ -353,6 +355,12 @@ def parse_opts(tmpcmdline, silent=False):
 			"action" : "store"
 		},
 
+		"--binpkg-changed-deps": {
+			"help"    : ("reject binary packages with outdated "
+				"dependencies"),
+			"choices" : true_y_or_n
+		},
+
 		"--buildpkg": {
 			"shortopt" : "-b",
 			"help"     : "build binary packages",
@@ -367,6 +375,12 @@ def parse_opts(tmpcmdline, silent=False):
 			"action" : "append"
 		},
 
+		"--changed-deps": {
+			"help"    : ("replace installed packages with "
+				"outdated dependencies"),
+			"choices" : true_y_or_n
+		},
+
 		"--config-root": {
 			"help":"specify the location for portage configuration files",
 			"action":"store"
@@ -722,6 +736,12 @@ def parse_opts(tmpcmdline, silent=False):
 	if myoptions.autounmask_write in true_y:
 		myoptions.autounmask_write = True
 
+	if myoptions.binpkg_changed_deps is not None:
+		if myoptions.binpkg_changed_deps in true_y:
+			myoptions.binpkg_changed_deps = 'y'
+		else:
+			myoptions.binpkg_changed_deps = 'n'
+
 	if myoptions.buildpkg in true_y:
 		myoptions.buildpkg = True
 
@@ -731,6 +751,12 @@ def parse_opts(tmpcmdline, silent=False):
 			parser.error("Invalid Atom(s) in --buildpkg-exclude parameter: '%s'\n" % \
 				(",".join(bad_atoms),))
 
+	if myoptions.changed_deps is not None:
+		if myoptions.changed_deps in true_y:
+			myoptions.changed_deps = 'y'
+		else:
+			myoptions.changed_deps = 'n'
+
 	if myoptions.changed_use is not False:
 		myoptions.reinstall = "changed-use"
 		myoptions.changed_use = False
diff --git a/pym/portage/dep/_slot_operator.py b/pym/portage/dep/_slot_operator.py
index 8b67fc5..8ce570d 100644
--- a/pym/portage/dep/_slot_operator.py
+++ b/pym/portage/dep/_slot_operator.py
@@ -8,6 +8,19 @@ from portage.eapi import _get_eapi_attrs
 from portage.exception import InvalidData
 from _emerge.Package import Package
 
+def strip_slots(dep_struct):
+	"""
+	Search dep_struct for any slot := operators and remove the
+	slot/sub-slot part, while preserving the operator. The result
+	is suitable for --changed-deps comparisons.
+	"""
+	for i, x in enumerate(dep_struct):
+		if isinstance(x, list):
+			strip_slots(x)
+		elif (isinstance(x, Atom) and
+			x.slot_operator == "=" and x.slot is not None):
+			dep_struct[i] = x.with_slot("=")
+
 def find_built_slot_operator_atoms(pkg):
 	atoms = {}
 	for k in Package._dep_keys:
diff --git a/pym/portage/tests/resolver/test_changed_deps.py b/pym/portage/tests/resolver/test_changed_deps.py
new file mode 100644
index 0000000..2421c53
--- /dev/null
+++ b/pym/portage/tests/resolver/test_changed_deps.py
@@ -0,0 +1,120 @@
+# Copyright 2014 Gentoo Foundation
+# Distributed under the terms of the GNU General Public License v2
+
+from portage.tests import TestCase
+from portage.tests.resolver.ResolverPlayground import (
+	ResolverPlayground, ResolverPlaygroundTestCase)
+
+class ChangedDepsTestCase(TestCase):
+
+	def testChangedDeps(self):
+
+		ebuilds = {
+			"app-misc/A-0": {
+				"DEPEND": "app-misc/B",
+				"RDEPEND": "app-misc/B",
+			},
+			"app-misc/B-0": {
+			}
+		}
+
+		binpkgs = {
+			"app-misc/A-0": {},
+		}
+
+		installed = {
+			"app-misc/A-0": {},
+		}
+
+		world= (
+			"app-misc/A",
+		)
+
+		test_cases = (
+
+			# --dynamic-deps=n causes the original deps to be respected
+			ResolverPlaygroundTestCase(
+				["@world"],
+				success = True,
+				options = {
+					"--update": True,
+					"--deep": True,
+					"--dynamic-deps": "n",
+					"--usepkg": True,
+				},
+				mergelist = []
+			),
+
+			# --dynamic-deps causes app-misc/B to get pulled in
+			ResolverPlaygroundTestCase(
+				["@world"],
+				success = True,
+				options = {
+					"--update": True,
+					"--deep": True,
+					"--usepkg": True,
+				},
+				mergelist = ["app-misc/B-0"]
+			),
+
+			# --changed-deps causes app-misc/A to be rebuilt
+			ResolverPlaygroundTestCase(
+				["@world"],
+				success = True,
+				options = {
+					"--update": True,
+					"--deep": True,
+					"--changed-deps": "y",
+					"--usepkg": True,
+				},
+				mergelist = ["app-misc/B-0", "app-misc/A-0"]
+			),
+
+			# --usepkgonly prevents automatic --binpkg-changed-deps
+			ResolverPlaygroundTestCase(
+				["app-misc/A"],
+				success = True,
+				options = {
+					"--changed-deps": "y",
+					"--usepkgonly": True,
+				},
+				mergelist = ["[binary]app-misc/A-0"]
+			),
+
+			# Test automatic --binpkg-changed-deps, which cases the
+			# binpkg with stale deps to be ignored (with warning
+			# message)
+			ResolverPlaygroundTestCase(
+				["app-misc/A"],
+				success = True,
+				options = {
+					"--usepkg": True,
+				},
+				mergelist = ["app-misc/B-0", "app-misc/A-0"]
+			),
+		)
+		test_cases = (
+
+			# Forcibly disable --binpkg-changed-deps, which causes
+			# --changed-deps to be overridden by --binpkg-changed-deps
+			ResolverPlaygroundTestCase(
+				["app-misc/A"],
+				success = True,
+				options = {
+					"--binpkg-changed-deps": "n",
+					"--changed-deps": "y",
+					"--usepkg": True,
+				},
+				mergelist = ["[binary]app-misc/A-0"]
+			),
+		)
+
+		playground = ResolverPlayground(debug=False, ebuilds=ebuilds,
+			binpkgs=binpkgs, installed=installed, world=world)
+		try:
+			for test_case in test_cases:
+				playground.run_TestCase(test_case)
+				self.assertEqual(test_case.test_success,
+					True, test_case.fail_msg)
+		finally:
+			playground.cleanup()
-- 
2.0.5