[gentoo-portage-dev] [PATCH] emerge --usepkgonly: propagate implicit IUSE and USE_EXPAND (bug 640318)

[Zac Medico]

For emerge --usepkgonly mode, it's useful to be able to operate without
dependence on a local ebuild repository and profile. Therefore,
merge Packages header settings from the binary package database
(including binhost(s) if enabled) into the profile configuration,
in order to propagate implicit IUSE and USE_EXPAND settings for use
with binary and installed packages. Values are appended, so the result
is a union of elements.

Also use ARCH from the binary package database if it's not defined by
the profile, since ARCH is used for validation by emerge's profile_check
function, and also for KEYWORDS logic in the _getmaskingstatus function.

All changes are currently confined to --usepkgonly mode, since this is
the mode where it is needed the most, and this ensures that behavior of
source-based builds is completely unaffected.

The changes only affect dependency calculations (where implicit IUSE
plays a role) and the user interface (display of USE_EXPAND flags).
The bash execution environment for binary and installed packages is
completely unaffected, since that environment relies on variables loaded
from environment.bz2 files.

Bug: https://bugs.gentoo.org/640318
---
 pym/_emerge/actions.py                     | 48 +++++++++++++++++--------
 pym/_emerge/main.py                        |  4 ---
 pym/portage/dbapi/bintree.py               | 56 ++++++++++++++++++++++++++++++
 pym/portage/package/ebuild/config.py       |  7 ++--
 pym/portage/package/ebuild/profile_iuse.py | 32 +++++++++++++++++
 5 files changed, 127 insertions(+), 20 deletions(-)
 create mode 100644 pym/portage/package/ebuild/profile_iuse.py

diff --git a/pym/_emerge/actions.py b/pym/_emerge/actions.py
index 347ef5fbc..b90aa8cb0 100644
--- a/pym/_emerge/actions.py
+++ b/pym/_emerge/actions.py
@@ -78,6 +78,7 @@ from _emerge.depgraph import backtrack_depgraph, depgraph, resume_depgraph
 from _emerge.DepPrioritySatisfiedRange import DepPrioritySatisfiedRange
 from _emerge.emergelog import emergelog
 from _emerge.is_valid_package_atom import is_valid_package_atom
+from _emerge.main import profile_check
 from _emerge.MetadataRegen import MetadataRegen
 from _emerge.Package import Package
 from _emerge.ProgressHandler import ProgressHandler
@@ -2209,9 +2210,21 @@ def action_uninstall(settings, trees, ldpath_mtimes,
 	return rval
 
 def adjust_configs(myopts, trees):
-	for myroot in trees:
+	for myroot, mytrees in trees.items():
 		mysettings =  trees[myroot]["vartree"].settings
 		mysettings.unlock()
+
+		# For --usepkgonly mode, propagate settings from the binary package
+		# database, so that it's possible to operate without dependence on
+		# a local ebuild repository and profile.
+		if ('--usepkgonly' in myopts and
+			mytrees['bintree']._propagate_config(mysettings)):
+			# Also propagate changes to the portdbapi doebuild_settings
+			# attribute which is used by Package instances for USE
+			# calculations (in support of --binpkg-respect-use).
+			mytrees['porttree'].dbapi.doebuild_settings = \
+				portage.config(clone=mysettings)
+
 		adjust_config(myopts, mysettings)
 		mysettings.lock()
 
@@ -2868,7 +2881,27 @@ def run_action(emerge_config):
 			"--usepkg", "--usepkgonly"):
 			emerge_config.opts.pop(opt, None)
 
+	# Populate the bintree with current --getbinpkg setting.
+	# This needs to happen before:
+	# * expand_set_arguments, in case any sets use the bintree
+	# * adjust_configs and profile_check, in order to propagate settings
+	#   implicit IUSE and USE_EXPAND settings from the binhost(s)
+	if (emerge_config.action in ('search', None) and
+		'--usepkg' in emerge_config.opts):
+		for mytrees in emerge_config.trees.values():
+			try:
+				mytrees['bintree'].populate(
+					getbinpkgs='--getbinpkg' in emerge_config.opts)
+			except ParseError as e:
+				writemsg('\n\n!!!%s.\nSee make.conf(5) for more info.\n'
+						 % (e,), noiselevel=-1)
+				return 1
+
 	adjust_configs(emerge_config.opts, emerge_config.trees)
+
+	if profile_check(emerge_config.trees, emerge_config.action) != os.EX_OK:
+		return 1
+
 	apply_priorities(emerge_config.target_config.settings)
 
 	if ("--autounmask-continue" in emerge_config.opts and
@@ -2919,19 +2952,6 @@ def run_action(emerge_config):
 		# Freeze the portdbapi for performance (memoize all xmatch results).
 		mydb.freeze()
 
-		if emerge_config.action in ('search', None) and \
-			"--usepkg" in emerge_config.opts:
-			# Populate the bintree with current --getbinpkg setting.
-			# This needs to happen before expand_set_arguments(), in case
-			# any sets use the bintree.
-			try:
-				mytrees["bintree"].populate(
-					getbinpkgs="--getbinpkg" in emerge_config.opts)
-			except ParseError as e:
-				writemsg("\n\n!!!%s.\nSee make.conf(5) for more info.\n"
-						 % e, noiselevel=-1)
-				return 1
-
 	del mytrees, mydb
 
 	for x in emerge_config.args:
diff --git a/pym/_emerge/main.py b/pym/_emerge/main.py
index 4aa9585fa..226387495 100644
--- a/pym/_emerge/main.py
+++ b/pym/_emerge/main.py
@@ -1269,10 +1269,6 @@ def emerge_main(args=None):
 	except locale.Error as e:
 		writemsg_level("setlocale: %s\n" % e, level=logging.WARN)
 
-	rval = profile_check(emerge_config.trees, emerge_config.action)
-	if rval != os.EX_OK:
-		return rval
-
 	tmpcmdline = []
 	if "--ignore-default-opts" not in myopts:
 		tmpcmdline.extend(portage.util.shlex_split(
diff --git a/pym/portage/dbapi/bintree.py b/pym/portage/dbapi/bintree.py
index a963c578e..269a7b226 100644
--- a/pym/portage/dbapi/bintree.py
+++ b/pym/portage/dbapi/bintree.py
@@ -29,6 +29,7 @@ from portage.dep import Atom, use_reduce, paren_enclose
 from portage.exception import AlarmSignal, InvalidData, InvalidPackageName, \
 	ParseError, PermissionDenied, PortageException
 from portage.localization import _
+from portage.package.ebuild.profile_iuse import iter_iuse_vars
 from portage import _movefile
 from portage import os
 from portage import _encodings
@@ -322,6 +323,7 @@ class binarytree(object):
 			self._pkgindex_use_evaluated_keys = \
 				("BDEPEND", "DEPEND", "HDEPEND", "LICENSE", "RDEPEND",
 				"PDEPEND", "PROPERTIES", "RESTRICT")
+			self._pkgindex_header = None
 			self._pkgindex_header_keys = set([
 				"ACCEPT_KEYWORDS", "ACCEPT_LICENSE",
 				"ACCEPT_PROPERTIES", "ACCEPT_RESTRICT", "CBUILD",
@@ -805,6 +807,10 @@ class binarytree(object):
 				pkgindex.packages.extend(iter(metadata.values()))
 				self._update_pkgindex_header(pkgindex.header)
 
+			self._pkgindex_header = {}
+			self._merge_pkgindex_header(pkgindex.header,
+				self._pkgindex_header)
+
 		return pkgindex if update_pkgindex else None
 
 	def _populate_remote(self, getbinpkg_refresh=True):
@@ -1041,6 +1047,8 @@ class binarytree(object):
 					self.dbapi.cpv_inject(cpv)
 
 				self._remote_has_index = True
+				self._merge_pkgindex_header(pkgindex.header,
+					self._pkgindex_header)
 
 	def inject(self, cpv, filename=None):
 		"""Add a freshly built package to the database.  This updates
@@ -1292,6 +1300,54 @@ class binarytree(object):
 			inherited_keys=self._pkgindex_inherited_keys,
 			translated_keys=self._pkgindex_translated_keys)
 
+	@staticmethod
+	def _merge_pkgindex_header(src, dest):
+		"""
+		Merge Packages header settings from src to dest, in order to
+		propagate implicit IUSE and USE_EXPAND settings for use with
+		binary and installed packages. Values are appended, so the
+		result is a union of elements from src and dest.
+
+		Pull in ARCH if it's not defined, since it's used for validation
+		by emerge's profile_check function, and also for KEYWORDS logic
+		in the _getmaskingstatus function.
+
+		@param src: source mapping (read only)
+		@type src: Mapping
+		@param dest: destination mapping
+		@type dest: MutableMapping
+		"""
+		for k, v in iter_iuse_vars(src):
+			v_before = dest.get(k)
+			if v_before is not None:
+				v = v_before + ' ' + v
+			dest[k] = v
+
+		if 'ARCH' not in dest and 'ARCH' in src:
+			dest['ARCH'] = src['ARCH']
+
+	def _propagate_config(self, config):
+		"""
+		Propagate implicit IUSE and USE_EXPAND settings from the binary
+		package database to a config instance. If settings are not
+		available to propagate, then this will do nothing and return
+		False.
+
+		@param config: config instance
+		@type config: portage.config
+		@rtype: bool
+		@return: True if settings successfully propagated, False if settings
+			were not available to propagate.
+		"""
+		if self._pkgindex_header is None:
+			return False
+
+		self._merge_pkgindex_header(self._pkgindex_header,
+			config.configdict['defaults'])
+		config.regenerate()
+		config._init_iuse()
+		return True
+
 	def _update_pkgindex_header(self, header):
 		"""
 		Add useful settings to the Packages file header, for use by
diff --git a/pym/portage/package/ebuild/config.py b/pym/portage/package/ebuild/config.py
index a32892e94..071385684 100644
--- a/pym/portage/package/ebuild/config.py
+++ b/pym/portage/package/ebuild/config.py
@@ -943,8 +943,7 @@ class config(object):
 			if bsd_chflags:
 				self.features.add('chflags')
 
-			self._iuse_effective = self._calc_iuse_effective()
-			self._iuse_implicit_match = _iuse_implicit_match_cache(self)
+			self._init_iuse()
 
 			self._validate_commands()
 
@@ -961,6 +960,10 @@ class config(object):
 		if mycpv:
 			self.setcpv(mycpv)
 
+	def _init_iuse(self):
+		self._iuse_effective = self._calc_iuse_effective()
+		self._iuse_implicit_match = _iuse_implicit_match_cache(self)
+
 	@property
 	def mygcfg(self):
 		warnings.warn("portage.config.mygcfg is deprecated", stacklevel=3)
diff --git a/pym/portage/package/ebuild/profile_iuse.py b/pym/portage/package/ebuild/profile_iuse.py
new file mode 100644
index 000000000..d3f201e54
--- /dev/null
+++ b/pym/portage/package/ebuild/profile_iuse.py
@@ -0,0 +1,32 @@
+# Copyright 2018 Gentoo Foundation
+# Distributed under the terms of the GNU General Public License v2
+
+__all__ = (
+	'iter_iuse_vars',
+)
+
+
+def iter_iuse_vars(env):
+	"""
+	Iterate over (key, value) pairs of profile variables that contribute
+	to implicit IUSE for EAPI 5 and later.
+
+	@param env: Ebuild environment
+	@type env: Mapping
+	@rtype: iterator
+	@return: iterator over (key, value) pairs of profile variables
+	"""
+
+	for k in ('IUSE_IMPLICIT', 'USE_EXPAND_IMPLICIT', 'USE_EXPAND_UNPREFIXED', 'USE_EXPAND'):
+		v = env.get(k)
+		if v is not None:
+			yield (k, v)
+
+	use_expand_implicit = frozenset(env.get('USE_EXPAND_IMPLICIT', '').split())
+
+	for v in env.get('USE_EXPAND_UNPREFIXED', '').split() + env.get('USE_EXPAND', '').split():
+		if v in use_expand_implicit:
+			k = 'USE_EXPAND_VALUES_' + v
+			v = env.get(k)
+			if v is not None:
+				yield (k, v)
-- 
2.13.6