[gentoo-commits] proj/portage:master commit in: man/, lib/_emerge/, lib/portage/dbapi/, lib/portage/tests/emerge/

[Zac Medico]

commit:     8faad11a18fcc33329931a75002f293e8fa462eb
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Mon Nov 25 05:08:14 2019 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Wed Nov 27 03:19:20 2019 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=8faad11a

emerge: add --quickpkg-direct option

Enable use of installed packages directly as binary
packages. This is similar to using binary packages produced by
quickpkg(1), but installed packages are used directly as though
they are binary packages. This option only works in combination
with the --root=DIR option, and it comes with the caveat that
packages are only allowed to be installed into the root that
is specified by the --root=DIR option. The other root which
serves as a source of packages is assumed to be immutable
during the entire operation (similar to --buildpkgonly mode).

Default behavior for handling of protected configuration files
is controlled by the QUICKPKG_DEFAULT_OPTS variable. When a
configuration file is not included because it is protected, an
ewarn message is logged.

Suggested use cases:

* Install packages from a buildtime container into an empty root,
  in order to create a minimal runtime container (which need not
  include a package manager). In a multi-stage Dockerfile, install
  runtime files to an empty directory in the build stage, and in
  the final stage use COPY to populate a container with the
  contents of that directory. For greater efficiency, use buildah
  to install directly into a mounted container, avoiding the COPY
  step. Use the emerge --usepkgonly and --ignore-soname-deps=n
  options to account for soname dependencies, allowing implicit
  system dependencies such as glibc to be automatically pulled
  into the runtime image.

* Enable a live usb, iso, or pxe image to act as a binary
  installer that uses packages installed in the live image as a
  source of binary packages.

Bug: https://bugs.gentoo.org/699986
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>

 lib/_emerge/Binpkg.py                   |  59 +++++++++-------
 lib/_emerge/Scheduler.py                |   7 +-
 lib/_emerge/actions.py                  |  37 ++++++++--
 lib/_emerge/depgraph.py                 |  19 ++++++
 lib/_emerge/main.py                     |   7 +-
 lib/portage/dbapi/__init__.py           |   5 +-
 lib/portage/dbapi/bintree.py            | 112 ++++++++++++++++++++++++++++--
 lib/portage/dbapi/vartree.py            | 117 ++++++++++++++++++++++++++++++--
 lib/portage/tests/emerge/test_simple.py |   3 +-
 man/emerge.1                            |  19 +++++-
 10 files changed, 338 insertions(+), 47 deletions(-)

diff --git a/lib/_emerge/Binpkg.py b/lib/_emerge/Binpkg.py
index f9cffa26d..b5a69f8e7 100644
--- a/lib/_emerge/Binpkg.py
+++ b/lib/_emerge/Binpkg.py
@@ -7,7 +7,6 @@ import _emerge.emergelog
 from _emerge.EbuildPhase import EbuildPhase
 from _emerge.BinpkgFetcher import BinpkgFetcher
 from _emerge.BinpkgEnvExtractor import BinpkgEnvExtractor
-from _emerge.BinpkgExtractorAsync import BinpkgExtractorAsync
 from _emerge.CompositeTask import CompositeTask
 from _emerge.BinpkgVerifier import BinpkgVerifier
 from _emerge.EbuildMerge import EbuildMerge
@@ -16,6 +15,7 @@ from _emerge.SpawnProcess import SpawnProcess
 from portage.eapi import eapi_exports_replace_vars
 from portage.util import ensure_dirs
 from portage.util._async.AsyncTaskFuture import AsyncTaskFuture
+from portage.util.futures.compat_coroutine import coroutine
 import portage
 from portage import os
 from portage import shutil
@@ -135,11 +135,14 @@ class Binpkg(CompositeTask):
 
 		pkg = self.pkg
 		pkg_count = self.pkg_count
-		fetcher = BinpkgFetcher(background=self.background,
-			logfile=self.settings.get("PORTAGE_LOG_FILE"), pkg=self.pkg,
-			pretend=self.opts.pretend, scheduler=self.scheduler)
+		fetcher = None
 
 		if self.opts.getbinpkg and self._bintree.isremote(pkg.cpv):
+
+			fetcher = BinpkgFetcher(background=self.background,
+				logfile=self.settings.get("PORTAGE_LOG_FILE"), pkg=self.pkg,
+				pretend=self.opts.pretend, scheduler=self.scheduler)
+
 			msg = " --- (%s of %s) Fetching Binary (%s::%s)" %\
 				(pkg_count.curval, pkg_count.maxval, pkg.cpv,
 					fetcher.pkg_path)
@@ -160,7 +163,7 @@ class Binpkg(CompositeTask):
 
 		# The fetcher only has a returncode when
 		# --getbinpkg is enabled.
-		if fetcher.returncode is not None:
+		if fetcher is not None:
 			self._fetched_pkg = fetcher.pkg_path
 			if self._default_exit(fetcher) != os.EX_OK:
 				self._async_unlock_builddir(returncode=self.returncode)
@@ -209,7 +212,8 @@ class Binpkg(CompositeTask):
 
 		# This gives bashrc users an opportunity to do various things
 		# such as remove binary packages after they're installed.
-		self.settings["PORTAGE_BINPKG_FILE"] = pkg_path
+		if pkg_path is not None:
+			self.settings["PORTAGE_BINPKG_FILE"] = pkg_path
 		self._pkg_path = pkg_path
 
 		logfile = self.settings.get("PORTAGE_LOG_FILE")
@@ -245,6 +249,13 @@ class Binpkg(CompositeTask):
 			self._async_unlock_builddir(returncode=self.returncode)
 			return
 
+		self._start_task(
+			AsyncTaskFuture(future=self._unpack_metadata()),
+			self._unpack_metadata_exit)
+
+	@coroutine
+	def _unpack_metadata(self):
+
 		dir_path = self.settings['PORTAGE_BUILDDIR']
 
 		infloc = self._infloc
@@ -260,8 +271,7 @@ class Binpkg(CompositeTask):
 		portage.prepare_build_dirs(self.settings["ROOT"], self.settings, 1)
 		self._writemsg_level(">>> Extracting info\n")
 
-		pkg_xpak = portage.xpak.tbz2(self._pkg_path)
-		pkg_xpak.unpackinfo(infloc)
+		yield self._bintree.dbapi.unpack_metadata(self.settings, infloc)
 		check_missing_metadata = ("CATEGORY", "PF")
 		for k, v in zip(check_missing_metadata,
 			self._bintree.dbapi.aux_get(self.pkg.cpv, check_missing_metadata)):
@@ -295,11 +305,14 @@ class Binpkg(CompositeTask):
 
 		env_extractor = BinpkgEnvExtractor(background=self.background,
 			scheduler=self.scheduler, settings=self.settings)
-
-		self._start_task(env_extractor, self._env_extractor_exit)
-
-	def _env_extractor_exit(self, env_extractor):
-		if self._default_exit(env_extractor) != os.EX_OK:
+		env_extractor.start()
+		yield env_extractor.async_wait()
+		if env_extractor.returncode != os.EX_OK:
+			raise portage.exception.PortageException('failed to extract environment for {}'.format(self.pkg.cpv))
+
+	def _unpack_metadata_exit(self, unpack_metadata):
+		if self._default_exit(unpack_metadata) != os.EX_OK:
+			unpack_metadata.future.result()
 			self._async_unlock_builddir(returncode=self.returncode)
 			return
 
@@ -316,18 +329,16 @@ class Binpkg(CompositeTask):
 			self._async_unlock_builddir(returncode=self.returncode)
 			return
 
-		extractor = BinpkgExtractorAsync(background=self.background,
-			env=self.settings.environ(),
-			features=self.settings.features,
-			image_dir=self._image_dir,
-			pkg=self.pkg, pkg_path=self._pkg_path,
-			logfile=self.settings.get("PORTAGE_LOG_FILE"),
-			scheduler=self.scheduler)
 		self._writemsg_level(">>> Extracting %s\n" % self.pkg.cpv)
-		self._start_task(extractor, self._extractor_exit)
-
-	def _extractor_exit(self, extractor):
-		if self._default_exit(extractor) != os.EX_OK:
+		self._start_task(
+			AsyncTaskFuture(future=self._bintree.dbapi.unpack_contents(
+				self.settings,
+				self._image_dir)),
+			self._unpack_contents_exit)
+
+	def _unpack_contents_exit(self, unpack_contents):
+		if self._default_exit(unpack_contents) != os.EX_OK:
+			unpack_contents.future.result()
 			self._writemsg_level("!!! Error Extracting '%s'\n" % \
 				self._pkg_path, noiselevel=-1, level=logging.ERROR)
 			self._async_unlock_builddir(returncode=self.returncode)

diff --git a/lib/_emerge/Scheduler.py b/lib/_emerge/Scheduler.py
index af43a2e24..7fa3992e7 100644
--- a/lib/_emerge/Scheduler.py
+++ b/lib/_emerge/Scheduler.py
@@ -1,4 +1,4 @@
-# Copyright 1999-2014 Gentoo Foundation
+# Copyright 1999-2019 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 
 from __future__ import division, print_function, unicode_literals
@@ -868,10 +868,11 @@ class Scheduler(PollScheduler):
 
 					if fetched:
 						bintree.inject(x.cpv, filename=fetched)
-					tbz2_file = bintree.getname(x.cpv)
+
 					infloc = os.path.join(build_dir_path, "build-info")
 					ensure_dirs(infloc)
-					portage.xpak.tbz2(tbz2_file).unpackinfo(infloc)
+					self._sched_iface.run_until_complete(
+						bintree.dbapi.unpack_metadata(settings, infloc))
 					ebuild_path = os.path.join(infloc, x.pf + ".ebuild")
 					settings.configdict["pkg"]["EMERGE_FROM"] = "binary"
 					settings.configdict["pkg"]["MERGE_TYPE"] = "binary"

diff --git a/lib/_emerge/actions.py b/lib/_emerge/actions.py
index 705a3ff1c..6f815bff2 100644
--- a/lib/_emerge/actions.py
+++ b/lib/_emerge/actions.py
@@ -1,4 +1,4 @@
-# Copyright 1999-2018 Gentoo Foundation
+# Copyright 1999-2019 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 
 from __future__ import division, print_function, unicode_literals
@@ -122,6 +122,23 @@ def action_build(emerge_config, trees=DeprecationWarning,
 	# before we get here, so warn if they're not (bug #267103).
 	chk_updated_cfg_files(settings['EROOT'], ['/etc/portage'])
 
+	quickpkg_direct = ("--usepkg" in emerge_config.opts and
+		emerge_config.opts.get('--quickpkg-direct', 'n') == 'y' and
+		emerge_config.target_config is not emerge_config.running_config)
+	if '--getbinpkg' in emerge_config.opts or quickpkg_direct:
+		kwargs = {}
+		if quickpkg_direct:
+			kwargs['add_repos'] = (emerge_config.running_config.trees['vartree'].dbapi,)
+
+		try:
+			emerge_config.target_config.trees['bintree'].populate(
+				getbinpkgs='--getbinpkg' in emerge_config.opts,
+				**kwargs)
+		except ParseError as e:
+			writemsg("\n\n!!!%s.\nSee make.conf(5) for more info.\n"
+					 % e, noiselevel=-1)
+			return 1
+
 	# validate the state of the resume data
 	# so that we can make assumptions later.
 	for k in ("resume", "resume_backup"):
@@ -352,12 +369,17 @@ def action_build(emerge_config, trees=DeprecationWarning,
 			# instances need to load remote metadata if --getbinpkg
 			# is enabled. Use getbinpkg_refresh=False to use cached
 			# metadata, since the cache is already fresh.
-			if "--getbinpkg" in emerge_config.opts:
+			if "--getbinpkg" in emerge_config.opts or quickpkg_direct:
 				for root_trees in emerge_config.trees.values():
+					kwargs = {}
+					if quickpkg_direct:
+						kwargs['add_repos'] = (emerge_config.running_config.trees['vartree'].dbapi,)
+
 					try:
 						root_trees["bintree"].populate(
 							getbinpkgs=True,
-							getbinpkg_refresh=False)
+							getbinpkg_refresh=False,
+							**kwargs)
 					except ParseError as e:
 						writemsg("\n\n!!!%s.\nSee make.conf(5) for more info.\n"
 								 % e, noiselevel=-1)
@@ -2898,9 +2920,16 @@ def run_action(emerge_config):
 	if (emerge_config.action in ('search', None) and
 		'--usepkg' in emerge_config.opts):
 		for mytrees in emerge_config.trees.values():
+			kwargs = {}
+			if (mytrees is emerge_config.target_config.trees and
+				emerge_config.target_config is not emerge_config.running_config and
+				emerge_config.opts.get('--quickpkg-direct', 'n') == 'y'):
+				kwargs['add_repos'] = (emerge_config.running_config.trees['vartree'].dbapi,)
+
 			try:
 				mytrees['bintree'].populate(
-					getbinpkgs='--getbinpkg' in emerge_config.opts)
+					getbinpkgs='--getbinpkg' in emerge_config.opts,
+					**kwargs)
 			except ParseError as e:
 				writemsg('\n\n!!!%s.\nSee make.conf(5) for more info.\n'
 						 % (e,), noiselevel=-1)

diff --git a/lib/_emerge/depgraph.py b/lib/_emerge/depgraph.py
index 1127a6234..6d8e73172 100644
--- a/lib/_emerge/depgraph.py
+++ b/lib/_emerge/depgraph.py
@@ -495,6 +495,7 @@ class _dynamic_depgraph_config(object):
 		self._backtrack_infos = {}
 
 		self._buildpkgonly_deps_unsatisfied = False
+		self._quickpkg_direct_deps_unsatisfied = False
 		self._autounmask = self.myparams['autounmask']
 		self._displayed_autounmask = False
 		self._success_without_autounmask = False
@@ -4526,6 +4527,16 @@ class depgraph(object):
 				self._dynamic_config._skip_restart = True
 				return False, myfavorites
 
+		if (self._frozen_config.myopts.get('--quickpkg-direct', 'n') == 'y' and
+			self._frozen_config.target_root is not self._frozen_config._running_root):
+			running_root = self._frozen_config._running_root.root
+			for node in self._dynamic_config.digraph:
+				if (isinstance(node, Package) and node.operation in ('merge', 'uninstall') and
+					node.root == running_root):
+					self._dynamic_config._quickpkg_direct_deps_unsatisfied = True
+					self._dynamic_config._skip_restart = True
+					return False, myfavorites
+
 		if (not self._dynamic_config._prune_rebuilds and
 			self._ignored_binaries_autounmask_backtrack()):
 			config = self._dynamic_config._backtrack_infos.setdefault("config", {})
@@ -9062,6 +9073,14 @@ class depgraph(object):
 			writemsg("!!! Cannot merge requested packages. "
 				"Merge deps and try again.\n\n", noiselevel=-1)
 
+		if self._dynamic_config._quickpkg_direct_deps_unsatisfied:
+			self._show_merge_list()
+			writemsg("\n!!! --quickpkg-direct requires all "
+				"dependencies to be merged for root '{}'.\n".format(
+				self._frozen_config._running_root.root), noiselevel=-1)
+			writemsg("!!! Cannot merge requested packages. "
+				"Merge deps and try again.\n\n", noiselevel=-1)
+
 	def saveNomergeFavorites(self):
 		"""Find atoms in favorites that are not in the mergelist and add them
 		to the world file if necessary."""

diff --git a/lib/_emerge/main.py b/lib/_emerge/main.py
index 0d2c45a4f..8c72cdf9c 100644
--- a/lib/_emerge/main.py
+++ b/lib/_emerge/main.py
@@ -1,4 +1,4 @@
-# Copyright 1999-2018 Gentoo Foundation
+# Copyright 1999-2019 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 
 from __future__ import print_function
@@ -637,6 +637,11 @@ def parse_opts(tmpcmdline, silent=False):
 			"action"   : "store",
 		},
 
+		"--quickpkg-direct": {
+			"help": "Enable use of installed packages directly as binary packages",
+			"choices": y_or_n
+		},
+
 		"--quiet": {
 			"shortopt" : "-q",
 			"help"     : "reduced or condensed output",

diff --git a/lib/portage/dbapi/__init__.py b/lib/portage/dbapi/__init__.py
index 80f8a689f..37728714e 100644
--- a/lib/portage/dbapi/__init__.py
+++ b/lib/portage/dbapi/__init__.py
@@ -1,4 +1,4 @@
-# Copyright 1998-2018 Gentoo Foundation
+# Copyright 1998-2019 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 
 from __future__ import unicode_literals
@@ -32,8 +32,7 @@ class dbapi(object):
 	_use_mutable = False
 	_known_keys = frozenset(x for x in auxdbkeys
 		if not x.startswith("UNUSED_0"))
-	_pkg_str_aux_keys = ("BUILD_TIME", "EAPI", "BUILD_ID",
-		"KEYWORDS", "SLOT", "repository")
+	_pkg_str_aux_keys = ("EAPI", "KEYWORDS", "SLOT", "repository")
 
 	def __init__(self):
 		pass

diff --git a/lib/portage/dbapi/bintree.py b/lib/portage/dbapi/bintree.py
index ba21e6d23..9d3ea039b 100644
--- a/lib/portage/dbapi/bintree.py
+++ b/lib/portage/dbapi/bintree.py
@@ -1,4 +1,4 @@
-# Copyright 1998-2018 Gentoo Foundation
+# Copyright 1998-2019 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 
 from __future__ import unicode_literals
@@ -7,6 +7,7 @@ __all__ = ["bindbapi", "binarytree"]
 
 import portage
 portage.proxy.lazyimport.lazyimport(globals(),
+	'_emerge.BinpkgExtractorAsync:BinpkgExtractorAsync',
 	'portage.checksum:get_valid_checksum_keys,perform_multiple_checksums,' + \
 		'verify_all,_apply_hash_filter,_hash_filter',
 	'portage.dbapi.dep_expand:dep_expand',
@@ -18,6 +19,7 @@ portage.proxy.lazyimport.lazyimport(globals(),
 	'portage.util:atomic_ofstream,ensure_dirs,normalize_path,' + \
 		'writemsg,writemsg_stdout',
 	'portage.util.path:first_existing',
+	'portage.util._async.SchedulerInterface:SchedulerInterface',
 	'portage.util._urlopen:urlopen@_urlopen,have_pep_476@_have_pep_476',
 	'portage.versions:best,catpkgsplit,catsplit,_pkg_str',
 )
@@ -30,6 +32,9 @@ from portage.exception import AlarmSignal, InvalidData, InvalidPackageName, \
 	ParseError, PermissionDenied, PortageException
 from portage.localization import _
 from portage.package.ebuild.profile_iuse import iter_iuse_vars
+from portage.util.futures import asyncio
+from portage.util.futures.compat_coroutine import coroutine
+from portage.util.futures.executor.fork import ForkExecutor
 from portage import _movefile
 from portage import os
 from portage import _encodings
@@ -70,6 +75,8 @@ class UseCachedCopyOfRemoteIndex(Exception):
 class bindbapi(fakedbapi):
 	_known_keys = frozenset(list(fakedbapi._known_keys) + \
 		["CHOST", "repository", "USE"])
+	_pkg_str_aux_keys = fakedbapi._pkg_str_aux_keys + ("BUILD_ID", "BUILD_TIME", "_mtime_")
+
 	def __init__(self, mybintree=None, **kwargs):
 		# Always enable multi_instance mode for bindbapi indexing. This
 		# does not affect the local PKGDIR file layout, since that is
@@ -142,7 +149,10 @@ class bindbapi(fakedbapi):
 				return [aux_cache.get(x, "") for x in wants]
 		mysplit = mycpv.split("/")
 		mylist = []
-		if not self.bintree._remotepkgs or \
+		add_pkg = self.bintree._additional_pkgs.get(instance_key)
+		if add_pkg is not None:
+			return add_pkg._db.aux_get(add_pkg, wants)
+		elif not self.bintree._remotepkgs or \
 			not self.bintree.isremote(mycpv):
 			try:
 				tbz2_path = self.bintree._pkg_paths[instance_key]
@@ -218,6 +228,73 @@ class bindbapi(fakedbapi):
 		# inject will clear stale caches via cpv_inject.
 		self.bintree.inject(cpv, filename=tbz2path)
 
+
+	@coroutine
+	def unpack_metadata(self, pkg, dest_dir):
+		"""
+		Unpack package metadata to a directory. This method is a coroutine.
+
+		@param pkg: package to unpack
+		@type pkg: _pkg_str or portage.config
+		@param dest_dir: destination directory
+		@type dest_dir: str
+		"""
+		loop = asyncio._wrap_loop()
+		if isinstance(pkg, _pkg_str):
+			cpv = pkg
+		else:
+			cpv = pkg.mycpv
+		key = self._instance_key(cpv)
+		add_pkg = self.bintree._additional_pkgs.get(key)
+		if add_pkg is not None:
+			yield add_pkg._db.unpack_metadata(pkg, dest_dir)
+		else:
+			tbz2_file = self.bintree.getname(cpv)
+			yield loop.run_in_executor(ForkExecutor(loop=loop),
+				portage.xpak.tbz2(tbz2_file).unpackinfo, dest_dir)
+
+	@coroutine
+	def unpack_contents(self, pkg, dest_dir):
+		"""
+		Unpack package contents to a directory. This method is a coroutine.
+
+		@param pkg: package to unpack
+		@type pkg: _pkg_str or portage.config
+		@param dest_dir: destination directory
+		@type dest_dir: str
+		"""
+		loop = asyncio._wrap_loop()
+		if isinstance(pkg, _pkg_str):
+			settings = self.settings
+			cpv = pkg
+		else:
+			settings = pkg
+			cpv = settings.mycpv
+
+		pkg_path = self.bintree.getname(cpv)
+		if pkg_path is not None:
+
+			extractor = BinpkgExtractorAsync(
+				background=settings.get('PORTAGE_BACKGROUND') == '1',
+				env=settings.environ(),
+				features=settings.features,
+				image_dir=dest_dir,
+				pkg=cpv, pkg_path=pkg_path,
+				logfile=settings.get('PORTAGE_LOG_FILE'),
+				scheduler=SchedulerInterface(loop))
+
+			extractor.start()
+			yield extractor.async_wait()
+			if extractor.returncode != os.EX_OK:
+				raise PortageException("Error Extracting '{}'".format(pkg_path))
+
+		else:
+			instance_key = self._instance_key(cpv)
+			add_pkg = self.bintree._additional_pkgs.get(instance_key)
+			if add_pkg is None:
+				raise portage.exception.PackageNotFound(cpv)
+			yield add_pkg._db.unpack_contents(pkg, dest_dir)
+
 	def cp_list(self, *pargs, **kwargs):
 		if not self.bintree.populated:
 			self.bintree.populate()
@@ -261,6 +338,7 @@ class bindbapi(fakedbapi):
 
 		return filesdict
 
+
 class binarytree(object):
 	"this tree scans for a list of all packages available in PKGDIR"
 	def __init__(self, _unused=DeprecationWarning, pkgdir=None,
@@ -301,6 +379,7 @@ class binarytree(object):
 			self.tree = {}
 			self._remote_has_index = False
 			self._remotepkgs = None # remote metadata indexed by cpv
+			self._additional_pkgs = {}
 			self.invalids = []
 			self.settings = settings
 			self._pkg_paths = {}
@@ -511,7 +590,7 @@ class binarytree(object):
 			except PortageException:
 				pass
 
-	def populate(self, getbinpkgs=False, getbinpkg_refresh=True):
+	def populate(self, getbinpkgs=False, getbinpkg_refresh=True, add_repos=()):
 		"""
 		Populates the binarytree with package metadata.
 
@@ -520,12 +599,14 @@ class binarytree(object):
 		@param getbinpkg_refresh: attempt to refresh the cache
 			of remote package metadata if getbinpkgs is also True
 		@type getbinpkg_refresh: bool
+		@param add_repos: additional binary package repositories
+		@type add_repos: sequence
 		"""
 
 		if self._populating:
 			return
 
-		if not os.path.isdir(self.pkgdir) and not getbinpkgs:
+		if not os.path.isdir(self.pkgdir) and not (getbinpkgs or add_repos):
 			self.populated = True
 			return
 
@@ -557,6 +638,9 @@ class binarytree(object):
 					if pkgindex_lock:
 						unlockfile(pkgindex_lock)
 
+			if add_repos:
+				self._populate_additional(add_repos)
+
 			if getbinpkgs:
 				if not self.settings.get("PORTAGE_BINHOST"):
 					writemsg(_("!!! PORTAGE_BINHOST unset, but use is requested.\n"),
@@ -1066,6 +1150,16 @@ class binarytree(object):
 				self._merge_pkgindex_header(pkgindex.header,
 					self._pkgindex_header)
 
+	def _populate_additional(self, repos):
+		for repo in repos:
+			aux_keys = list(set(chain(repo._aux_cache_keys, repo._pkg_str_aux_keys)))
+			for cpv in repo.cpv_all():
+				metadata = dict(zip(aux_keys, repo.aux_get(cpv, aux_keys)))
+				pkg = _pkg_str(cpv, metadata=metadata, settings=repo.settings, db=repo)
+				instance_key = self.dbapi._instance_key(pkg)
+				self._additional_pkgs[instance_key] = pkg
+				self.dbapi.cpv_inject(pkg)
+
 	def inject(self, cpv, filename=None):
 		"""Add a freshly built package to the database.  This updates
 		$PKGDIR/Packages with the new package metadata (including MD5).
@@ -1500,6 +1594,8 @@ class binarytree(object):
 				filename = self._pkg_paths.get(instance_key)
 				if filename is not None:
 					filename = os.path.join(self.pkgdir, filename)
+				elif instance_key in self._additional_pkgs:
+					return None
 
 		if filename is None:
 			if self._multi_instance:
@@ -1570,8 +1666,12 @@ class binarytree(object):
 	def isremote(self, pkgname):
 		"""Returns true if the package is kept remotely and it has not been
 		downloaded (or it is only partially downloaded)."""
-		if (self._remotepkgs is None or
-		self.dbapi._instance_key(pkgname) not in self._remotepkgs):
+		if self._remotepkgs is None:
+			return False
+		instance_key = self.dbapi._instance_key(pkgname)
+		if instance_key not in self._remotepkgs:
+			return False
+		elif instance_key in self._additional_pkgs:
 			return False
 		# Presence in self._remotepkgs implies that it's remote. When a
 		# package is downloaded, state is updated by self.inject().

diff --git a/lib/portage/dbapi/vartree.py b/lib/portage/dbapi/vartree.py
index 603d58015..039e520d5 100644
--- a/lib/portage/dbapi/vartree.py
+++ b/lib/portage/dbapi/vartree.py
@@ -33,7 +33,7 @@ portage.proxy.lazyimport.lazyimport(globals(),
 	'portage.util._compare_files:compare_files',
 	'portage.util.digraph:digraph',
 	'portage.util.env_update:env_update',
-	'portage.util.install_mask:install_mask_dir,InstallMask',
+	'portage.util.install_mask:install_mask_dir,InstallMask,_raise_exc',
 	'portage.util.listdir:dircache,listdir',
 	'portage.util.movefile:movefile',
 	'portage.util.monotonic:monotonic',
@@ -71,6 +71,8 @@ from portage import _os_merge
 from portage import _selinux_merge
 from portage import _unicode_decode
 from portage import _unicode_encode
+from portage.util.futures.compat_coroutine import coroutine
+from portage.util.futures.executor.fork import ForkExecutor
 from ._VdbMetadataDelta import VdbMetadataDelta
 
 from _emerge.EbuildBuildDir import EbuildBuildDir
@@ -80,8 +82,10 @@ from _emerge.MiscFunctionsProcess import MiscFunctionsProcess
 from _emerge.SpawnProcess import SpawnProcess
 from ._ContentsCaseSensitivityManager import ContentsCaseSensitivityManager
 
+import argparse
 import errno
 import fnmatch
+import functools
 import gc
 import grp
 import io
@@ -128,6 +132,7 @@ class vardbapi(dbapi):
 
 	_aux_cache_keys_re = re.compile(r'^NEEDED\..*$')
 	_aux_multi_line_re = re.compile(r'^(CONTENTS|NEEDED\..*)$')
+	_pkg_str_aux_keys = dbapi._pkg_str_aux_keys + ("BUILD_ID", "BUILD_TIME", "_mtime_")
 
 	def __init__(self, _unused_param=DeprecationWarning,
 		categories=None, settings=None, vartree=None):
@@ -953,6 +958,110 @@ class vardbapi(dbapi):
 					pass
 		self._bump_mtime(cpv)
 
+	@coroutine
+	def unpack_metadata(self, pkg, dest_dir):
+		"""
+		Unpack package metadata to a directory. This method is a coroutine.
+
+		@param pkg: package to unpack
+		@type pkg: _pkg_str or portage.config
+		@param dest_dir: destination directory
+		@type dest_dir: str
+		"""
+		loop = asyncio._wrap_loop()
+		if not isinstance(pkg, portage.config):
+			cpv = pkg
+		else:
+			cpv = pkg.mycpv
+		dbdir = self.getpath(cpv)
+		def async_copy():
+			for parent, dirs, files in os.walk(dbdir, onerror=_raise_exc):
+				for key in files:
+					shutil.copy(os.path.join(parent, key),
+						os.path.join(dest_dir, key))
+				break
+		yield loop.run_in_executor(ForkExecutor(loop=loop), async_copy)
+
+	@coroutine
+	def unpack_contents(self, pkg, dest_dir,
+		include_config=None, include_unmodified_config=None):
+		"""
+		Unpack package contents to a directory. This method is a coroutine.
+
+		This copies files from the installed system, in the same way
+		as the quickpkg(1) command. Default behavior for handling
+		of protected configuration files is controlled by the
+		QUICKPKG_DEFAULT_OPTS variable. The relevant quickpkg options
+		are --include-config and --include-unmodified-config. When
+		a configuration file is not included because it is protected,
+		an ewarn message is logged.
+
+		@param pkg: package to unpack
+		@type pkg: _pkg_str or portage.config
+		@param dest_dir: destination directory
+		@type dest_dir: str
+		@param include_config: Include all files protected by
+			CONFIG_PROTECT (as a security precaution, default is False
+			unless modified by QUICKPKG_DEFAULT_OPTS).
+		@type include_config: bool
+		@param include_unmodified_config: Include files protected by
+			CONFIG_PROTECT that have not been modified since installation
+			(as a security precaution, default is False unless modified
+			by QUICKPKG_DEFAULT_OPTS).
+		@type include_unmodified_config: bool
+		"""
+		loop = asyncio._wrap_loop()
+		if not isinstance(pkg, portage.config):
+			settings = self.settings
+			cpv = pkg
+		else:
+			settings = pkg
+			cpv = settings.mycpv
+
+		scheduler = SchedulerInterface(loop)
+		parser = argparse.ArgumentParser()
+		parser.add_argument('--include-config',
+			choices=('y', 'n'),
+			default='n')
+		parser.add_argument('--include-unmodified-config',
+			choices=('y', 'n'),
+			default='n')
+
+		# Method parameters may override QUICKPKG_DEFAULT_OPTS.
+		opts_list = portage.util.shlex_split(settings.get('QUICKPKG_DEFAULT_OPTS', ''))
+		if include_config is not None:
+			opts_list.append('--include-config={}'.format(
+				'y' if include_config else 'n'))
+		if include_unmodified_config is not None:
+			opts_list.append('--include-unmodified-config={}'.format(
+				'y' if include_unmodified_config else 'n'))
+
+		opts, args = parser.parse_known_args(opts_list)
+
+		tar_cmd = ('tar', '-x', '--xattrs', '--xattrs-include=*', '-C', dest_dir)
+		pr, pw = os.pipe()
+		proc = (yield asyncio.create_subprocess_exec(*tar_cmd, stdin=pr))
+		os.close(pr)
+		with os.fdopen(pw, 'wb', 0) as pw_file:
+			excluded_config_files = (yield loop.run_in_executor(ForkExecutor(loop=loop),
+				functools.partial(self._dblink(cpv).quickpkg,
+				pw_file,
+				include_config=opts.include_config == 'y',
+				include_unmodified_config=opts.include_unmodified_config == 'y')))
+		yield proc.wait()
+		if proc.returncode != os.EX_OK:
+			raise PortageException('command failed: {}'.format(tar_cmd))
+
+		if excluded_config_files:
+			log_lines = ([_("Config files excluded by QUICKPKG_DEFAULT_OPTS (see quickpkg(1) man page):")] +
+				['\t{}'.format(name) for name in excluded_config_files])
+			out = io.StringIO()
+			for line in log_lines:
+				portage.elog.messages.ewarn(line, phase='install', key=cpv, out=out)
+			scheduler.output(out.getvalue(),
+				background=self.settings.get("PORTAGE_BACKGROUND") == "1",
+				log_path=settings.get("PORTAGE_LOG_FILE"))
+
 	def counter_tick(self, myroot=None, mycpv=None):
 		"""
 		@param myroot: ignored, self._eroot is used instead
@@ -1892,10 +2001,10 @@ class dblink(object):
 		@param include_config: Include all files protected by CONFIG_PROTECT
 			(as a security precaution, default is False).
 		@type include_config: bool
-		@param include_config: Include files protected by CONFIG_PROTECT that
-			have not been modified since installation (as a security precaution,
+		@param include_unmodified_config: Include files protected by CONFIG_PROTECT
+			that have not been modified since installation (as a security precaution,
 			default is False).
-		@type include_config: bool
+		@type include_unmodified_config: bool
 		@rtype: list
 		@return: Paths of protected configuration files which have been omitted.
 		"""

diff --git a/lib/portage/tests/emerge/test_simple.py b/lib/portage/tests/emerge/test_simple.py
index 866521488..f5cd6f3d2 100644
--- a/lib/portage/tests/emerge/test_simple.py
+++ b/lib/portage/tests/emerge/test_simple.py
@@ -1,4 +1,4 @@
-# Copyright 2011-2018 Gentoo Foundation
+# Copyright 2011-2019 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 
 import subprocess
@@ -254,6 +254,7 @@ call_has_and_best_version() {
 		cross_eroot = os.path.join(cross_root, eprefix.lstrip(os.sep))
 
 		test_commands = (
+			emerge_cmd + ("--usepkgonly", "--root", cross_root, "--quickpkg-direct=y", "dev-libs/A"),
 			env_update_cmd,
 			portageq_cmd + ("envvar", "-v", "CONFIG_PROTECT", "EROOT",
 				"PORTAGE_CONFIGROOT", "PORTAGE_TMPDIR", "USERLAND"),

diff --git a/man/emerge.1 b/man/emerge.1
index 8d3e74074..8238515a6 100644
--- a/man/emerge.1
+++ b/man/emerge.1
@@ -1,4 +1,4 @@
-.TH "EMERGE" "1" "Jun 2019" "Portage VERSION" "Portage"
+.TH "EMERGE" "1" "Nov 2019" "Portage VERSION" "Portage"
 .SH "NAME"
 emerge \- Command\-line interface to the Portage system
 .SH "SYNOPSIS"
@@ -829,6 +829,23 @@ B	blocked by another package (unresolved conflict)
 b	blocked by another package (automatically resolved conflict)
 .TE
 .TP
+.BR "\-\-quickpkg\-direct < y | n >"
+Enable use of installed packages directly as binary packages. This is
+similar to using binary packages produced by \fBquickpkg\fR(1), but
+installed packages are used directly as though they are binary packages.
+This option only works in combination with the \fB\-\-root=DIR\fR option,
+and it comes with the caveat that packages are only allowed to be
+installed into the root that is specified by the \fB\-\-root=DIR\fR
+option (the other root which serves as a source of packages is
+assumed to be immutable during the entire operation).
+
+Default behavior for handling of protected configuration files is
+controlled by the \fBQUICKPKG_DEFAULT_OPTS\fR variable. The relevant
+quickpkg options are \fI\-\-include\-config\fR and
+\fI\-\-include\-unmodified\-config\fR (refer to the \fBquickpkg\fR(1)
+man page). When a configuration file is not included because it is
+protected, an ewarn message is logged.
+.TP
 .BR "\-\-quiet [ y | n ]" ", " \-q
 Results may vary, but the general outcome is a reduced or condensed
 output from portage's displays.