1 |
Enable use of installed packages directly as binary |
2 |
packages. This is similar to using binary packages produced by |
3 |
quickpkg(1), but installed packages are used directly as though |
4 |
they are binary packages. This option only works in combination |
5 |
with the --root=DIR option, and it comes with the caveat that |
6 |
packages are only allowed to be installed into the root that |
7 |
is specified by the --root=DIR option. The other root which |
8 |
serves as a source of packages is assumed to be immutabled |
9 |
during the entire operation (similar to --buildpkgonly mode). |
10 |
|
11 |
Default behavior for handling of protected configuration files |
12 |
is controlled by the QUICKPKG_DEFAULT_OPTS variable. When a |
13 |
configuration file is not included because it is protected, an |
14 |
ewarn message is logged. |
15 |
|
16 |
Suggested use cases: |
17 |
|
18 |
* Install packages from a buildtime container into an empty root, |
19 |
in order to create a minimal runtime container (which need not |
20 |
include a package manager). In a multi-stage Dockerfile, install |
21 |
runtime files to an empty directory in the build stage, and in |
22 |
the final stage use COPY to populate a container with the |
23 |
contents of that directory. For greater efficiency, use buildah |
24 |
to install directly into a mounted container, avoiding the COPY |
25 |
step. Use the emerge --usepkgonly and --ignore-soname-deps=n |
26 |
options to account for soname dependencies, allowing implicit |
27 |
system dependencies such as glibc to be automatically pulled |
28 |
into the runtime image. |
29 |
|
30 |
* Enable a live usb, iso, or pxe image to act as a binary |
31 |
installer that uses packages installed in the live image as a |
32 |
source of binary packages. |
33 |
|
34 |
Bug: https://bugs.gentoo.org/699986 |
35 |
Signed-off-by: Zac Medico <zmedico@g.o> |
36 |
--- |
37 |
lib/_emerge/Binpkg.py | 59 +++++++------ |
38 |
lib/_emerge/Scheduler.py | 5 +- |
39 |
lib/_emerge/actions.py | 34 +++++++- |
40 |
lib/_emerge/depgraph.py | 19 ++++ |
41 |
lib/_emerge/main.py | 5 ++ |
42 |
lib/portage/dbapi/__init__.py | 3 +- |
43 |
lib/portage/dbapi/bintree.py | 110 ++++++++++++++++++++++-- |
44 |
lib/portage/dbapi/vartree.py | 110 +++++++++++++++++++++++- |
45 |
lib/portage/tests/emerge/test_simple.py | 1 + |
46 |
man/emerge.1 | 17 ++++ |
47 |
10 files changed, 326 insertions(+), 37 deletions(-) |
48 |
|
49 |
diff --git a/lib/_emerge/Binpkg.py b/lib/_emerge/Binpkg.py |
50 |
index f9cffa26d..b5a69f8e7 100644 |
51 |
--- a/lib/_emerge/Binpkg.py |
52 |
+++ b/lib/_emerge/Binpkg.py |
53 |
@@ -7,7 +7,6 @@ import _emerge.emergelog |
54 |
from _emerge.EbuildPhase import EbuildPhase |
55 |
from _emerge.BinpkgFetcher import BinpkgFetcher |
56 |
from _emerge.BinpkgEnvExtractor import BinpkgEnvExtractor |
57 |
-from _emerge.BinpkgExtractorAsync import BinpkgExtractorAsync |
58 |
from _emerge.CompositeTask import CompositeTask |
59 |
from _emerge.BinpkgVerifier import BinpkgVerifier |
60 |
from _emerge.EbuildMerge import EbuildMerge |
61 |
@@ -16,6 +15,7 @@ from _emerge.SpawnProcess import SpawnProcess |
62 |
from portage.eapi import eapi_exports_replace_vars |
63 |
from portage.util import ensure_dirs |
64 |
from portage.util._async.AsyncTaskFuture import AsyncTaskFuture |
65 |
+from portage.util.futures.compat_coroutine import coroutine |
66 |
import portage |
67 |
from portage import os |
68 |
from portage import shutil |
69 |
@@ -135,11 +135,14 @@ class Binpkg(CompositeTask): |
70 |
|
71 |
pkg = self.pkg |
72 |
pkg_count = self.pkg_count |
73 |
- fetcher = BinpkgFetcher(background=self.background, |
74 |
- logfile=self.settings.get("PORTAGE_LOG_FILE"), pkg=self.pkg, |
75 |
- pretend=self.opts.pretend, scheduler=self.scheduler) |
76 |
+ fetcher = None |
77 |
|
78 |
if self.opts.getbinpkg and self._bintree.isremote(pkg.cpv): |
79 |
+ |
80 |
+ fetcher = BinpkgFetcher(background=self.background, |
81 |
+ logfile=self.settings.get("PORTAGE_LOG_FILE"), pkg=self.pkg, |
82 |
+ pretend=self.opts.pretend, scheduler=self.scheduler) |
83 |
+ |
84 |
msg = " --- (%s of %s) Fetching Binary (%s::%s)" %\ |
85 |
(pkg_count.curval, pkg_count.maxval, pkg.cpv, |
86 |
fetcher.pkg_path) |
87 |
@@ -160,7 +163,7 @@ class Binpkg(CompositeTask): |
88 |
|
89 |
# The fetcher only has a returncode when |
90 |
# --getbinpkg is enabled. |
91 |
- if fetcher.returncode is not None: |
92 |
+ if fetcher is not None: |
93 |
self._fetched_pkg = fetcher.pkg_path |
94 |
if self._default_exit(fetcher) != os.EX_OK: |
95 |
self._async_unlock_builddir(returncode=self.returncode) |
96 |
@@ -209,7 +212,8 @@ class Binpkg(CompositeTask): |
97 |
|
98 |
# This gives bashrc users an opportunity to do various things |
99 |
# such as remove binary packages after they're installed. |
100 |
- self.settings["PORTAGE_BINPKG_FILE"] = pkg_path |
101 |
+ if pkg_path is not None: |
102 |
+ self.settings["PORTAGE_BINPKG_FILE"] = pkg_path |
103 |
self._pkg_path = pkg_path |
104 |
|
105 |
logfile = self.settings.get("PORTAGE_LOG_FILE") |
106 |
@@ -245,6 +249,13 @@ class Binpkg(CompositeTask): |
107 |
self._async_unlock_builddir(returncode=self.returncode) |
108 |
return |
109 |
|
110 |
+ self._start_task( |
111 |
+ AsyncTaskFuture(future=self._unpack_metadata()), |
112 |
+ self._unpack_metadata_exit) |
113 |
+ |
114 |
+ @coroutine |
115 |
+ def _unpack_metadata(self): |
116 |
+ |
117 |
dir_path = self.settings['PORTAGE_BUILDDIR'] |
118 |
|
119 |
infloc = self._infloc |
120 |
@@ -260,8 +271,7 @@ class Binpkg(CompositeTask): |
121 |
portage.prepare_build_dirs(self.settings["ROOT"], self.settings, 1) |
122 |
self._writemsg_level(">>> Extracting info\n") |
123 |
|
124 |
- pkg_xpak = portage.xpak.tbz2(self._pkg_path) |
125 |
- pkg_xpak.unpackinfo(infloc) |
126 |
+ yield self._bintree.dbapi.unpack_metadata(self.settings, infloc) |
127 |
check_missing_metadata = ("CATEGORY", "PF") |
128 |
for k, v in zip(check_missing_metadata, |
129 |
self._bintree.dbapi.aux_get(self.pkg.cpv, check_missing_metadata)): |
130 |
@@ -295,11 +305,14 @@ class Binpkg(CompositeTask): |
131 |
|
132 |
env_extractor = BinpkgEnvExtractor(background=self.background, |
133 |
scheduler=self.scheduler, settings=self.settings) |
134 |
- |
135 |
- self._start_task(env_extractor, self._env_extractor_exit) |
136 |
- |
137 |
- def _env_extractor_exit(self, env_extractor): |
138 |
- if self._default_exit(env_extractor) != os.EX_OK: |
139 |
+ env_extractor.start() |
140 |
+ yield env_extractor.async_wait() |
141 |
+ if env_extractor.returncode != os.EX_OK: |
142 |
+ raise portage.exception.PortageException('failed to extract environment for {}'.format(self.pkg.cpv)) |
143 |
+ |
144 |
+ def _unpack_metadata_exit(self, unpack_metadata): |
145 |
+ if self._default_exit(unpack_metadata) != os.EX_OK: |
146 |
+ unpack_metadata.future.result() |
147 |
self._async_unlock_builddir(returncode=self.returncode) |
148 |
return |
149 |
|
150 |
@@ -316,18 +329,16 @@ class Binpkg(CompositeTask): |
151 |
self._async_unlock_builddir(returncode=self.returncode) |
152 |
return |
153 |
|
154 |
- extractor = BinpkgExtractorAsync(background=self.background, |
155 |
- env=self.settings.environ(), |
156 |
- features=self.settings.features, |
157 |
- image_dir=self._image_dir, |
158 |
- pkg=self.pkg, pkg_path=self._pkg_path, |
159 |
- logfile=self.settings.get("PORTAGE_LOG_FILE"), |
160 |
- scheduler=self.scheduler) |
161 |
self._writemsg_level(">>> Extracting %s\n" % self.pkg.cpv) |
162 |
- self._start_task(extractor, self._extractor_exit) |
163 |
- |
164 |
- def _extractor_exit(self, extractor): |
165 |
- if self._default_exit(extractor) != os.EX_OK: |
166 |
+ self._start_task( |
167 |
+ AsyncTaskFuture(future=self._bintree.dbapi.unpack_contents( |
168 |
+ self.settings, |
169 |
+ self._image_dir)), |
170 |
+ self._unpack_contents_exit) |
171 |
+ |
172 |
+ def _unpack_contents_exit(self, unpack_contents): |
173 |
+ if self._default_exit(unpack_contents) != os.EX_OK: |
174 |
+ unpack_contents.future.result() |
175 |
self._writemsg_level("!!! Error Extracting '%s'\n" % \ |
176 |
self._pkg_path, noiselevel=-1, level=logging.ERROR) |
177 |
self._async_unlock_builddir(returncode=self.returncode) |
178 |
diff --git a/lib/_emerge/Scheduler.py b/lib/_emerge/Scheduler.py |
179 |
index af43a2e24..cb0de9fca 100644 |
180 |
--- a/lib/_emerge/Scheduler.py |
181 |
+++ b/lib/_emerge/Scheduler.py |
182 |
@@ -868,10 +868,11 @@ class Scheduler(PollScheduler): |
183 |
|
184 |
if fetched: |
185 |
bintree.inject(x.cpv, filename=fetched) |
186 |
- tbz2_file = bintree.getname(x.cpv) |
187 |
+ |
188 |
infloc = os.path.join(build_dir_path, "build-info") |
189 |
ensure_dirs(infloc) |
190 |
- portage.xpak.tbz2(tbz2_file).unpackinfo(infloc) |
191 |
+ self._sched_iface.run_until_complete( |
192 |
+ bintree.dbapi.unpack_metadata(settings, infloc)) |
193 |
ebuild_path = os.path.join(infloc, x.pf + ".ebuild") |
194 |
settings.configdict["pkg"]["EMERGE_FROM"] = "binary" |
195 |
settings.configdict["pkg"]["MERGE_TYPE"] = "binary" |
196 |
diff --git a/lib/_emerge/actions.py b/lib/_emerge/actions.py |
197 |
index 705a3ff1c..01c00cef3 100644 |
198 |
--- a/lib/_emerge/actions.py |
199 |
+++ b/lib/_emerge/actions.py |
200 |
@@ -122,6 +122,22 @@ def action_build(emerge_config, trees=DeprecationWarning, |
201 |
# before we get here, so warn if they're not (bug #267103). |
202 |
chk_updated_cfg_files(settings['EROOT'], ['/etc/portage']) |
203 |
|
204 |
+ quickpkg_direct = (emerge_config.opts.get('--quickpkg-direct', 'n') == 'y' and |
205 |
+ emerge_config.target_config is not emerge_config.running_config) |
206 |
+ if '--getbinpkg' in emerge_config.opts or quickpkg_direct: |
207 |
+ kwargs = {} |
208 |
+ if quickpkg_direct: |
209 |
+ kwargs['add_repos'] = (emerge_config.running_config.trees['vartree'].dbapi,) |
210 |
+ |
211 |
+ try: |
212 |
+ emerge_config.target_config.trees['bintree'].populate( |
213 |
+ getbinpkgs='--getbinpkg' in emerge_config.opts, |
214 |
+ **kwargs) |
215 |
+ except ParseError as e: |
216 |
+ writemsg("\n\n!!!%s.\nSee make.conf(5) for more info.\n" |
217 |
+ % e, noiselevel=-1) |
218 |
+ return 1 |
219 |
+ |
220 |
# validate the state of the resume data |
221 |
# so that we can make assumptions later. |
222 |
for k in ("resume", "resume_backup"): |
223 |
@@ -352,12 +368,17 @@ def action_build(emerge_config, trees=DeprecationWarning, |
224 |
# instances need to load remote metadata if --getbinpkg |
225 |
# is enabled. Use getbinpkg_refresh=False to use cached |
226 |
# metadata, since the cache is already fresh. |
227 |
- if "--getbinpkg" in emerge_config.opts: |
228 |
+ if "--getbinpkg" in emerge_config.opts or quickpkg_direct: |
229 |
for root_trees in emerge_config.trees.values(): |
230 |
+ kwargs = {} |
231 |
+ if quickpkg_direct: |
232 |
+ kwargs['add_repos'] = (emerge_config.running_config.trees['vartree'].dbapi,) |
233 |
+ |
234 |
try: |
235 |
root_trees["bintree"].populate( |
236 |
getbinpkgs=True, |
237 |
- getbinpkg_refresh=False) |
238 |
+ getbinpkg_refresh=False, |
239 |
+ **kwargs) |
240 |
except ParseError as e: |
241 |
writemsg("\n\n!!!%s.\nSee make.conf(5) for more info.\n" |
242 |
% e, noiselevel=-1) |
243 |
@@ -2898,9 +2919,16 @@ def run_action(emerge_config): |
244 |
if (emerge_config.action in ('search', None) and |
245 |
'--usepkg' in emerge_config.opts): |
246 |
for mytrees in emerge_config.trees.values(): |
247 |
+ kwargs = {} |
248 |
+ if (mytrees is emerge_config.target_config.trees and |
249 |
+ emerge_config.target_config is not emerge_config.running_config and |
250 |
+ emerge_config.opts.get('--quickpkg-direct', 'n') == 'y'): |
251 |
+ kwargs['add_repos'] = (emerge_config.running_config.trees['vartree'].dbapi,) |
252 |
+ |
253 |
try: |
254 |
mytrees['bintree'].populate( |
255 |
- getbinpkgs='--getbinpkg' in emerge_config.opts) |
256 |
+ getbinpkgs='--getbinpkg' in emerge_config.opts, |
257 |
+ **kwargs) |
258 |
except ParseError as e: |
259 |
writemsg('\n\n!!!%s.\nSee make.conf(5) for more info.\n' |
260 |
% (e,), noiselevel=-1) |
261 |
diff --git a/lib/_emerge/depgraph.py b/lib/_emerge/depgraph.py |
262 |
index 68b5bdb2e..0cc3c8259 100644 |
263 |
--- a/lib/_emerge/depgraph.py |
264 |
+++ b/lib/_emerge/depgraph.py |
265 |
@@ -495,6 +495,7 @@ class _dynamic_depgraph_config(object): |
266 |
self._backtrack_infos = {} |
267 |
|
268 |
self._buildpkgonly_deps_unsatisfied = False |
269 |
+ self._quickpkg_direct_deps_unsatisfied = False |
270 |
self._autounmask = self.myparams['autounmask'] |
271 |
self._displayed_autounmask = False |
272 |
self._success_without_autounmask = False |
273 |
@@ -4528,6 +4529,16 @@ class depgraph(object): |
274 |
self._dynamic_config._skip_restart = True |
275 |
return False, myfavorites |
276 |
|
277 |
+ if (self._frozen_config.myopts.get('--quickpkg-direct', 'n') == 'y' and |
278 |
+ self._frozen_config.target_root is not self._frozen_config._running_root): |
279 |
+ running_root = self._frozen_config._running_root.root |
280 |
+ for node in self._dynamic_config.digraph: |
281 |
+ if (isinstance(node, Package) and node.operation == "merge" and |
282 |
+ node.root == running_root): |
283 |
+ self._dynamic_config._quickpkg_direct_deps_unsatisfied = True |
284 |
+ self._dynamic_config._skip_restart = True |
285 |
+ return False, myfavorites |
286 |
+ |
287 |
if (not self._dynamic_config._prune_rebuilds and |
288 |
self._ignored_binaries_autounmask_backtrack()): |
289 |
config = self._dynamic_config._backtrack_infos.setdefault("config", {}) |
290 |
@@ -9054,6 +9065,14 @@ class depgraph(object): |
291 |
writemsg("!!! Cannot merge requested packages. " |
292 |
"Merge deps and try again.\n\n", noiselevel=-1) |
293 |
|
294 |
+ if self._dynamic_config._quickpkg_direct_deps_unsatisfied: |
295 |
+ self._show_merge_list() |
296 |
+ writemsg("\n!!! --quickpkg-direct requires all " |
297 |
+ "dependencies to be merged for root '{}'.\n".format( |
298 |
+ self._frozen_config._running_root.root), noiselevel=-1) |
299 |
+ writemsg("!!! Cannot merge requested packages. " |
300 |
+ "Merge deps and try again.\n\n", noiselevel=-1) |
301 |
+ |
302 |
def saveNomergeFavorites(self): |
303 |
"""Find atoms in favorites that are not in the mergelist and add them |
304 |
to the world file if necessary.""" |
305 |
diff --git a/lib/_emerge/main.py b/lib/_emerge/main.py |
306 |
index 0d2c45a4f..3e1de8fa7 100644 |
307 |
--- a/lib/_emerge/main.py |
308 |
+++ b/lib/_emerge/main.py |
309 |
@@ -637,6 +637,11 @@ def parse_opts(tmpcmdline, silent=False): |
310 |
"action" : "store", |
311 |
}, |
312 |
|
313 |
+ "--quickpkg-direct": { |
314 |
+ "help": "Enable use of installed packages directly as binary packages", |
315 |
+ "choices": y_or_n |
316 |
+ }, |
317 |
+ |
318 |
"--quiet": { |
319 |
"shortopt" : "-q", |
320 |
"help" : "reduced or condensed output", |
321 |
diff --git a/lib/portage/dbapi/__init__.py b/lib/portage/dbapi/__init__.py |
322 |
index 80f8a689f..bf541c167 100644 |
323 |
--- a/lib/portage/dbapi/__init__.py |
324 |
+++ b/lib/portage/dbapi/__init__.py |
325 |
@@ -32,8 +32,7 @@ class dbapi(object): |
326 |
_use_mutable = False |
327 |
_known_keys = frozenset(x for x in auxdbkeys |
328 |
if not x.startswith("UNUSED_0")) |
329 |
- _pkg_str_aux_keys = ("BUILD_TIME", "EAPI", "BUILD_ID", |
330 |
- "KEYWORDS", "SLOT", "repository") |
331 |
+ _pkg_str_aux_keys = ("EAPI", "KEYWORDS", "SLOT", "repository") |
332 |
|
333 |
def __init__(self): |
334 |
pass |
335 |
diff --git a/lib/portage/dbapi/bintree.py b/lib/portage/dbapi/bintree.py |
336 |
index ba21e6d23..1c25b816c 100644 |
337 |
--- a/lib/portage/dbapi/bintree.py |
338 |
+++ b/lib/portage/dbapi/bintree.py |
339 |
@@ -7,6 +7,7 @@ __all__ = ["bindbapi", "binarytree"] |
340 |
|
341 |
import portage |
342 |
portage.proxy.lazyimport.lazyimport(globals(), |
343 |
+ '_emerge.BinpkgExtractorAsync:BinpkgExtractorAsync', |
344 |
'portage.checksum:get_valid_checksum_keys,perform_multiple_checksums,' + \ |
345 |
'verify_all,_apply_hash_filter,_hash_filter', |
346 |
'portage.dbapi.dep_expand:dep_expand', |
347 |
@@ -18,6 +19,7 @@ portage.proxy.lazyimport.lazyimport(globals(), |
348 |
'portage.util:atomic_ofstream,ensure_dirs,normalize_path,' + \ |
349 |
'writemsg,writemsg_stdout', |
350 |
'portage.util.path:first_existing', |
351 |
+ 'portage.util._async.SchedulerInterface:SchedulerInterface', |
352 |
'portage.util._urlopen:urlopen@_urlopen,have_pep_476@_have_pep_476', |
353 |
'portage.versions:best,catpkgsplit,catsplit,_pkg_str', |
354 |
) |
355 |
@@ -30,6 +32,9 @@ from portage.exception import AlarmSignal, InvalidData, InvalidPackageName, \ |
356 |
ParseError, PermissionDenied, PortageException |
357 |
from portage.localization import _ |
358 |
from portage.package.ebuild.profile_iuse import iter_iuse_vars |
359 |
+from portage.util.futures import asyncio |
360 |
+from portage.util.futures.compat_coroutine import coroutine |
361 |
+from portage.util.futures.executor.fork import ForkExecutor |
362 |
from portage import _movefile |
363 |
from portage import os |
364 |
from portage import _encodings |
365 |
@@ -70,6 +75,8 @@ class UseCachedCopyOfRemoteIndex(Exception): |
366 |
class bindbapi(fakedbapi): |
367 |
_known_keys = frozenset(list(fakedbapi._known_keys) + \ |
368 |
["CHOST", "repository", "USE"]) |
369 |
+ _pkg_str_aux_keys = fakedbapi._pkg_str_aux_keys + ("BUILD_ID", "BUILD_TIME", "_mtime_") |
370 |
+ |
371 |
def __init__(self, mybintree=None, **kwargs): |
372 |
# Always enable multi_instance mode for bindbapi indexing. This |
373 |
# does not affect the local PKGDIR file layout, since that is |
374 |
@@ -142,7 +149,10 @@ class bindbapi(fakedbapi): |
375 |
return [aux_cache.get(x, "") for x in wants] |
376 |
mysplit = mycpv.split("/") |
377 |
mylist = [] |
378 |
- if not self.bintree._remotepkgs or \ |
379 |
+ add_pkg = self.bintree._additional_pkgs.get(instance_key) |
380 |
+ if add_pkg is not None: |
381 |
+ return add_pkg._db.aux_get(add_pkg, wants) |
382 |
+ elif not self.bintree._remotepkgs or \ |
383 |
not self.bintree.isremote(mycpv): |
384 |
try: |
385 |
tbz2_path = self.bintree._pkg_paths[instance_key] |
386 |
@@ -218,6 +228,73 @@ class bindbapi(fakedbapi): |
387 |
# inject will clear stale caches via cpv_inject. |
388 |
self.bintree.inject(cpv, filename=tbz2path) |
389 |
|
390 |
+ |
391 |
+ @coroutine |
392 |
+ def unpack_metadata(self, pkg, dest_dir): |
393 |
+ """ |
394 |
+ Unpack package metadata to a directory. This method is a coroutine. |
395 |
+ |
396 |
+ @param pkg: package to unpack |
397 |
+ @type pkg: _pkg_str or portage.config |
398 |
+ @param dest_dir: destination directory |
399 |
+ @type dest_dir: str |
400 |
+ """ |
401 |
+ loop = asyncio._wrap_loop() |
402 |
+ if isinstance(pkg, _pkg_str): |
403 |
+ cpv = pkg |
404 |
+ else: |
405 |
+ cpv = pkg.mycpv |
406 |
+ key = self._instance_key(cpv) |
407 |
+ add_pkg = self.bintree._additional_pkgs.get(key) |
408 |
+ if add_pkg is not None: |
409 |
+ yield add_pkg._db.unpack_metadata(pkg, dest_dir) |
410 |
+ else: |
411 |
+ tbz2_file = self.bintree.getname(cpv) |
412 |
+ yield loop.run_in_executor(ForkExecutor(loop=loop), |
413 |
+ portage.xpak.tbz2(tbz2_file).unpackinfo, dest_dir) |
414 |
+ |
415 |
+ @coroutine |
416 |
+ def unpack_contents(self, pkg, dest_dir): |
417 |
+ """ |
418 |
+ Unpack package contents to a directory. This method is a coroutine. |
419 |
+ |
420 |
+ @param pkg: package to unpack |
421 |
+ @type pkg: _pkg_str or portage.config |
422 |
+ @param dest_dir: destination directory |
423 |
+ @type dest_dir: str |
424 |
+ """ |
425 |
+ loop = asyncio._wrap_loop() |
426 |
+ if isinstance(pkg, _pkg_str): |
427 |
+ settings = self.settings |
428 |
+ cpv = pkg |
429 |
+ else: |
430 |
+ settings = pkg |
431 |
+ cpv = settings.mycpv |
432 |
+ |
433 |
+ pkg_path = self.bintree.getname(cpv) |
434 |
+ if pkg_path is not None: |
435 |
+ |
436 |
+ extractor = BinpkgExtractorAsync( |
437 |
+ background=settings.get('PORTAGE_BACKGROUND') == '1', |
438 |
+ env=settings.environ(), |
439 |
+ features=settings.features, |
440 |
+ image_dir=dest_dir, |
441 |
+ pkg=cpv, pkg_path=pkg_path, |
442 |
+ logfile=settings.get('PORTAGE_LOG_FILE'), |
443 |
+ scheduler=SchedulerInterface(loop)) |
444 |
+ |
445 |
+ extractor.start() |
446 |
+ yield extractor.async_wait() |
447 |
+ if extractor.returncode != os.EX_OK: |
448 |
+ raise PortageException("Error Extracting '{}'".format(pkg_path)) |
449 |
+ |
450 |
+ else: |
451 |
+ instance_key = self._instance_key(cpv) |
452 |
+ add_pkg = self.bintree._additional_pkgs.get(instance_key) |
453 |
+ if add_pkg is None: |
454 |
+ raise portage.exception.PackageNotFound(cpv) |
455 |
+ yield add_pkg._db.unpack_contents(pkg, dest_dir) |
456 |
+ |
457 |
def cp_list(self, *pargs, **kwargs): |
458 |
if not self.bintree.populated: |
459 |
self.bintree.populate() |
460 |
@@ -261,6 +338,7 @@ class bindbapi(fakedbapi): |
461 |
|
462 |
return filesdict |
463 |
|
464 |
+ |
465 |
class binarytree(object): |
466 |
"this tree scans for a list of all packages available in PKGDIR" |
467 |
def __init__(self, _unused=DeprecationWarning, pkgdir=None, |
468 |
@@ -301,6 +379,7 @@ class binarytree(object): |
469 |
self.tree = {} |
470 |
self._remote_has_index = False |
471 |
self._remotepkgs = None # remote metadata indexed by cpv |
472 |
+ self._additional_pkgs = {} |
473 |
self.invalids = [] |
474 |
self.settings = settings |
475 |
self._pkg_paths = {} |
476 |
@@ -511,7 +590,7 @@ class binarytree(object): |
477 |
except PortageException: |
478 |
pass |
479 |
|
480 |
- def populate(self, getbinpkgs=False, getbinpkg_refresh=True): |
481 |
+ def populate(self, getbinpkgs=False, getbinpkg_refresh=True, add_repos=()): |
482 |
""" |
483 |
Populates the binarytree with package metadata. |
484 |
|
485 |
@@ -520,12 +599,14 @@ class binarytree(object): |
486 |
@param getbinpkg_refresh: attempt to refresh the cache |
487 |
of remote package metadata if getbinpkgs is also True |
488 |
@type getbinpkg_refresh: bool |
489 |
+ @param add_repos: additional binary package repositories |
490 |
+ @type add_repos: sequence |
491 |
""" |
492 |
|
493 |
if self._populating: |
494 |
return |
495 |
|
496 |
- if not os.path.isdir(self.pkgdir) and not getbinpkgs: |
497 |
+ if not os.path.isdir(self.pkgdir) and not (getbinpkgs or add_repos): |
498 |
self.populated = True |
499 |
return |
500 |
|
501 |
@@ -557,6 +638,9 @@ class binarytree(object): |
502 |
if pkgindex_lock: |
503 |
unlockfile(pkgindex_lock) |
504 |
|
505 |
+ if add_repos: |
506 |
+ self._populate_additional(add_repos) |
507 |
+ |
508 |
if getbinpkgs: |
509 |
if not self.settings.get("PORTAGE_BINHOST"): |
510 |
writemsg(_("!!! PORTAGE_BINHOST unset, but use is requested.\n"), |
511 |
@@ -1066,6 +1150,16 @@ class binarytree(object): |
512 |
self._merge_pkgindex_header(pkgindex.header, |
513 |
self._pkgindex_header) |
514 |
|
515 |
+ def _populate_additional(self, repos): |
516 |
+ for repo in repos: |
517 |
+ aux_keys = list(set(chain(repo._aux_cache_keys, repo._pkg_str_aux_keys))) |
518 |
+ for cpv in repo.cpv_all(): |
519 |
+ metadata = dict(zip(aux_keys, repo.aux_get(cpv, aux_keys))) |
520 |
+ pkg = _pkg_str(cpv, metadata=metadata, settings=repo.settings, db=repo) |
521 |
+ instance_key = self.dbapi._instance_key(pkg) |
522 |
+ self._additional_pkgs[instance_key] = pkg |
523 |
+ self.dbapi.cpv_inject(pkg) |
524 |
+ |
525 |
def inject(self, cpv, filename=None): |
526 |
"""Add a freshly built package to the database. This updates |
527 |
$PKGDIR/Packages with the new package metadata (including MD5). |
528 |
@@ -1500,6 +1594,8 @@ class binarytree(object): |
529 |
filename = self._pkg_paths.get(instance_key) |
530 |
if filename is not None: |
531 |
filename = os.path.join(self.pkgdir, filename) |
532 |
+ elif instance_key in self._additional_pkgs: |
533 |
+ return None |
534 |
|
535 |
if filename is None: |
536 |
if self._multi_instance: |
537 |
@@ -1570,8 +1666,12 @@ class binarytree(object): |
538 |
def isremote(self, pkgname): |
539 |
"""Returns true if the package is kept remotely and it has not been |
540 |
downloaded (or it is only partially downloaded).""" |
541 |
- if (self._remotepkgs is None or |
542 |
- self.dbapi._instance_key(pkgname) not in self._remotepkgs): |
543 |
+ if self._remotepkgs is None: |
544 |
+ return False |
545 |
+ instance_key = self.dbapi._instance_key(pkgname) |
546 |
+ if instance_key not in self._remotepkgs: |
547 |
+ return False |
548 |
+ elif instance_key in self._additional_pkgs: |
549 |
return False |
550 |
# Presence in self._remotepkgs implies that it's remote. When a |
551 |
# package is downloaded, state is updated by self.inject(). |
552 |
diff --git a/lib/portage/dbapi/vartree.py b/lib/portage/dbapi/vartree.py |
553 |
index 603d58015..8a5715e54 100644 |
554 |
--- a/lib/portage/dbapi/vartree.py |
555 |
+++ b/lib/portage/dbapi/vartree.py |
556 |
@@ -33,7 +33,7 @@ portage.proxy.lazyimport.lazyimport(globals(), |
557 |
'portage.util._compare_files:compare_files', |
558 |
'portage.util.digraph:digraph', |
559 |
'portage.util.env_update:env_update', |
560 |
- 'portage.util.install_mask:install_mask_dir,InstallMask', |
561 |
+ 'portage.util.install_mask:install_mask_dir,InstallMask,_raise_exc', |
562 |
'portage.util.listdir:dircache,listdir', |
563 |
'portage.util.movefile:movefile', |
564 |
'portage.util.monotonic:monotonic', |
565 |
@@ -71,6 +71,8 @@ from portage import _os_merge |
566 |
from portage import _selinux_merge |
567 |
from portage import _unicode_decode |
568 |
from portage import _unicode_encode |
569 |
+from portage.util.futures.compat_coroutine import coroutine |
570 |
+from portage.util.futures.executor.fork import ForkExecutor |
571 |
from ._VdbMetadataDelta import VdbMetadataDelta |
572 |
|
573 |
from _emerge.EbuildBuildDir import EbuildBuildDir |
574 |
@@ -80,8 +82,10 @@ from _emerge.MiscFunctionsProcess import MiscFunctionsProcess |
575 |
from _emerge.SpawnProcess import SpawnProcess |
576 |
from ._ContentsCaseSensitivityManager import ContentsCaseSensitivityManager |
577 |
|
578 |
+import argparse |
579 |
import errno |
580 |
import fnmatch |
581 |
+import functools |
582 |
import gc |
583 |
import grp |
584 |
import io |
585 |
@@ -128,6 +132,7 @@ class vardbapi(dbapi): |
586 |
|
587 |
_aux_cache_keys_re = re.compile(r'^NEEDED\..*$') |
588 |
_aux_multi_line_re = re.compile(r'^(CONTENTS|NEEDED\..*)$') |
589 |
+ _pkg_str_aux_keys = dbapi._pkg_str_aux_keys + ("BUILD_ID", "BUILD_TIME", "_mtime_") |
590 |
|
591 |
def __init__(self, _unused_param=DeprecationWarning, |
592 |
categories=None, settings=None, vartree=None): |
593 |
@@ -953,6 +958,109 @@ class vardbapi(dbapi): |
594 |
pass |
595 |
self._bump_mtime(cpv) |
596 |
|
597 |
+ @coroutine |
598 |
+ def unpack_metadata(self, pkg, dest_dir): |
599 |
+ """ |
600 |
+ Unpack package metadata to a directory. This method is a coroutine. |
601 |
+ |
602 |
+ @param pkg: package to unpack |
603 |
+ @type pkg: _pkg_str or portage.config |
604 |
+ @param dest_dir: destination directory |
605 |
+ @type dest_dir: str |
606 |
+ """ |
607 |
+ loop = asyncio._wrap_loop() |
608 |
+ if not isinstance(pkg, portage.config): |
609 |
+ cpv = pkg |
610 |
+ else: |
611 |
+ cpv = pkg.mycpv |
612 |
+ dbdir = self.getpath(cpv) |
613 |
+ def async_copy(): |
614 |
+ for parent, dirs, files in os.walk(dbdir, onerror=_raise_exc): |
615 |
+ for key in files: |
616 |
+ shutil.copy(os.path.join(parent, key), |
617 |
+ os.path.join(dest_dir, key)) |
618 |
+ break |
619 |
+ yield loop.run_in_executor(ForkExecutor(loop=loop), async_copy) |
620 |
+ |
621 |
+ @coroutine |
622 |
+ def unpack_contents(self, pkg, dest_dir, |
623 |
+ include_config=None, include_unmodified_config=None): |
624 |
+ """ |
625 |
+ Unpack package contents to a directory. This method is a coroutine. |
626 |
+ |
627 |
+ This copies files from the installed system, in the same way |
628 |
+ as the quickpkg(1) command. Default behavior for handling |
629 |
+ of protected configuration files is controlled by the |
630 |
+ QUICKPKG_DEFAULT_OPTS variable. The relevant quickpkg options |
631 |
+ are --include-config and --include-unmodified-config. When |
632 |
+ a configuration file is not included because it is protected, |
633 |
+ an ewarn message is logged. |
634 |
+ |
635 |
+ @param pkg: package to unpack |
636 |
+ @type pkg: _pkg_str or portage.config |
637 |
+ @param dest_dir: destination directory |
638 |
+ @type dest_dir: str |
639 |
+ @param include_config: Include all files protected by |
640 |
+ CONFIG_PROTECT (as a security precaution, default is False |
641 |
+ unless modified by QUICKPKG_DEFAULT_OPTS). |
642 |
+ @type include_config: bool |
643 |
+ @param include_unmodified_config: Include files protected by |
644 |
+ CONFIG_PROTECT that have not been modified since installation |
645 |
+ (as a security precaution, default is False unless modified |
646 |
+ by QUICKPKG_DEFAULT_OPTS). |
647 |
+ @type include_unmodified_config: bool |
648 |
+ """ |
649 |
+ loop = asyncio._wrap_loop() |
650 |
+ if not isinstance(pkg, portage.config): |
651 |
+ settings = self.settings |
652 |
+ cpv = pkg |
653 |
+ else: |
654 |
+ settings = pkg |
655 |
+ cpv = settings.mycpv |
656 |
+ |
657 |
+ scheduler = SchedulerInterface(loop) |
658 |
+ parser = argparse.ArgumentParser() |
659 |
+ parser.add_argument('--include-config', |
660 |
+ choices=('y', 'n'), |
661 |
+ default='n') |
662 |
+ parser.add_argument('--include-unmodified-config', |
663 |
+ choices=('y', 'n'), |
664 |
+ default='n') |
665 |
+ |
666 |
+ # Method parameters may override QUICKPKG_DEFAULT_OPTS. |
667 |
+ opts_list = portage.util.shlex_split(settings.get('QUICKPKG_DEFAULT_OPTS', '')) |
668 |
+ if include_config is not None: |
669 |
+ opts_list.append('--include-config={}'.format( |
670 |
+ 'y' if include_config else 'n')) |
671 |
+ if include_unmodified_config is not None: |
672 |
+ opts_list.append('--include-unmodified-config={}'.format( |
673 |
+ 'y' if include_unmodified_config else 'n')) |
674 |
+ |
675 |
+ opts, args = parser.parse_known_args(opts_list) |
676 |
+ |
677 |
+ tar_cmd = ('tar', '-x', '--xattrs', '--xattrs-include=*', '-C', dest_dir) |
678 |
+ pr, pw = os.pipe() |
679 |
+ proc = (yield asyncio.create_subprocess_exec(*tar_cmd, stdin=pr)) |
680 |
+ os.close(pr) |
681 |
+ excluded_config_files = (yield loop.run_in_executor(ForkExecutor(loop=loop), |
682 |
+ functools.partial(self._dblink(cpv).quickpkg, |
683 |
+ os.fdopen(pw, 'wb', 0), |
684 |
+ include_config=opts.include_config == 'y', |
685 |
+ include_unmodified_config=opts.include_unmodified_config == 'y'))) |
686 |
+ yield proc.wait() |
687 |
+ if proc.returncode != os.EX_OK: |
688 |
+ raise PortageException('command failed: {}'.format(tar_cmd)) |
689 |
+ |
690 |
+ if excluded_config_files: |
691 |
+ log_lines = ([_("Config files excluded by QUICKPKG_DEFAULT_OPTS (see quickpkg(1) man page):")] + |
692 |
+ ['\t{}'.format(name) for name in excluded_config_files]) |
693 |
+ out = io.StringIO() |
694 |
+ for line in log_lines: |
695 |
+ portage.elog.messages.ewarn(line, phase='install', key=cpv, out=out) |
696 |
+ scheduler.output(out.getvalue(), |
697 |
+ background=self.settings.get("PORTAGE_BACKGROUND") == "1", |
698 |
+ log_path=settings.get("PORTAGE_LOG_FILE")) |
699 |
+ |
700 |
def counter_tick(self, myroot=None, mycpv=None): |
701 |
""" |
702 |
@param myroot: ignored, self._eroot is used instead |
703 |
diff --git a/lib/portage/tests/emerge/test_simple.py b/lib/portage/tests/emerge/test_simple.py |
704 |
index 866521488..2f9d4d1e7 100644 |
705 |
--- a/lib/portage/tests/emerge/test_simple.py |
706 |
+++ b/lib/portage/tests/emerge/test_simple.py |
707 |
@@ -254,6 +254,7 @@ call_has_and_best_version() { |
708 |
cross_eroot = os.path.join(cross_root, eprefix.lstrip(os.sep)) |
709 |
|
710 |
test_commands = ( |
711 |
+ emerge_cmd + ("--usepkgonly", "--root", cross_root, "--quickpkg-direct=y", "dev-libs/A"), |
712 |
env_update_cmd, |
713 |
portageq_cmd + ("envvar", "-v", "CONFIG_PROTECT", "EROOT", |
714 |
"PORTAGE_CONFIGROOT", "PORTAGE_TMPDIR", "USERLAND"), |
715 |
diff --git a/man/emerge.1 b/man/emerge.1 |
716 |
index dd6cdfd6e..56f3b6b2e 100644 |
717 |
--- a/man/emerge.1 |
718 |
+++ b/man/emerge.1 |
719 |
@@ -829,6 +829,23 @@ B blocked by another package (unresolved conflict) |
720 |
b blocked by another package (automatically resolved conflict) |
721 |
.TE |
722 |
.TP |
723 |
+.BR "\-\-quickpkg\-direct < y | n >" |
724 |
+Enable use of installed packages directly as binary packages. This is |
725 |
+similar to using binary packages produced by \fBquickpkg\fR(1), but |
726 |
+installed packages are used directly as though they are binary packages. |
727 |
+This option only works in combination with the \fB\-\-root=DIR\fR option, |
728 |
+and it comes with the caveat that packages are only allowed to be |
729 |
+installed into the root that is specified by the \fB\-\-root=DIR\fR |
730 |
+option (the other root which serves as a source of packages is |
731 |
+assumed to be immutabled during the entire operation). |
732 |
+ |
733 |
+Default behavior for handling of protected configuration files is |
734 |
+controlled by the \fBQUICKPKG_DEFAULT_OPTS\fR variable. The relevant |
735 |
+quickpkg options are \fI\-\-include\-config\fR and |
736 |
+\fI\-\-include\-unmodified\-config\fR (refer to the \fBquickpkg\fR(1) |
737 |
+man page). When a configuration file is not included because it is |
738 |
+protected, an ewarn message is logged. |
739 |
+.TP |
740 |
.BR "\-\-quiet [ y | n ]" ", " \-q |
741 |
Results may vary, but the general outcome is a reduced or condensed |
742 |
output from portage's displays. |
743 |
-- |
744 |
2.21.0 |