Gentoo Archives: gentoo-commits

From: Sam James <sam@g.o>
To: gentoo-commits@l.g.o
Subject: [gentoo-commits] proj/portage:master commit in: repoman/lib/repoman/tests/, lib/_emerge/, lib/portage/, man/, ...
Date: Mon, 24 Jan 2022 01:17:26
Message-Id: 1642987030.b094ba47368bb9b16fb17ee47a54644a09860823.sam@gentoo
1 commit: b094ba47368bb9b16fb17ee47a54644a09860823
2 Author: Rin Cat (鈴猫) <dev <AT> rincat <DOT> ch>
3 AuthorDate: Sun Jan 23 23:23:14 2022 +0000
4 Commit: Sam James <sam <AT> gentoo <DOT> org>
5 CommitDate: Mon Jan 24 01:17:10 2022 +0000
6 URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=b094ba47
7
8 */*: GPKG (new binpkg format) Support (GLEP 78, provisionally)
9
10 Bug: https://bugs.gentoo.org/500630
11 Bug: https://bugs.gentoo.org/659864
12 Bug: https://bugs.gentoo.org/672672
13 Bug: https://bugs.gentoo.org/773259
14 Bug: https://bugs.gentoo.org/820578
15 Signed-off-by: Rin Cat (鈴猫) <dev <AT> rincat.ch>
16 Closes: https://github.com/gentoo/portage/pull/562
17 Signed-off-by: Sam James <sam <AT> gentoo.org>
18
19 MANIFEST.in | 3 +
20 bin/gpkg-helper.py | 71 +
21 bin/misc-functions.sh | 67 +-
22 bin/quickpkg | 108 +-
23 cnf/make.conf.example | 36 +
24 cnf/make.globals | 21 +
25 lib/_emerge/Binpkg.py | 13 +-
26 lib/_emerge/BinpkgExtractorAsync.py | 9 +
27 lib/_emerge/BinpkgFetcher.py | 31 +-
28 lib/_emerge/EbuildBinpkg.py | 16 +-
29 lib/_emerge/EbuildPhase.py | 33 +-
30 lib/_emerge/Package.py | 1 +
31 lib/_emerge/actions.py | 44 +-
32 lib/_emerge/depgraph.py | 30 +-
33 lib/portage/__init__.py | 1 +
34 lib/portage/binpkg.py | 56 +
35 lib/portage/const.py | 12 +
36 lib/portage/dbapi/bintree.py | 445 ++++-
37 lib/portage/dbapi/vartree.py | 77 +-
38 lib/portage/exception.py | 20 +
39 lib/portage/gpg.py | 106 +
40 lib/portage/gpkg.py | 2015 ++++++++++++++++++++
41 .../package/ebuild/_config/special_env_vars.py | 1 +
42 lib/portage/package/ebuild/config.py | 10 +
43 lib/portage/package/ebuild/doebuild.py | 13 +
44 .../06B3A311BD775C280D22A9305D90EA06352177F6.rev | 37 +
45 .../8DEDA2CDED49C8809287B89D8812797DDF1DD192.rev | 37 +
46 .../273B030399E7BEA66A9AD42216DE7CA17BA5D42E.key | Bin 0 -> 2055 bytes
47 .../C99796FB85B0C3DF03314A11B5850C51167D6282.key | Bin 0 -> 2055 bytes
48 lib/portage/tests/.gnupg/pubring.kbx | Bin 0 -> 2774 bytes
49 lib/portage/tests/.gnupg/trustdb.gpg | Bin 0 -> 1360 bytes
50 lib/portage/tests/__init__.py | 37 +-
51 lib/portage/tests/emerge/test_simple.py | 47 +-
52 lib/portage/tests/gpkg/__init__.py | 2 +
53 lib/portage/tests/gpkg/__test__.py | 0
54 lib/portage/tests/gpkg/test_gpkg_checksum.py | 396 ++++
55 lib/portage/tests/gpkg/test_gpkg_gpg.py | 398 ++++
56 .../tests/gpkg/test_gpkg_metadata_update.py | 59 +
57 lib/portage/tests/gpkg/test_gpkg_metadata_url.py | 173 ++
58 lib/portage/tests/gpkg/test_gpkg_path.py | 390 ++++
59 lib/portage/tests/gpkg/test_gpkg_size.py | 58 +
60 lib/portage/tests/gpkg/test_gpkg_stream.py | 112 ++
61 lib/portage/tests/resolver/ResolverPlayground.py | 55 +-
62 .../test_build_id_profile_format.py | 50 +-
63 .../binpkg_multi_instance/test_rebuilt_binaries.py | 44 +-
64 .../tests/resolver/soname/test_autounmask.py | 38 +-
65 .../tests/resolver/soname/test_downgrade.py | 84 +-
66 .../tests/resolver/soname/test_or_choices.py | 39 +-
67 .../tests/resolver/soname/test_reinstall.py | 40 +-
68 .../tests/resolver/soname/test_skip_update.py | 39 +-
69 .../soname/test_slot_conflict_reinstall.py | 135 +-
70 .../resolver/soname/test_slot_conflict_update.py | 38 +-
71 .../tests/resolver/soname/test_soname_provided.py | 45 +-
72 .../tests/resolver/soname/test_unsatisfiable.py | 40 +-
73 .../tests/resolver/soname/test_unsatisfied.py | 40 +-
74 .../tests/resolver/test_autounmask_binpkg_use.py | 38 +-
75 lib/portage/tests/resolver/test_bdeps.py | 44 +-
76 .../resolver/test_binary_pkg_ebuild_visibility.py | 35 +-
77 lib/portage/tests/resolver/test_changed_deps.py | 41 +-
78 ...test_complete_if_new_subslot_without_revbump.py | 40 +-
79 .../resolver/test_disjunctive_depend_order.py | 34 +-
80 lib/portage/tests/resolver/test_multirepo.py | 62 +-
81 .../test_regular_slot_change_without_revbump.py | 41 +-
82 lib/portage/tests/resolver/test_simple.py | 34 +-
83 lib/portage/tests/resolver/test_slot_abi.py | 113 +-
84 .../tests/resolver/test_slot_abi_downgrade.py | 77 +-
85 .../resolver/test_slot_change_without_revbump.py | 40 +-
86 .../resolver/test_slot_operator_autounmask.py | 40 +-
87 .../tests/resolver/test_slot_operator_bdeps.py | 74 +-
88 .../tests/resolver/test_slot_operator_rebuild.py | 40 +-
89 .../tests/resolver/test_slot_operator_unsolved.py | 41 +-
90 lib/portage/tests/resolver/test_useflags.py | 37 +-
91 lib/portage/tests/runTests.py | 12 +
92 lib/portage/tests/update/test_move_ent.py | 129 +-
93 lib/portage/tests/update/test_move_slot_ent.py | 139 +-
94 lib/portage/tests/update/test_update_dbentry.py | 182 +-
95 lib/portage/util/_urlopen.py | 4 +-
96 lib/portage/versions.py | 16 +
97 man/make.conf.5 | 80 +
98 .../06B3A311BD775C280D22A9305D90EA06352177F6.rev | 37 +
99 .../8DEDA2CDED49C8809287B89D8812797DDF1DD192.rev | 37 +
100 .../273B030399E7BEA66A9AD42216DE7CA17BA5D42E.key | Bin 0 -> 2055 bytes
101 .../C99796FB85B0C3DF03314A11B5850C51167D6282.key | Bin 0 -> 2055 bytes
102 repoman/lib/repoman/tests/.gnupg/pubring.kbx | Bin 0 -> 2774 bytes
103 repoman/lib/repoman/tests/.gnupg/trustdb.gpg | Bin 0 -> 1360 bytes
104 repoman/lib/repoman/tests/runTests.py | 10 +
105 repoman/setup.py | 12 +-
106 setup.py | 14 +-
107 88 files changed, 6272 insertions(+), 903 deletions(-)
108
109 diff --git a/MANIFEST.in b/MANIFEST.in
110 index 7d9ea761a..31ca9c166 100644
111 --- a/MANIFEST.in
112 +++ b/MANIFEST.in
113 @@ -25,3 +25,6 @@ include misc/*
114
115 # extensions
116 include src/*
117 +
118 +# GPG test keys
119 +recursive-include lib/portage/tests/.gnupg *
120
121 diff --git a/bin/gpkg-helper.py b/bin/gpkg-helper.py
122 new file mode 100755
123 index 000000000..d79430212
124 --- /dev/null
125 +++ b/bin/gpkg-helper.py
126 @@ -0,0 +1,71 @@
127 +#!/usr/bin/python -b
128 +# Copyright 2020 Gentoo Authors
129 +# Distributed under the terms of the GNU General Public License v2
130 +
131 +import argparse
132 +import sys
133 +import portage
134 +
135 +portage._internal_caller = True
136 +from portage import os
137 +
138 +
139 +def command_compose(args):
140 +
141 + usage = "usage: compose <package_cpv> <binpkg_path> <metadata_dir> <image_dir>\n"
142 +
143 + if len(args) != 4:
144 + sys.stderr.write(usage)
145 + sys.stderr.write("4 arguments are required, got %s\n" % len(args))
146 + return 1
147 +
148 + cpv, binpkg_path, metadata_dir, image_dir = args
149 +
150 + if not os.path.isdir(metadata_dir):
151 + sys.stderr.write(usage)
152 + sys.stderr.write("Argument 3 is not a directory: '%s'\n" % metadata_dir)
153 + return 1
154 +
155 + if not os.path.isdir(image_dir):
156 + sys.stderr.write(usage)
157 + sys.stderr.write("Argument 4 is not a directory: '%s'\n" % image_dir)
158 + return 1
159 +
160 + gpkg_file = portage.gpkg.gpkg(portage.settings, cpv, binpkg_path)
161 + metadata = gpkg_file._generate_metadata_from_dir(metadata_dir)
162 + gpkg_file.compress(image_dir, metadata)
163 + return os.EX_OK
164 +
165 +
166 +def main(argv):
167 +
168 + if argv and isinstance(argv[0], bytes):
169 + for i, x in enumerate(argv):
170 + argv[i] = portage._unicode_decode(x, errors="strict")
171 +
172 + valid_commands = ("compress",)
173 + description = "Perform metadata operations on a binary package."
174 + usage = "usage: %s COMMAND [args]" % os.path.basename(argv[0])
175 +
176 + parser = argparse.ArgumentParser(description=description, usage=usage)
177 + options, args = parser.parse_known_args(argv[1:])
178 +
179 + if not args:
180 + parser.error("missing command argument")
181 +
182 + command = args[0]
183 +
184 + if command not in valid_commands:
185 + parser.error("invalid command: '%s'" % command)
186 +
187 + if command == "compress":
188 + rval = command_compose(args[1:])
189 + else:
190 + raise AssertionError("invalid command: '%s'" % command)
191 +
192 + return rval
193 +
194 +
195 +if __name__ == "__main__":
196 + rval = main(sys.argv[:])
197 + sys.exit(rval)
198
199 diff --git a/bin/misc-functions.sh b/bin/misc-functions.sh
200 index e4defa550..ccb07075f 100755
201 --- a/bin/misc-functions.sh
202 +++ b/bin/misc-functions.sh
203 @@ -503,38 +503,53 @@ __dyn_package() {
204 # in there in case any tools were built with -pg in CFLAGS.
205 cd "${T}" || die
206
207 - local tar_options=""
208 - [[ $PORTAGE_VERBOSE = 1 ]] && tar_options+=" -v"
209 - has xattr ${FEATURES} && [[ $(tar --help 2> /dev/null) == *--xattrs* ]] && tar_options+=" --xattrs"
210 # Sandbox is disabled in case the user wants to use a symlink
211 # for $PKGDIR and/or $PKGDIR/All.
212 export SANDBOX_ON="0"
213 - [ -z "${PORTAGE_BINPKG_TMPFILE}" ] && \
214 + [[ -z "${PORTAGE_BINPKG_TMPFILE}" ]] && \
215 die "PORTAGE_BINPKG_TMPFILE is unset"
216 mkdir -p "${PORTAGE_BINPKG_TMPFILE%/*}" || die "mkdir failed"
217 - [ -z "${PORTAGE_COMPRESSION_COMMAND}" ] && \
218 - die "PORTAGE_COMPRESSION_COMMAND is unset"
219 - tar $tar_options -cf - $PORTAGE_BINPKG_TAR_OPTS -C "${D}" . | \
220 - $PORTAGE_COMPRESSION_COMMAND > "$PORTAGE_BINPKG_TMPFILE"
221 - assert "failed to pack binary package: '$PORTAGE_BINPKG_TMPFILE'"
222 - PYTHONPATH=${PORTAGE_PYTHONPATH:-${PORTAGE_PYM_PATH}} \
223 - "${PORTAGE_PYTHON:-/usr/bin/python}" "$PORTAGE_BIN_PATH"/xpak-helper.py recompose \
224 - "$PORTAGE_BINPKG_TMPFILE" "$PORTAGE_BUILDDIR/build-info"
225 - if [ $? -ne 0 ]; then
226 - rm -f "${PORTAGE_BINPKG_TMPFILE}"
227 - die "Failed to append metadata to the tbz2 file"
228 - fi
229 - local md5_hash=""
230 - if type md5sum &>/dev/null ; then
231 - md5_hash=$(md5sum "${PORTAGE_BINPKG_TMPFILE}")
232 - md5_hash=${md5_hash%% *}
233 - elif type md5 &>/dev/null ; then
234 - md5_hash=$(md5 "${PORTAGE_BINPKG_TMPFILE}")
235 - md5_hash=${md5_hash##* }
236 +
237 + if [[ "${BINPKG_FORMAT}" == "xpak" ]]; then
238 + local tar_options=""
239 + [[ $PORTAGE_VERBOSE = 1 ]] && tar_options+=" -v"
240 + has xattr ${FEATURES} && [[ $(tar --help 2> /dev/null) == *--xattrs* ]] && tar_options+=" --xattrs"
241 + [[ -z "${PORTAGE_COMPRESSION_COMMAND}" ]] && \
242 + die "PORTAGE_COMPRESSION_COMMAND is unset"
243 + tar $tar_options -cf - $PORTAGE_BINPKG_TAR_OPTS -C "${D}" . | \
244 + $PORTAGE_COMPRESSION_COMMAND > "$PORTAGE_BINPKG_TMPFILE"
245 + assert "failed to pack binary package: '$PORTAGE_BINPKG_TMPFILE'"
246 + PYTHONPATH=${PORTAGE_PYTHONPATH:-${PORTAGE_PYM_PATH}} \
247 + "${PORTAGE_PYTHON:-/usr/bin/python}" "$PORTAGE_BIN_PATH"/xpak-helper.py recompose \
248 + "$PORTAGE_BINPKG_TMPFILE" "$PORTAGE_BUILDDIR/build-info"
249 + if [[ $? -ne 0 ]]; then
250 + rm -f "${PORTAGE_BINPKG_TMPFILE}"
251 + die "Failed to append metadata to the tbz2 file"
252 + fi
253 + local md5_hash=""
254 + if type md5sum &>/dev/null ; then
255 + md5_hash=$(md5sum "${PORTAGE_BINPKG_TMPFILE}")
256 + md5_hash=${md5_hash%% *}
257 + elif type md5 &>/dev/null ; then
258 + md5_hash=$(md5 "${PORTAGE_BINPKG_TMPFILE}")
259 + md5_hash=${md5_hash##* }
260 + fi
261 + [[ -n "${md5_hash}" ]] && \
262 + echo ${md5_hash} > "${PORTAGE_BUILDDIR}"/build-info/BINPKGMD5
263 + __vecho ">>> Done."
264 +
265 + elif [[ "${BINPKG_FORMAT}" == "gpkg" ]]; then
266 + PYTHONPATH=${PORTAGE_PYTHONPATH:-${PORTAGE_PYM_PATH}} \
267 + "${PORTAGE_PYTHON:-/usr/bin/python}" "$PORTAGE_BIN_PATH"/gpkg-helper.py compress \
268 + "${CATEGORY}/${PF}" "$PORTAGE_BINPKG_TMPFILE" "$PORTAGE_BUILDDIR/build-info" "${D}"
269 + if [[ $? -ne 0 ]]; then
270 + rm -f "${PORTAGE_BINPKG_TMPFILE}"
271 + die "Failed to create binpkg file"
272 + fi
273 + __vecho ">>> Done."
274 + else
275 + die "Unknown BINPKG_FORMAT ${BINPKG_FORMAT}"
276 fi
277 - [ -n "${md5_hash}" ] && \
278 - echo ${md5_hash} > "${PORTAGE_BUILDDIR}"/build-info/BINPKGMD5
279 - __vecho ">>> Done."
280
281 cd "${PORTAGE_BUILDDIR}"
282 >> "$PORTAGE_BUILDDIR/.packaged" || \
283
284 diff --git a/bin/quickpkg b/bin/quickpkg
285 index 1b7ad666c..7733bc833 100755
286 --- a/bin/quickpkg
287 +++ b/bin/quickpkg
288 @@ -15,17 +15,20 @@ if osp.isfile(osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), ".porta
289 import portage
290 portage._internal_caller = True
291 from portage import os
292 -from portage import xpak
293 +from portage import xpak, gpkg
294 +from portage.const import SUPPORTED_GENTOO_BINPKG_FORMATS
295 from portage.dbapi.dep_expand import dep_expand
296 from portage.dep import Atom, use_reduce
297 from portage.exception import (AmbiguousPackageName, InvalidAtom, InvalidData,
298 - InvalidDependString, PackageSetNotFound, PermissionDenied)
299 + InvalidBinaryPackageFormat, InvalidDependString, PackageSetNotFound,
300 + PermissionDenied)
301 from portage.util import ensure_dirs, shlex_split, varexpand, _xattr
302 xattr = _xattr.xattr
303 from portage._sets import load_default_config, SETPREFIX
304 from portage.process import find_binary
305 from portage.util.compression_probe import _compressors
306 from portage.util._eventloop.global_event_loop import global_event_loop
307 +from portage.gpg import GPG
308
309
310 def quickpkg_atom(options, infos, arg, eout):
311 @@ -109,52 +112,66 @@ def quickpkg_atom(options, infos, arg, eout):
312 update_metadata[k] = v
313 if update_metadata:
314 vardb.aux_update(cpv, update_metadata)
315 - xpdata = xpak.xpak(dblnk.dbdir)
316 - binpkg_tmpfile = os.path.join(bintree.pkgdir,
317 - cpv + ".tbz2." + str(portage.getpid()))
318 - ensure_dirs(os.path.dirname(binpkg_tmpfile))
319 - binpkg_compression = settings.get("BINPKG_COMPRESS", "bzip2")
320 - try:
321 - compression = _compressors[binpkg_compression]
322 - except KeyError as e:
323 - if binpkg_compression:
324 +
325 + binpkg_format = settings.get("BINPKG_FORMAT", SUPPORTED_GENTOO_BINPKG_FORMATS[0])
326 + if binpkg_format == "xpak":
327 + xpdata = xpak.xpak(dblnk.dbdir)
328 + binpkg_tmpfile = os.path.join(bintree.pkgdir,
329 + cpv + ".tbz2." + str(portage.getpid()))
330 + ensure_dirs(os.path.dirname(binpkg_tmpfile))
331 + binpkg_compression = settings.get("BINPKG_COMPRESS", "bzip2")
332 + try:
333 + compression = _compressors[binpkg_compression]
334 + except KeyError as e:
335 + if binpkg_compression:
336 + eout.eerror("Invalid or unsupported compression method: %s" % e.args[0])
337 + return 1
338 + # Empty BINPKG_COMPRESS disables compression.
339 + binpkg_compression = 'none'
340 + compression = {
341 + 'compress': 'cat',
342 + 'package': 'sys-apps/coreutils',
343 + }
344 + try:
345 + compression_binary = shlex_split(varexpand(compression["compress"], mydict=settings))[0]
346 + except IndexError as e:
347 eout.eerror("Invalid or unsupported compression method: %s" % e.args[0])
348 return 1
349 - # Empty BINPKG_COMPRESS disables compression.
350 - binpkg_compression = 'none'
351 - compression = {
352 - 'compress': 'cat',
353 - 'package': 'sys-apps/coreutils',
354 - }
355 - try:
356 - compression_binary = shlex_split(varexpand(compression["compress"], mydict=settings))[0]
357 - except IndexError as e:
358 - eout.eerror("Invalid or unsupported compression method: %s" % e.args[0])
359 - return 1
360 - if find_binary(compression_binary) is None:
361 - missing_package = compression["package"]
362 - eout.eerror("File compression unsupported %s. Missing package: %s" % (binpkg_compression, missing_package))
363 - return 1
364 - cmd = shlex_split(varexpand(compression["compress"], mydict=settings))
365 - # Filter empty elements that make Popen fail
366 - cmd = [x for x in cmd if x != ""]
367 - with open(binpkg_tmpfile, "wb") as fobj:
368 - proc = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=fobj)
369 - excluded_config_files = dblnk.quickpkg(proc.stdin,
370 + if find_binary(compression_binary) is None:
371 + missing_package = compression["package"]
372 + eout.eerror("File compression unsupported %s. Missing package: %s" % (binpkg_compression, missing_package))
373 + return 1
374 + cmd = [varexpand(x, mydict=settings) for x in shlex_split(compression["compress"])]
375 + # Filter empty elements that make Popen fail
376 + cmd = [x for x in cmd if x != ""]
377 + with open(binpkg_tmpfile, "wb") as fobj:
378 + proc = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=fobj)
379 + excluded_config_files = dblnk.quickpkg(proc.stdin,
380 + include_config=include_config,
381 + include_unmodified_config=include_unmodified_config)
382 + proc.stdin.close()
383 + if proc.wait() != os.EX_OK:
384 + eout.eend(1)
385 + eout.eerror("Compressor failed for package %s" % cpv)
386 + retval |= 1
387 + try:
388 + os.unlink(binpkg_tmpfile)
389 + except OSError as e:
390 + if e.errno not in (errno.ENOENT, errno.ESTALE):
391 + raise
392 + continue
393 + xpak.tbz2(binpkg_tmpfile).recompose_mem(xpdata)
394 + elif binpkg_format == "gpkg":
395 + metadata = gpkg.gpkg(settings)._generate_metadata_from_dir(dblnk.dbdir)
396 + binpkg_tmpfile = os.path.join(bintree.pkgdir,
397 + cpv + ".gpkg.tar." + str(os.getpid()))
398 + ensure_dirs(os.path.dirname(binpkg_tmpfile))
399 + excluded_config_files = dblnk.quickpkg(binpkg_tmpfile,
400 + metadata,
401 include_config=include_config,
402 include_unmodified_config=include_unmodified_config)
403 - proc.stdin.close()
404 - if proc.wait() != os.EX_OK:
405 - eout.eend(1)
406 - eout.eerror("Compressor failed for package %s" % cpv)
407 - retval |= 1
408 - try:
409 - os.unlink(binpkg_tmpfile)
410 - except OSError as e:
411 - if e.errno not in (errno.ENOENT, errno.ESTALE):
412 - raise
413 - continue
414 - xpak.tbz2(binpkg_tmpfile).recompose_mem(xpdata)
415 + else:
416 + raise InvalidBinaryPackageFormat(binpkg_format)
417 finally:
418 if have_lock:
419 dblnk.unlockdb()
420 @@ -268,6 +285,9 @@ def quickpkg_main(options, args, eout):
421 portage.settings.features.remove('xattr')
422 portage.settings.lock()
423
424 + gpg = GPG(portage.settings)
425 + gpg.unlock()
426 +
427 infos = {}
428 infos["successes"] = []
429 infos["missing"] = []
430
431 diff --git a/cnf/make.conf.example b/cnf/make.conf.example
432 index a309a5c43..5b2229465 100644
433 --- a/cnf/make.conf.example
434 +++ b/cnf/make.conf.example
435 @@ -183,6 +183,42 @@
436 # This ftp connection is active ftp.
437 #PORTAGE_BINHOST="ftp://login:pass@××××××××××.site:21*/pub/grp/i686/athlon-xp/"
438
439 +# Binary packages GPG commands
440 +# ============================
441 +#
442 +# Only works with GPKG format.
443 +# "binpkg-signing" needed to be set in FEATURES if need signing packages.
444 +# "binpkg-request-signature" needed to be set in FEATURES if you want all
445 +# binpkgs must have signature.
446 +# You need uncomment related commands and set "USER" and "SIGN_KEY" to yours.
447 +#
448 +# Binary package GPG singing base command
449 +# Basic command for all signature operations.
450 +# You need change this if you want to use other configurations,
451 +# Note that some configurations are configured separately below,
452 +# please do not add duplicate configurations
453 +#BINPKG_GPG_SIGNING_BASE_COMMAND="/usr/bin/flock /run/lock/portage-binpkg-gpg.lock /usr/bin/gpg --sign --armor [PORTAGE_CONFIG]"
454 +
455 +# Binary package GPG signature digests algorithm.
456 +#BINPKG_GPG_SIGNING_DIGEST="SHA512"
457 +
458 +# gnupg home directory used for signing.
459 +#BINPKG_GPG_SIGNING_GPG_HOME="/root/.gnupg"
460 +
461 +# GPG key ID used for signing.
462 +#BINPKG_GPG_SIGNING_KEY="0x1234567890ABCD!"
463 +
464 +# Binary package GPG verify base command.
465 +# Basic command for all verify operations.
466 +#BINPKG_GPG_VERIFY_BASE_COMMAND="/usr/bin/gpg --verify --batch --no-tty --no-auto-check-trustdb --status-fd 2 [PORTAGE_CONFIG] [SIGNATURE]"
467 +
468 +# GPG home directory where store all trust binary package public keys.
469 +#BINPKG_GPG_VERIFY_GPG_HOME="/etc/portage/gnupg"
470 +
471 +# The user and group will be used when drop root privileges during GPG verify
472 +#GPG_VERIFY_USER_DROP="nobody"
473 +#GPG_VERIFY_GROUP_DROP="nogroup"
474 +
475 # Synchronizing Portage
476 # =====================
477 #
478
479 diff --git a/cnf/make.globals b/cnf/make.globals
480 index cf4ad3533..69b365f71 100644
481 --- a/cnf/make.globals
482 +++ b/cnf/make.globals
483 @@ -38,6 +38,27 @@ PORTAGE_TMPDIR="/var/tmp"
484 # existing installs where bzip2 is used for backward compatibility.
485 BINPKG_COMPRESS="zstd"
486
487 +# The format used for binary packages. The default is use old "xpak" format.
488 +# Set to "gpkg" to use new gentoo binary package format.
489 +BINPKG_FORMAT="xpak"
490 +
491 +# The binary package default GPG signing command.
492 +# flock is used to avoid a racing condition of gnupg
493 +BINPKG_GPG_SIGNING_BASE_COMMAND="/usr/bin/flock /run/lock/portage-binpkg-gpg.lock /usr/bin/gpg --sign --armor [PORTAGE_CONFIG]"
494 +
495 +# The default binary package GPG digests algorithm.
496 +BINPKG_GPG_SIGNING_DIGEST="SHA512"
497 +
498 +# The binary package default GPG verify command.
499 +BINPKG_GPG_VERIFY_BASE_COMMAND="/usr/bin/gpg --verify --batch --no-tty --no-auto-check-trustdb --status-fd 2 [PORTAGE_CONFIG] [SIGNATURE]"
500 +
501 +# The binary package default GPG home directory for verify
502 +BINPKG_GPG_VERIFY_GPG_HOME="/etc/portage/gnupg"
503 +
504 +# The user and group will be used when drop root privileges during GPG verify
505 +GPG_VERIFY_USER_DROP="nobody"
506 +GPG_VERIFY_GROUP_DROP="nogroup"
507 +
508 # Fetching command (3 tries, passive ftp for firewall compatibility)
509 FETCHCOMMAND="wget -t 3 -T 60 --passive-ftp -O \"\${DISTDIR}/\${FILE}\" \"\${URI}\""
510 RESUMECOMMAND="wget -c -t 3 -T 60 --passive-ftp -O \"\${DISTDIR}/\${FILE}\" \"\${URI}\""
511
512 diff --git a/lib/_emerge/Binpkg.py b/lib/_emerge/Binpkg.py
513 index 001283611..15eb56092 100644
514 --- a/lib/_emerge/Binpkg.py
515 +++ b/lib/_emerge/Binpkg.py
516 @@ -12,6 +12,7 @@ from _emerge.EbuildMerge import EbuildMerge
517 from _emerge.EbuildBuildDir import EbuildBuildDir
518 from _emerge.SpawnProcess import SpawnProcess
519 from portage.eapi import eapi_exports_replace_vars
520 +from portage.exception import PortageException
521 from portage.output import colorize
522 from portage.util import ensure_dirs
523 from portage.util._async.AsyncTaskFuture import AsyncTaskFuture
524 @@ -417,9 +418,17 @@ class Binpkg(CompositeTask):
525
526 def _unpack_contents_exit(self, unpack_contents):
527 if self._default_exit(unpack_contents) != os.EX_OK:
528 - unpack_contents.future.result()
529 + try:
530 + unpack_contents.future.result()
531 + err = ""
532 + except PortageException as e:
533 + err = e
534 +
535 self._writemsg_level(
536 - "!!! Error Extracting '%s'\n" % self._pkg_path,
537 + colorize(
538 + "BAD",
539 + f"!!! Error Extracting '{self._pkg_path}', {err}\n",
540 + ),
541 noiselevel=-1,
542 level=logging.ERROR,
543 )
544
545 diff --git a/lib/_emerge/BinpkgExtractorAsync.py b/lib/_emerge/BinpkgExtractorAsync.py
546 index a0380a0a4..919837fc1 100644
547 --- a/lib/_emerge/BinpkgExtractorAsync.py
548 +++ b/lib/_emerge/BinpkgExtractorAsync.py
549 @@ -15,6 +15,8 @@ from portage.util import (
550 shlex_split,
551 varexpand,
552 )
553 +from portage.exception import InvalidBinaryPackageFormat
554 +from portage.binpkg import get_binpkg_format
555 import signal
556 import subprocess
557 import tarfile
558 @@ -27,6 +29,13 @@ class BinpkgExtractorAsync(SpawnProcess):
559 _shell_binary = portage.const.BASH_BINARY
560
561 def _start(self):
562 + binpkg_format = get_binpkg_format(self.pkg_path)
563 + if binpkg_format == "xpak":
564 + self._xpak_start()
565 + else:
566 + raise InvalidBinaryPackageFormat(self.pkg_path)
567 +
568 + def _xpak_start(self):
569 tar_options = ""
570 if "xattr" in self.features:
571 process = subprocess.Popen(
572
573 diff --git a/lib/_emerge/BinpkgFetcher.py b/lib/_emerge/BinpkgFetcher.py
574 index de3dd42ed..d5275ea11 100644
575 --- a/lib/_emerge/BinpkgFetcher.py
576 +++ b/lib/_emerge/BinpkgFetcher.py
577 @@ -11,6 +11,8 @@ import stat
578 import sys
579 import portage
580 from portage import os
581 +from portage.const import SUPPORTED_GENTOO_BINPKG_FORMATS
582 +from portage.exception import FileNotFound, InvalidBinaryPackageFormat
583 from portage.util._async.AsyncTaskFuture import AsyncTaskFuture
584 from portage.util._pty import _create_pty_or_pipe
585
586 @@ -21,8 +23,23 @@ class BinpkgFetcher(CompositeTask):
587
588 def __init__(self, **kwargs):
589 CompositeTask.__init__(self, **kwargs)
590 +
591 pkg = self.pkg
592 - self.pkg_path = pkg.root_config.trees["bintree"].getname(pkg.cpv) + ".partial"
593 + bintree = pkg.root_config.trees["bintree"]
594 + binpkg_path = None
595 +
596 + if bintree._remote_has_index:
597 + instance_key = bintree.dbapi._instance_key(pkg.cpv)
598 + binpkg_path = bintree._remotepkgs[instance_key].get("PATH")
599 + if binpkg_path:
600 + self.pkg_path = binpkg_path + ".partial"
601 + else:
602 + self.pkg_path = (
603 + pkg.root_config.trees["bintree"].getname(pkg.cpv, allocate_new=True)
604 + + ".partial"
605 + )
606 + else:
607 + raise FileNotFound("Binary packages index not found")
608
609 def _start(self):
610 fetcher = _BinpkgFetcherProcess(
611 @@ -106,15 +123,23 @@ class _BinpkgFetcherProcess(SpawnProcess):
612 resumecommand = None
613 if bintree._remote_has_index:
614 remote_metadata = bintree._remotepkgs[bintree.dbapi._instance_key(pkg.cpv)]
615 + binpkg_format = remote_metadata.get(
616 + "BINPKG_FORMAT", SUPPORTED_GENTOO_BINPKG_FORMATS[0]
617 + )
618 + if binpkg_format not in SUPPORTED_GENTOO_BINPKG_FORMATS:
619 + raise InvalidBinaryPackageFormat(binpkg_format)
620 rel_uri = remote_metadata.get("PATH")
621 if not rel_uri:
622 - rel_uri = pkg.cpv + ".tbz2"
623 + if binpkg_format == "xpak":
624 + rel_uri = pkg.cpv + ".tbz2"
625 + elif binpkg_format == "gpkg":
626 + rel_uri = pkg.cpv + ".gpkg.tar"
627 remote_base_uri = remote_metadata["BASE_URI"]
628 uri = remote_base_uri.rstrip("/") + "/" + rel_uri.lstrip("/")
629 fetchcommand = remote_metadata.get("FETCHCOMMAND")
630 resumecommand = remote_metadata.get("RESUMECOMMAND")
631 else:
632 - uri = settings["PORTAGE_BINHOST"].rstrip("/") + "/" + pkg.pf + ".tbz2"
633 + raise FileNotFound("Binary packages index not found")
634
635 if pretend:
636 portage.writemsg_stdout("\n%s\n" % uri, noiselevel=-1)
637
638 diff --git a/lib/_emerge/EbuildBinpkg.py b/lib/_emerge/EbuildBinpkg.py
639 index 5942d245a..ccdd30f7b 100644
640 --- a/lib/_emerge/EbuildBinpkg.py
641 +++ b/lib/_emerge/EbuildBinpkg.py
642 @@ -6,6 +6,8 @@ from _emerge.EbuildPhase import EbuildPhase
643
644 import portage
645 from portage import os
646 +from portage.const import SUPPORTED_GENTOO_BINPKG_FORMATS
647 +from portage.exception import InvalidBinaryPackageFormat
648
649
650 class EbuildBinpkg(CompositeTask):
651 @@ -19,9 +21,19 @@ class EbuildBinpkg(CompositeTask):
652 pkg = self.pkg
653 root_config = pkg.root_config
654 bintree = root_config.trees["bintree"]
655 - binpkg_tmpfile = os.path.join(
656 - bintree.pkgdir, pkg.cpv + ".tbz2." + str(portage.getpid())
657 + binpkg_format = self.settings.get(
658 + "BINPKG_FORMAT", SUPPORTED_GENTOO_BINPKG_FORMATS[0]
659 )
660 + if binpkg_format == "xpak":
661 + binpkg_tmpfile = os.path.join(
662 + bintree.pkgdir, pkg.cpv + ".tbz2." + str(portage.getpid())
663 + )
664 + elif binpkg_format == "gpkg":
665 + binpkg_tmpfile = os.path.join(
666 + bintree.pkgdir, pkg.cpv + ".gpkg.tar." + str(portage.getpid())
667 + )
668 + else:
669 + raise InvalidBinaryPackageFormat(binpkg_format)
670 bintree._ensure_dir(os.path.dirname(binpkg_tmpfile))
671
672 self._binpkg_tmpfile = binpkg_tmpfile
673
674 diff --git a/lib/_emerge/EbuildPhase.py b/lib/_emerge/EbuildPhase.py
675 index 12326fffd..9a04f9c1f 100644
676 --- a/lib/_emerge/EbuildPhase.py
677 +++ b/lib/_emerge/EbuildPhase.py
678 @@ -29,6 +29,8 @@ from portage.util._async.AsyncTaskFuture import AsyncTaskFuture
679 from portage.util._async.BuildLogger import BuildLogger
680 from portage.util.futures import asyncio
681 from portage.util.futures.executor.fork import ForkExecutor
682 +from portage.exception import InvalidBinaryPackageFormat
683 +from portage.const import SUPPORTED_GENTOO_BINPKG_FORMATS
684
685 try:
686 from portage.xml.metadata import MetaDataXML
687 @@ -157,14 +159,31 @@ class EbuildPhase(CompositeTask):
688
689 if self.phase == "package":
690 if "PORTAGE_BINPKG_TMPFILE" not in self.settings:
691 - self.settings["PORTAGE_BINPKG_TMPFILE"] = (
692 - os.path.join(
693 - self.settings["PKGDIR"],
694 - self.settings["CATEGORY"],
695 - self.settings["PF"],
696 - )
697 - + ".tbz2"
698 + binpkg_format = self.settings.get(
699 + "BINPKG_FORMAT", SUPPORTED_GENTOO_BINPKG_FORMATS[0]
700 )
701 + if binpkg_format == "xpak":
702 + self.settings["BINPKG_FORMAT"] = "xpak"
703 + self.settings["PORTAGE_BINPKG_TMPFILE"] = (
704 + os.path.join(
705 + self.settings["PKGDIR"],
706 + self.settings["CATEGORY"],
707 + self.settings["PF"],
708 + )
709 + + ".tbz2"
710 + )
711 + elif binpkg_format == "gpkg":
712 + self.settings["BINPKG_FORMAT"] = "gpkg"
713 + self.settings["PORTAGE_BINPKG_TMPFILE"] = (
714 + os.path.join(
715 + self.settings["PKGDIR"],
716 + self.settings["CATEGORY"],
717 + self.settings["PF"],
718 + )
719 + + ".gpkg.tar"
720 + )
721 + else:
722 + raise InvalidBinaryPackageFormat(binpkg_format)
723
724 def _async_start_exit(self, task):
725 task.future.cancelled() or task.future.result()
726
727 diff --git a/lib/_emerge/Package.py b/lib/_emerge/Package.py
728 index 90dfccdef..cc2cb07e1 100644
729 --- a/lib/_emerge/Package.py
730 +++ b/lib/_emerge/Package.py
731 @@ -63,6 +63,7 @@ class Package(Task):
732
733 metadata_keys = [
734 "BDEPEND",
735 + "BINPKG_FORMAT",
736 "BUILD_ID",
737 "BUILD_TIME",
738 "CHOST",
739
740 diff --git a/lib/_emerge/actions.py b/lib/_emerge/actions.py
741 index 515b22b66..0255e3e97 100644
742 --- a/lib/_emerge/actions.py
743 +++ b/lib/_emerge/actions.py
744 @@ -41,7 +41,7 @@ from portage.dbapi._expand_new_virt import expand_new_virt
745 from portage.dbapi.IndexedPortdb import IndexedPortdb
746 from portage.dbapi.IndexedVardb import IndexedVardb
747 from portage.dep import Atom, _repo_separator, _slot_separator
748 -from portage.exception import InvalidAtom, InvalidData, ParseError
749 +from portage.exception import InvalidAtom, InvalidData, ParseError, GPGException
750 from portage.output import (
751 colorize,
752 create_color_func,
753 @@ -77,6 +77,8 @@ from portage.sync.old_tree_timestamp import old_tree_timestamp_warn
754 from portage.localization import _
755 from portage.metadata import action_metadata
756 from portage.emaint.main import print_results
757 +from portage.gpg import GPG
758 +from portage.binpkg import get_binpkg_format
759
760 from _emerge.clear_caches import clear_caches
761 from _emerge.create_depgraph_params import create_depgraph_params
762 @@ -604,6 +606,28 @@ def action_build(
763 )
764 return 1
765
766 + # unlock GPG if needed
767 + if (
768 + need_write_bindb
769 + and (eroot in ebuild_eroots)
770 + and (
771 + "binpkg-signing"
772 + in trees[eroot]["root_config"].settings.features
773 + )
774 + ):
775 + portage.writemsg_stdout(">>> Unlocking GPG... ")
776 + sys.stdout.flush()
777 + gpg = GPG(trees[eroot]["root_config"].settings)
778 + try:
779 + gpg.unlock()
780 + except GPGException as e:
781 + writemsg_level(
782 + colorize("BAD", "!!! %s\n" % e),
783 + level=logging.ERROR,
784 + noiselevel=-1,
785 + )
786 + return 1
787 +
788 if "--resume" in myopts:
789 favorites = mtimedb["resume"]["favorites"]
790
791 @@ -2271,11 +2295,21 @@ def action_info(settings, trees, myopts, myfiles):
792 elif pkg_type == "ebuild":
793 ebuildpath = portdb.findname(pkg.cpv, myrepo=pkg.repo)
794 elif pkg_type == "binary":
795 - tbz2_file = bindb.bintree.getname(pkg.cpv)
796 + binpkg_file = bindb.bintree.getname(pkg.cpv)
797 ebuild_file_name = pkg.cpv.split("/")[1] + ".ebuild"
798 - ebuild_file_contents = portage.xpak.tbz2(tbz2_file).getfile(
799 - ebuild_file_name
800 - )
801 + binpkg_format = pkg.cpv._metadata.get("BINPKG_FORMAT", None)
802 + if not binpkg_format:
803 + binpkg_format = get_binpkg_format(binpkg_file)
804 + if binpkg_format == "xpak":
805 + ebuild_file_contents = portage.xpak.tbz2(binpkg_file).getfile(
806 + ebuild_file_name
807 + )
808 + elif binpkg_format == "gpkg":
809 + ebuild_file_contents = portage.gpkg.gpkg(
810 + settings, pkg.cpv, binpkg_file
811 + ).get_metadata(ebuild_file_name)
812 + else:
813 + continue
814 tmpdir = tempfile.mkdtemp()
815 ebuildpath = os.path.join(tmpdir, ebuild_file_name)
816 file = open(ebuildpath, "w")
817
818 diff --git a/lib/_emerge/depgraph.py b/lib/_emerge/depgraph.py
819 index f6549eba6..14a71a610 100644
820 --- a/lib/_emerge/depgraph.py
821 +++ b/lib/_emerge/depgraph.py
822 @@ -15,7 +15,12 @@ from itertools import chain
823 import portage
824 from portage import os
825 from portage import _unicode_decode, _unicode_encode, _encodings
826 -from portage.const import PORTAGE_PACKAGE_ATOM, USER_CONFIG_PATH, VCS_DIRS
827 +from portage.const import (
828 + PORTAGE_PACKAGE_ATOM,
829 + USER_CONFIG_PATH,
830 + VCS_DIRS,
831 + SUPPORTED_GPKG_EXTENSIONS,
832 +)
833 from portage.dbapi import dbapi
834 from portage.dbapi.dep_expand import dep_expand
835 from portage.dbapi.DummyTree import DummyTree
836 @@ -55,6 +60,7 @@ from portage.util.digraph import digraph
837 from portage.util.futures import asyncio
838 from portage.util._async.TaskScheduler import TaskScheduler
839 from portage.versions import _pkg_str, catpkgsplit
840 +from portage.binpkg import get_binpkg_format
841
842 from _emerge.AtomArg import AtomArg
843 from _emerge.Blocker import Blocker
844 @@ -4558,8 +4564,7 @@ class depgraph:
845 onlydeps = "--onlydeps" in self._frozen_config.myopts
846 lookup_owners = []
847 for x in myfiles:
848 - ext = os.path.splitext(x)[1]
849 - if ext == ".tbz2":
850 + if x.endswith(".tbz2") or x.endswith(SUPPORTED_GPKG_EXTENSIONS):
851 if not os.path.exists(x):
852 if os.path.exists(os.path.join(pkgsettings["PKGDIR"], "All", x)):
853 x = os.path.join(pkgsettings["PKGDIR"], "All", x)
854 @@ -4571,13 +4576,22 @@ class depgraph:
855 noiselevel=-1,
856 )
857 writemsg(
858 - "!!! Please ensure the tbz2 exists as specified.\n\n",
859 + "!!! Please ensure the binpkg exists as specified.\n\n",
860 noiselevel=-1,
861 )
862 return 0, myfavorites
863 - mytbz2 = portage.xpak.tbz2(x)
864 - mykey = None
865 - cat = mytbz2.getfile("CATEGORY")
866 + binpkg_format = get_binpkg_format(x)
867 + if binpkg_format == "xpak":
868 + mytbz2 = portage.xpak.tbz2(x)
869 + mykey = None
870 + cat = mytbz2.getfile("CATEGORY")
871 + elif binpkg_format == "gpkg":
872 + mygpkg = portage.gpkg.gpkg(self.frozen_config, None, x)
873 + mykey = None
874 + cat = mygpkg.get_metadata("CATEGORY")
875 + else:
876 + raise InvalidBinaryPackageFormat(x)
877 +
878 if cat is not None:
879 cat = _unicode_decode(
880 cat.strip(), encoding=_encodings["repo.content"]
881 @@ -4619,7 +4633,7 @@ class depgraph:
882 return 0, myfavorites
883
884 args.append(PackageArg(arg=x, package=pkg, root_config=root_config))
885 - elif ext == ".ebuild":
886 + elif x.endswith(".ebuild"):
887 ebuild_path = portage.util.normalize_path(os.path.abspath(x))
888 pkgdir = os.path.dirname(ebuild_path)
889 tree_root = os.path.dirname(os.path.dirname(pkgdir))
890
891 diff --git a/lib/portage/__init__.py b/lib/portage/__init__.py
892 index 13af8da09..3042de1aa 100644
893 --- a/lib/portage/__init__.py
894 +++ b/lib/portage/__init__.py
895 @@ -124,6 +124,7 @@ try:
896 + "cpv_getkey@getCPFromCPV,endversion_keys,"
897 + "suffix_value@endversion,pkgcmp,pkgsplit,vercmp,ververify",
898 "portage.xpak",
899 + "portage.gpkg",
900 "subprocess",
901 "time",
902 )
903
904 diff --git a/lib/portage/binpkg.py b/lib/portage/binpkg.py
905 new file mode 100644
906 index 000000000..ed2fda827
907 --- /dev/null
908 +++ b/lib/portage/binpkg.py
909 @@ -0,0 +1,56 @@
910 +# Copyright 2001-2020 Gentoo Authors
911 +# Distributed under the terms of the GNU General Public License v2
912 +
913 +import tarfile
914 +from portage.const import SUPPORTED_XPAK_EXTENSIONS, SUPPORTED_GPKG_EXTENSIONS
915 +from portage.output import colorize
916 +from portage.util import writemsg
917 +
918 +
919 +def get_binpkg_format(binpkg_path):
920 + if binpkg_path.endswith(SUPPORTED_XPAK_EXTENSIONS):
921 + file_ext_format = "xpak"
922 + elif binpkg_path.endswith(SUPPORTED_GPKG_EXTENSIONS):
923 + file_ext_format = "gpkg"
924 + else:
925 + file_ext_format = None
926 +
927 + try:
928 + with open(binpkg_path, "rb") as binpkg_file:
929 + header = binpkg_file.read(6)
930 + if header == b"gpkg-1":
931 + file_format = "gpkg"
932 + else:
933 + binpkg_file.seek(-16, 2)
934 + tail = binpkg_file.read(16)
935 + if (tail[0:8] == b"XPAKSTOP") and (tail[12:16] == b"STOP"):
936 + file_format = "xpak"
937 + else:
938 + file_format = None
939 +
940 + # check if wrong order gpkg
941 + if file_format is None:
942 + try:
943 + with tarfile.open(binpkg_path) as gpkg_tar:
944 + if "gpkg-1" in gpkg_tar.getnames():
945 + file_format = "gpkg"
946 + except tarfile.TarError:
947 + pass
948 +
949 + except OSError:
950 + file_format = None
951 +
952 + if file_format is None:
953 + return None
954 +
955 + if (file_ext_format is not None) and (file_ext_format != file_format):
956 + writemsg(
957 + colorize(
958 + "WARN",
959 + "File {} binpkg format mismatch, actual format: {}".format(
960 + binpkg_path, file_format
961 + ),
962 + )
963 + )
964 +
965 + return file_format
966
967 diff --git a/lib/portage/const.py b/lib/portage/const.py
968 index 1edc5fcf1..9e6f9311d 100644
969 --- a/lib/portage/const.py
970 +++ b/lib/portage/const.py
971 @@ -3,6 +3,7 @@
972 # Distributed under the terms of the GNU General Public License v2
973
974 import os
975 +import sys
976
977 # ===========================================================================
978 # START OF CONSTANTS -- START OF CONSTANTS -- START OF CONSTANTS -- START OF
979 @@ -128,8 +129,11 @@ SUPPORTED_FEATURES = frozenset(
980 "assume-digests",
981 "binpkg-docompress",
982 "binpkg-dostrip",
983 + "binpkg-ignore-signature",
984 "binpkg-logs",
985 "binpkg-multi-instance",
986 + "binpkg-request-signature",
987 + "binpkg-signing",
988 "buildpkg",
989 "buildpkg-live",
990 "buildsyspkg",
991 @@ -155,6 +159,7 @@ SUPPORTED_FEATURES = frozenset(
992 "force-mirror",
993 "force-prefix",
994 "getbinpkg",
995 + "gpg-keepalive",
996 "icecream",
997 "installsources",
998 "ipc-sandbox",
999 @@ -256,7 +261,14 @@ LIVE_ECLASSES = frozenset(
1000 )
1001
1002 SUPPORTED_BINPKG_FORMATS = ("tar", "rpm")
1003 +
1004 +if sys.version_info.major < 3:
1005 + SUPPORTED_GENTOO_BINPKG_FORMATS = ("xpak",)
1006 +else:
1007 + SUPPORTED_GENTOO_BINPKG_FORMATS = ("xpak", "gpkg")
1008 +
1009 SUPPORTED_XPAK_EXTENSIONS = (".tbz2", ".xpak")
1010 +SUPPORTED_GPKG_EXTENSIONS = (".gpkg.tar",)
1011
1012 # Time formats used in various places like metadata.chk.
1013 TIMESTAMP_FORMAT = "%a, %d %b %Y %H:%M:%S +0000" # to be used with time.gmtime()
1014
1015 diff --git a/lib/portage/dbapi/bintree.py b/lib/portage/dbapi/bintree.py
1016 index 9dbf9ee8b..8bfe5e97d 100644
1017 --- a/lib/portage/dbapi/bintree.py
1018 +++ b/lib/portage/dbapi/bintree.py
1019 @@ -26,20 +26,29 @@ portage.proxy.lazyimport.lazyimport(
1020
1021 from portage.binrepo.config import BinRepoConfigLoader
1022 from portage.cache.mappings import slot_dict_class
1023 -from portage.const import BINREPOS_CONF_FILE, CACHE_PATH, SUPPORTED_XPAK_EXTENSIONS
1024 +from portage.const import (
1025 + BINREPOS_CONF_FILE,
1026 + CACHE_PATH,
1027 + SUPPORTED_XPAK_EXTENSIONS,
1028 + SUPPORTED_GPKG_EXTENSIONS,
1029 + SUPPORTED_GENTOO_BINPKG_FORMATS,
1030 +)
1031 from portage.dbapi.virtual import fakedbapi
1032 from portage.dep import Atom, use_reduce, paren_enclose
1033 from portage.exception import (
1034 AlarmSignal,
1035 InvalidPackageName,
1036 + InvalidBinaryPackageFormat,
1037 ParseError,
1038 PortageException,
1039 + PortagePackageException,
1040 )
1041 from portage.localization import _
1042 from portage.package.ebuild.profile_iuse import iter_iuse_vars
1043 from portage.util.file_copy import copyfile
1044 from portage.util.futures import asyncio
1045 from portage.util.futures.executor.fork import ForkExecutor
1046 +from portage.binpkg import get_binpkg_format
1047 from portage import _movefile
1048 from portage import os
1049 from portage import _encodings
1050 @@ -49,6 +58,7 @@ from portage import _unicode_encode
1051 import codecs
1052 import errno
1053 import io
1054 +import re
1055 import stat
1056 import subprocess
1057 import tempfile
1058 @@ -72,6 +82,7 @@ class bindbapi(fakedbapi):
1059 list(fakedbapi._known_keys) + ["CHOST", "repository", "USE"]
1060 )
1061 _pkg_str_aux_keys = fakedbapi._pkg_str_aux_keys + (
1062 + "BINPKG_FORMAT",
1063 "BUILD_ID",
1064 "BUILD_TIME",
1065 "_mtime_",
1066 @@ -94,6 +105,7 @@ class bindbapi(fakedbapi):
1067 self._aux_cache_keys = set(
1068 [
1069 "BDEPEND",
1070 + "BINPKG_FORMAT",
1071 "BUILD_ID",
1072 "BUILD_TIME",
1073 "CHOST",
1074 @@ -170,28 +182,44 @@ class bindbapi(fakedbapi):
1075 return add_pkg._db.aux_get(add_pkg, wants)
1076 if not self.bintree._remotepkgs or not self.bintree.isremote(mycpv):
1077 try:
1078 - tbz2_path = self.bintree._pkg_paths[instance_key]
1079 + binpkg_path = self.bintree._pkg_paths[instance_key]
1080 except KeyError:
1081 raise KeyError(mycpv)
1082 - tbz2_path = os.path.join(self.bintree.pkgdir, tbz2_path)
1083 + binpkg_path = os.path.join(self.bintree.pkgdir, binpkg_path)
1084 try:
1085 - st = os.lstat(tbz2_path)
1086 + st = os.lstat(binpkg_path)
1087 except OSError:
1088 raise KeyError(mycpv)
1089 - metadata_bytes = portage.xpak.tbz2(tbz2_path).get_data()
1090 + binpkg_format = self.cpvdict[instance_key]["BINPKG_FORMAT"]
1091 + if binpkg_format == "xpak":
1092 + metadata_bytes = portage.xpak.tbz2(binpkg_path).get_data()
1093 + decode_metadata_name = False
1094 + elif binpkg_format == "gpkg":
1095 + metadata_bytes = portage.gpkg.gpkg(
1096 + self.settings, mycpv, binpkg_path
1097 + ).get_metadata()
1098 + decode_metadata_name = True
1099 + else:
1100 + raise InvalidBinaryPackageFormat(
1101 + "Unknown binary package format %s" % binpkg_path
1102 + )
1103
1104 def getitem(k):
1105 if k == "_mtime_":
1106 return str(st[stat.ST_MTIME])
1107 if k == "SIZE":
1108 return str(st.st_size)
1109 - v = metadata_bytes.get(
1110 - _unicode_encode(
1111 - k,
1112 - encoding=_encodings["repo.content"],
1113 - errors="backslashreplace",
1114 - )
1115 - )
1116 + else:
1117 + if decode_metadata_name:
1118 + v = metadata_bytes.get(k)
1119 + else:
1120 + v = metadata_bytes.get(
1121 + _unicode_encode(
1122 + k,
1123 + encoding=_encodings["repo.content"],
1124 + errors="backslashreplace",
1125 + )
1126 + )
1127 if v is not None:
1128 v = _unicode_decode(
1129 v, encoding=_encodings["repo.content"], errors="replace"
1130 @@ -202,6 +230,7 @@ class bindbapi(fakedbapi):
1131 getitem = self.cpvdict[instance_key].get
1132 mydata = {}
1133 mykeys = wants
1134 +
1135 for x in mykeys:
1136 myval = getitem(x)
1137 # myval is None if the key doesn't exist
1138 @@ -230,16 +259,29 @@ class bindbapi(fakedbapi):
1139 cpv = self._instance_key(cpv, support_string=True)[0]
1140 build_id = cpv.build_id
1141
1142 - tbz2path = self.bintree.getname(cpv)
1143 - if not os.path.exists(tbz2path):
1144 + binpkg_path = self.bintree.getname(cpv)
1145 + if not os.path.exists(binpkg_path):
1146 raise KeyError(cpv)
1147 - mytbz2 = portage.xpak.tbz2(tbz2path)
1148 - mydata = mytbz2.get_data()
1149
1150 - for k, v in values.items():
1151 - k = _unicode_encode(
1152 - k, encoding=_encodings["repo.content"], errors="backslashreplace"
1153 + binpkg_format = cpv.binpkg_format
1154 + if binpkg_format == "xpak":
1155 + mytbz2 = portage.xpak.tbz2(binpkg_path)
1156 + mydata = mytbz2.get_data()
1157 + encoding_key = True
1158 + elif binpkg_format == "gpkg":
1159 + mybinpkg = portage.gpkg.gpkg(self.settings, cpv, binpkg_path)
1160 + mydata = mybinpkg.get_metadata()
1161 + encoding_key = False
1162 + else:
1163 + raise InvalidBinaryPackageFormat(
1164 + "Unknown binary package format %s" % binpkg_path
1165 )
1166 +
1167 + for k, v in values.items():
1168 + if encoding_key:
1169 + k = _unicode_encode(
1170 + k, encoding=_encodings["repo.content"], errors="backslashreplace"
1171 + )
1172 v = _unicode_encode(
1173 v, encoding=_encodings["repo.content"], errors="backslashreplace"
1174 )
1175 @@ -248,7 +290,15 @@ class bindbapi(fakedbapi):
1176 for k, v in list(mydata.items()):
1177 if not v:
1178 del mydata[k]
1179 - mytbz2.recompose_mem(portage.xpak.xpak_mem(mydata))
1180 + if binpkg_format == "xpak":
1181 + mytbz2.recompose_mem(portage.xpak.xpak_mem(mydata))
1182 + elif binpkg_format == "gpkg":
1183 + mybinpkg.update_metadata(mydata)
1184 + else:
1185 + raise InvalidBinaryPackageFormat(
1186 + "Unknown binary package format %s" % binpkg_path
1187 + )
1188 +
1189 # inject will clear stale caches via cpv_inject.
1190 self.bintree.inject(cpv)
1191
1192 @@ -271,12 +321,24 @@ class bindbapi(fakedbapi):
1193 if add_pkg is not None:
1194 await add_pkg._db.unpack_metadata(pkg, dest_dir, loop=loop)
1195 else:
1196 - tbz2_file = self.bintree.getname(cpv)
1197 - await loop.run_in_executor(
1198 - ForkExecutor(loop=loop),
1199 - portage.xpak.tbz2(tbz2_file).unpackinfo,
1200 - dest_dir,
1201 - )
1202 + binpkg_file = self.bintree.getname(cpv)
1203 + binpkg_format = cpv.binpkg_format
1204 + if binpkg_format == "xpak":
1205 + await loop.run_in_executor(
1206 + ForkExecutor(loop=loop),
1207 + portage.xpak.tbz2(binpkg_file).unpackinfo,
1208 + dest_dir,
1209 + )
1210 + elif binpkg_format == "gpkg":
1211 + await loop.run_in_executor(
1212 + ForkExecutor(loop=loop),
1213 + portage.gpkg.gpkg(self.settings, cpv, binpkg_file).unpack_metadata,
1214 + dest_dir,
1215 + )
1216 + else:
1217 + raise InvalidBinaryPackageFormat(
1218 + "Unknown binary package format %s" % binpkg_file
1219 + )
1220
1221 async def unpack_contents(self, pkg, dest_dir, loop=None):
1222 """
1223 @@ -297,23 +359,31 @@ class bindbapi(fakedbapi):
1224
1225 pkg_path = self.bintree.getname(cpv)
1226 if pkg_path is not None:
1227 + binpkg_format = cpv.binpkg_format
1228 + if binpkg_format == "xpak":
1229 + extractor = BinpkgExtractorAsync(
1230 + background=settings.get("PORTAGE_BACKGROUND") == "1",
1231 + env=settings.environ(),
1232 + features=settings.features,
1233 + image_dir=dest_dir,
1234 + pkg=cpv,
1235 + pkg_path=pkg_path,
1236 + logfile=settings.get("PORTAGE_LOG_FILE"),
1237 + scheduler=SchedulerInterface(loop),
1238 + )
1239
1240 - extractor = BinpkgExtractorAsync(
1241 - background=settings.get("PORTAGE_BACKGROUND") == "1",
1242 - env=settings.environ(),
1243 - features=settings.features,
1244 - image_dir=dest_dir,
1245 - pkg=cpv,
1246 - pkg_path=pkg_path,
1247 - logfile=settings.get("PORTAGE_LOG_FILE"),
1248 - scheduler=SchedulerInterface(loop),
1249 - )
1250 -
1251 - extractor.start()
1252 - await extractor.async_wait()
1253 - if extractor.returncode != os.EX_OK:
1254 - raise PortageException("Error Extracting '{}'".format(pkg_path))
1255 -
1256 + extractor.start()
1257 + await extractor.async_wait()
1258 + if extractor.returncode != os.EX_OK:
1259 + raise PortageException("Error Extracting '{}'".format(pkg_path))
1260 + elif binpkg_format == "gpkg":
1261 + await loop.run_in_executor(
1262 + ForkExecutor(loop=loop),
1263 + portage.gpkg.gpkg(self.settings, cpv, pkg_path).decompress,
1264 + dest_dir,
1265 + )
1266 + else:
1267 + raise portage.exception.InvalidBinaryPackageFormat(pkg_path)
1268 else:
1269 instance_key = self._instance_key(cpv)
1270 add_pkg = self.bintree._additional_pkgs.get(instance_key)
1271 @@ -430,6 +500,7 @@ class binarytree:
1272 self._pkgindex_aux_keys = [
1273 "BASE_URI",
1274 "BDEPEND",
1275 + "BINPKG_FORMAT",
1276 "BUILD_ID",
1277 "BUILD_TIME",
1278 "CHOST",
1279 @@ -473,6 +544,7 @@ class binarytree:
1280 "ACCEPT_LICENSE",
1281 "ACCEPT_PROPERTIES",
1282 "ACCEPT_RESTRICT",
1283 + "BINPKG_FORMAT",
1284 "CBUILD",
1285 "CONFIG_PROTECT",
1286 "CONFIG_PROTECT_MASK",
1287 @@ -508,10 +580,13 @@ class binarytree:
1288 "SLOT": "0",
1289 "USE": "",
1290 }
1291 - self._pkgindex_inherited_keys = ["CHOST", "repository"]
1292 + self._pkgindex_inherited_keys = ["BINPKG_FORMAT", "CHOST", "repository"]
1293
1294 # Populate the header with appropriate defaults.
1295 self._pkgindex_default_header_data = {
1296 + "BINPKG_FORMAT": self.settings.get(
1297 + "BINPKG_FORMAT", SUPPORTED_GENTOO_BINPKG_FORMATS[0]
1298 + ),
1299 "CHOST": self.settings.get("CHOST", ""),
1300 "repository": "",
1301 }
1302 @@ -594,24 +669,42 @@ class binarytree:
1303 writemsg("!!! " + mycpv + " -> " + mynewcpv + "\n", noiselevel=-1)
1304 continue
1305
1306 - tbz2path = self.getname(mycpv)
1307 - if os.path.exists(tbz2path) and not os.access(tbz2path, os.W_OK):
1308 + binpkg_path = self.getname(mycpv)
1309 + if os.path.exists(binpkg_path) and not os.access(binpkg_path, os.W_OK):
1310 writemsg(
1311 _("!!! Cannot update readonly binary: %s\n") % mycpv, noiselevel=-1
1312 )
1313 continue
1314
1315 moves += 1
1316 - mytbz2 = portage.xpak.tbz2(tbz2path)
1317 - mydata = mytbz2.get_data()
1318 + binpkg_format = mycpv.binpkg_format
1319 + if binpkg_format == "xpak":
1320 + mytbz2 = portage.xpak.tbz2(binpkg_path)
1321 + mydata = mytbz2.get_data()
1322 + decode_metadata_name = False
1323 + elif binpkg_format == "gpkg":
1324 + mybinpkg = portage.gpkg.gpkg(self.settings, mycpv, binpkg_path)
1325 + mydata = mybinpkg.get_metadata()
1326 + decode_metadata_name = True
1327 + else:
1328 + continue
1329 +
1330 updated_items = update_dbentries([mylist], mydata, parent=mycpv)
1331 mydata.update(updated_items)
1332 - mydata[b"PF"] = _unicode_encode(
1333 - mynewpkg + "\n", encoding=_encodings["repo.content"]
1334 - )
1335 - mydata[b"CATEGORY"] = _unicode_encode(
1336 - mynewcat + "\n", encoding=_encodings["repo.content"]
1337 - )
1338 + if decode_metadata_name:
1339 + mydata["PF"] = _unicode_encode(
1340 + mynewpkg + "\n", encoding=_encodings["repo.content"]
1341 + )
1342 + mydata["CATEGORY"] = _unicode_encode(
1343 + mynewcat + "\n", encoding=_encodings["repo.content"]
1344 + )
1345 + else:
1346 + mydata[b"PF"] = _unicode_encode(
1347 + mynewpkg + "\n", encoding=_encodings["repo.content"]
1348 + )
1349 + mydata[b"CATEGORY"] = _unicode_encode(
1350 + mynewcat + "\n", encoding=_encodings["repo.content"]
1351 + )
1352 if mynewpkg != myoldpkg:
1353 ebuild_data = mydata.pop(
1354 _unicode_encode(
1355 @@ -628,7 +721,7 @@ class binarytree:
1356
1357 metadata = self.dbapi._aux_cache_slot_dict()
1358 for k in self.dbapi._aux_cache_keys:
1359 - v = mydata.get(_unicode_encode(k))
1360 + v = mydata.get(k if decode_metadata_name else _unicode_encode(k))
1361 if v is not None:
1362 v = _unicode_decode(v)
1363 metadata[k] = " ".join(v.split())
1364 @@ -643,9 +736,15 @@ class binarytree:
1365 update_path_lock = None
1366 try:
1367 update_path_lock = lockfile(update_path, wantnewlockfile=True)
1368 - copyfile(tbz2path, update_path)
1369 - mytbz2 = portage.xpak.tbz2(update_path)
1370 - mytbz2.recompose_mem(portage.xpak.xpak_mem(mydata))
1371 + copyfile(binpkg_path, update_path)
1372 + if binpkg_format == "xpak":
1373 + mytbz2 = portage.xpak.tbz2(update_path)
1374 + mytbz2.recompose_mem(portage.xpak.xpak_mem(mydata))
1375 + elif binpkg_format == "gpkg":
1376 + mybinpkg = portage.gpkg.gpkg(self.settings, mycpv, update_path)
1377 + mybinpkg.update_metadata(mydata, newcpv=mynewcpv)
1378 + else:
1379 + raise InvalidBinaryPackageFormat(binpkg_format)
1380 self.inject(mynewcpv, filename=update_path)
1381 finally:
1382 if update_path_lock is not None:
1383 @@ -810,7 +909,13 @@ class binarytree:
1384 metadata[_instance_key(cpv)] = d
1385 path = d.get("PATH")
1386 if not path:
1387 - path = cpv + ".tbz2"
1388 + binpkg_format = d["BINPKG_FORMAT"]
1389 + if binpkg_format == "xpak":
1390 + path = cpv + ".tbz2"
1391 + elif binpkg_format == "gpkg":
1392 + path = cpv + ".gpkg.tar"
1393 + else:
1394 + continue
1395
1396 if reindex:
1397 basename = os.path.basename(path)
1398 @@ -835,7 +940,9 @@ class binarytree:
1399 )
1400 except UnicodeDecodeError:
1401 continue
1402 - if not myfile.endswith(SUPPORTED_XPAK_EXTENSIONS):
1403 + if not myfile.endswith(
1404 + SUPPORTED_XPAK_EXTENSIONS + SUPPORTED_GPKG_EXTENSIONS
1405 + ):
1406 continue
1407 mypath = os.path.join(mydir, myfile)
1408 full_path = os.path.join(self.pkgdir, mypath)
1409 @@ -846,9 +953,9 @@ class binarytree:
1410
1411 # Validate data from the package index and try to avoid
1412 # reading the xpak if possible.
1413 + match = None
1414 possibilities = basename_index.get(myfile)
1415 if possibilities:
1416 - match = None
1417 for d in possibilities:
1418 try:
1419 if int(d["_mtime_"]) != s[stat.ST_MTIME]:
1420 @@ -873,7 +980,9 @@ class binarytree:
1421 update_pkgindex = True
1422 # Omit PATH if it is the default path for
1423 # the current Packages format version.
1424 - if mypath != mycpv + ".tbz2":
1425 + if (mypath != mycpv + ".tbz2") and (
1426 + mypath != mycpv + ".gpkg.tar"
1427 + ):
1428 d["PATH"] = mypath
1429 if not oldpath:
1430 update_pkgindex = True
1431 @@ -891,15 +1000,30 @@ class binarytree:
1432 )
1433 self.invalids.append(myfile[:-5])
1434 continue
1435 - pkg_metadata = self._read_metadata(
1436 - full_path,
1437 - s,
1438 - keys=chain(self.dbapi._aux_cache_keys, ("PF", "CATEGORY")),
1439 - )
1440 +
1441 + binpkg_format = None
1442 + if match:
1443 + binpkg_format = match.get("BINPKG_FORMAT", None)
1444 + try:
1445 + pkg_metadata = self._read_metadata(
1446 + full_path,
1447 + s,
1448 + keys=chain(self.dbapi._aux_cache_keys, ("PF", "CATEGORY")),
1449 + binpkg_format=binpkg_format,
1450 + )
1451 + except PortagePackageException as e:
1452 + writemsg(
1453 + f"!!! Invalid binary package: '{full_path}', {e}\n",
1454 + noiselevel=-1,
1455 + )
1456 + continue
1457 mycat = pkg_metadata.get("CATEGORY", "")
1458 mypf = pkg_metadata.get("PF", "")
1459 slot = pkg_metadata.get("SLOT", "")
1460 - mypkg = myfile[:-5]
1461 + for ext in SUPPORTED_XPAK_EXTENSIONS + SUPPORTED_GPKG_EXTENSIONS:
1462 + if myfile.endswith(ext):
1463 + mypkg = myfile[: -len(ext)]
1464 + break
1465 if not mycat or not mypf or not slot:
1466 # old-style or corrupt package
1467 writemsg(
1468 @@ -943,9 +1067,19 @@ class binarytree:
1469 invalid_name = True
1470 else:
1471 mypkg = mypkg[: -len(str(build_id)) - 1]
1472 + elif myfile.endswith(".gpkg.tar"):
1473 + build_id = self._parse_build_id(myfile)
1474 + if build_id > 0:
1475 + multi_instance = True
1476 + if myfile != "%s-%s.gpkg.tar" % (mypf, build_id):
1477 + invalid_name = True
1478 + else:
1479 + mypkg = mypkg[: -len(str(build_id)) - 1]
1480 + else:
1481 + if myfile != "%s.gpkg.tar" % mypf:
1482 + invalid_name = True
1483 elif myfile != mypf + ".tbz2":
1484 invalid_name = True
1485 -
1486 if invalid_name:
1487 writemsg(
1488 _("\n!!! Binary package name is " "invalid: '%s'\n")
1489 @@ -1043,7 +1177,7 @@ class binarytree:
1490 del pkg_paths[_instance_key(mycpv)]
1491
1492 # record location if it's non-default
1493 - if mypath != mycpv + ".tbz2":
1494 + if (mypath != mycpv + ".tbz2") and (mypath != mycpv + ".gpkg.tar"):
1495 d["PATH"] = mypath
1496 else:
1497 d.pop("PATH", None)
1498 @@ -1407,7 +1541,10 @@ class binarytree:
1499 noiselevel=-1,
1500 )
1501 return
1502 +
1503 metadata = self._read_metadata(full_path, s)
1504 + binpkg_format = metadata["BINPKG_FORMAT"]
1505 +
1506 invalid_depend = False
1507 try:
1508 self._eval_use_flags(cpv, metadata)
1509 @@ -1455,21 +1592,35 @@ class binarytree:
1510
1511 basename = os.path.basename(full_path)
1512 pf = catsplit(cpv)[1]
1513 - if build_id is None and not fetched and basename.endswith(".xpak"):
1514 + if (build_id is None) and (not fetched) and binpkg_format:
1515 # Apply the newly assigned BUILD_ID. This is intended
1516 # to occur only for locally built packages. If the
1517 # package was fetched, we want to preserve its
1518 # attributes, so that we can later distinguish that it
1519 # is identical to its remote counterpart.
1520 build_id = self._parse_build_id(basename)
1521 - metadata["BUILD_ID"] = str(build_id)
1522 - cpv = _pkg_str(
1523 - cpv, metadata=metadata, settings=self.settings, db=self.dbapi
1524 - )
1525 - binpkg = portage.xpak.tbz2(full_path)
1526 - binary_data = binpkg.get_data()
1527 - binary_data[b"BUILD_ID"] = _unicode_encode(metadata["BUILD_ID"])
1528 - binpkg.recompose_mem(portage.xpak.xpak_mem(binary_data))
1529 + if build_id > 0:
1530 + metadata["BUILD_ID"] = str(build_id)
1531 + cpv = _pkg_str(
1532 + cpv, metadata=metadata, settings=self.settings, db=self.dbapi
1533 + )
1534 + if binpkg_format == "xpak":
1535 + if basename.endswith(".xpak"):
1536 + binpkg = portage.xpak.tbz2(full_path)
1537 + binary_data = binpkg.get_data()
1538 + binary_data[b"BUILD_ID"] = _unicode_encode(
1539 + metadata["BUILD_ID"]
1540 + )
1541 + binpkg.recompose_mem(portage.xpak.xpak_mem(binary_data))
1542 + elif binpkg_format == "gpkg":
1543 + binpkg = portage.gpkg.gpkg(self.settings, cpv, full_path)
1544 + binpkg_metadata = binpkg.get_metadata()
1545 + binpkg_metadata["BUILD_ID"] = _unicode_encode(
1546 + metadata["BUILD_ID"]
1547 + )
1548 + binpkg.update_metadata(binpkg_metadata)
1549 + else:
1550 + raise InvalidBinaryPackageFormat(basename)
1551
1552 self._file_permissions(full_path)
1553 pkgindex = self._load_pkgindex()
1554 @@ -1490,7 +1641,7 @@ class binarytree:
1555
1556 return cpv
1557
1558 - def _read_metadata(self, filename, st, keys=None):
1559 + def _read_metadata(self, filename, st, keys=None, binpkg_format=None):
1560 """
1561 Read metadata from a binary package. The returned metadata
1562 dictionary will contain empty strings for any values that
1563 @@ -1511,14 +1662,35 @@ class binarytree:
1564 metadata = self.dbapi._aux_cache_slot_dict()
1565 else:
1566 metadata = {}
1567 - binary_metadata = portage.xpak.tbz2(filename).get_data()
1568 +
1569 + # xpak return key as binary, gpkg return key as str
1570 + decode_metadata_name = True
1571 +
1572 + if not binpkg_format:
1573 + binpkg_format = get_binpkg_format(filename)
1574 + if binpkg_format == "xpak":
1575 + binpkg_metadata = portage.xpak.tbz2(filename).get_data()
1576 + elif binpkg_format == "gpkg":
1577 + binpkg_metadata = portage.gpkg.gpkg(
1578 + self.settings, None, filename
1579 + ).get_metadata()
1580 + decode_metadata_name = False
1581 + else:
1582 + raise InvalidBinaryPackageFormat(
1583 + f"Unrecognized binary package format in '{filename}'"
1584 + )
1585 +
1586 for k in keys:
1587 if k == "_mtime_":
1588 metadata[k] = str(st[stat.ST_MTIME])
1589 elif k == "SIZE":
1590 metadata[k] = str(st.st_size)
1591 else:
1592 - v = binary_metadata.get(_unicode_encode(k))
1593 + if decode_metadata_name:
1594 + v = binpkg_metadata.get(_unicode_encode(k))
1595 + else:
1596 + # check gpkg
1597 + v = binpkg_metadata.get(k)
1598 if v is None:
1599 if k == "EAPI":
1600 metadata[k] = "0"
1601 @@ -1527,6 +1699,9 @@ class binarytree:
1602 else:
1603 v = _unicode_decode(v)
1604 metadata[k] = " ".join(v.split())
1605 +
1606 + metadata["BINPKG_FORMAT"] = binpkg_format
1607 +
1608 return metadata
1609
1610 def _inject_file(self, pkgindex, cpv, filename):
1611 @@ -1608,6 +1783,10 @@ class binarytree:
1612 """
1613
1614 pkg_path = self.getname(cpv)
1615 + try:
1616 + binpkg_format = cpv.binpkg_format
1617 + except AttributeError:
1618 + raise KeyError("{} metadata not found!".format(cpv))
1619
1620 d = dict(cpv._metadata.items())
1621 d.update(perform_multiple_checksums(pkg_path, hashes=self._pkgindex_hashes))
1622 @@ -1616,11 +1795,18 @@ class binarytree:
1623 st = os.lstat(pkg_path)
1624 d["_mtime_"] = str(st[stat.ST_MTIME])
1625 d["SIZE"] = str(st.st_size)
1626 + d["BINPKG_FORMAT"] = binpkg_format
1627
1628 rel_path = pkg_path[len(self.pkgdir) + 1 :]
1629 # record location if it's non-default
1630 - if rel_path != cpv + ".tbz2":
1631 - d["PATH"] = rel_path
1632 + if binpkg_format == "xpak":
1633 + if rel_path != cpv + ".tbz2":
1634 + d["PATH"] = rel_path
1635 + elif binpkg_format == "gpkg":
1636 + if rel_path != cpv + ".gpkg.tar":
1637 + d["PATH"] = rel_path
1638 + else:
1639 + raise InvalidBinaryPackageFormat(binpkg_format)
1640
1641 return d
1642
1643 @@ -1822,11 +2008,45 @@ class binarytree:
1644 return None
1645
1646 if filename is None:
1647 - if self._multi_instance:
1648 - pf = catsplit(cpv)[1]
1649 - filename = "%s-%s.xpak" % (os.path.join(self.pkgdir, cpv.cp, pf), "1")
1650 + try:
1651 + binpkg_format = cpv.binpkg_format
1652 + except AttributeError:
1653 + # In order to force the caller to clarify its intent, do not
1654 + # use default BINPKG_FORMAT unless allocate_new is True.
1655 + # The caller can set cpv.binpkg_format in advance if something
1656 + # other than the default is desired here.
1657 + if allocate_new:
1658 + binpkg_format = self.settings.get(
1659 + "BINPKG_FORMAT", SUPPORTED_GENTOO_BINPKG_FORMATS[0]
1660 + )
1661 + else:
1662 + binpkg_format = None
1663 +
1664 + if not binpkg_format:
1665 + # Raise an error if the desired binpkg_format is not clear.
1666 + # The caller should either set allocate_new to True or else
1667 + # ensure that cpv.binpkg_format is set to a particular format.
1668 + raise InvalidBinaryPackageFormat(binpkg_format)
1669 + elif binpkg_format == "xpak":
1670 + if self._multi_instance:
1671 + pf = catsplit(cpv)[1]
1672 + filename = "%s-%s.xpak" % (
1673 + os.path.join(self.pkgdir, cpv.cp, pf),
1674 + "1",
1675 + )
1676 + else:
1677 + filename = os.path.join(self.pkgdir, cpv + ".tbz2")
1678 + elif binpkg_format == "gpkg":
1679 + if self._multi_instance:
1680 + pf = catsplit(cpv)[1]
1681 + filename = "%s-%s.gpkg.tar" % (
1682 + os.path.join(self.pkgdir, cpv.cp, pf),
1683 + "1",
1684 + )
1685 + else:
1686 + filename = os.path.join(self.pkgdir, cpv + ".gpkg.tar")
1687 else:
1688 - filename = os.path.join(self.pkgdir, cpv + ".tbz2")
1689 + raise InvalidBinaryPackageFormat(binpkg_format)
1690
1691 return filename
1692
1693 @@ -1850,7 +2070,19 @@ class binarytree:
1694 return max_build_id
1695
1696 def _allocate_filename(self, cpv):
1697 - return os.path.join(self.pkgdir, cpv + ".tbz2")
1698 + try:
1699 + binpkg_format = cpv.binpkg_format
1700 + except AttributeError:
1701 + binpkg_format = self.settings.get(
1702 + "BINPKG_FORMAT", SUPPORTED_GENTOO_BINPKG_FORMATS[0]
1703 + )
1704 +
1705 + if binpkg_format == "xpak":
1706 + return os.path.join(self.pkgdir, cpv + ".tbz2")
1707 + elif binpkg_format == "gpkg":
1708 + return os.path.join(self.pkgdir, cpv + ".gpkg.tar")
1709 + else:
1710 + raise InvalidBinaryPackageFormat(binpkg_format)
1711
1712 def _allocate_filename_multi(self, cpv):
1713
1714 @@ -1864,8 +2096,25 @@ class binarytree:
1715 pf = catsplit(cpv)[1]
1716 build_id = max_build_id + 1
1717
1718 + try:
1719 + binpkg_format = cpv.binpkg_format
1720 + except AttributeError:
1721 + binpkg_format = self.settings.get(
1722 + "BINPKG_FORMAT", SUPPORTED_GENTOO_BINPKG_FORMATS[0]
1723 + )
1724 +
1725 + if binpkg_format == "xpak":
1726 + filename_format = "%s-%s.xpak"
1727 + elif binpkg_format == "gpkg":
1728 + filename_format = "%s-%s.gpkg.tar"
1729 + else:
1730 + raise InvalidBinaryPackageFormat(binpkg_format)
1731 +
1732 while True:
1733 - filename = "%s-%s.xpak" % (os.path.join(self.pkgdir, cpv.cp, pf), build_id)
1734 + filename = filename_format % (
1735 + os.path.join(self.pkgdir, cpv.cp, pf),
1736 + build_id,
1737 + )
1738 if os.path.exists(filename):
1739 build_id += 1
1740 else:
1741 @@ -1874,13 +2123,17 @@ class binarytree:
1742 @staticmethod
1743 def _parse_build_id(filename):
1744 build_id = -1
1745 - suffixlen = len(".xpak")
1746 - hyphen = filename.rfind("-", 0, -(suffixlen + 1))
1747 - if hyphen != -1:
1748 - try:
1749 - build_id = int(filename[hyphen + 1 : -suffixlen])
1750 - except ValueError:
1751 - pass
1752 + if filename.endswith(SUPPORTED_XPAK_EXTENSIONS):
1753 + suffixlen = len(".xpak")
1754 + elif filename.endswith(SUPPORTED_GPKG_EXTENSIONS):
1755 + suffixlen = len(".gpkg.tar")
1756 + else:
1757 + raise InvalidBinaryPackageFormat(filename)
1758 +
1759 + filename = filename[:-suffixlen]
1760 + if re.match(r".*-[\w.]*\d+[\w.]*-\d+$", filename):
1761 + build_id = int(filename.split("-")[-1])
1762 +
1763 return build_id
1764
1765 def isremote(self, pkgname):
1766
1767 diff --git a/lib/portage/dbapi/vartree.py b/lib/portage/dbapi/vartree.py
1768 index 8ffb23b1c..863efe9cc 100644
1769 --- a/lib/portage/dbapi/vartree.py
1770 +++ b/lib/portage/dbapi/vartree.py
1771 @@ -43,6 +43,7 @@ portage.proxy.lazyimport.lazyimport(
1772 "portage.util._eventloop.global_event_loop:global_event_loop",
1773 "portage.versions:best,catpkgsplit,catsplit,cpv_getkey,vercmp,"
1774 + "_get_slot_re,_pkgsplit@pkgsplit,_pkg_str,_unknown_repo",
1775 + "portage.gpkg",
1776 "subprocess",
1777 "tarfile",
1778 )
1779 @@ -54,6 +55,7 @@ from portage.const import (
1780 PORTAGE_PACKAGE_ATOM,
1781 PRIVATE_PATH,
1782 VDB_PATH,
1783 + SUPPORTED_GENTOO_BINPKG_FORMATS,
1784 )
1785 from portage.dbapi import dbapi
1786 from portage.exception import (
1787 @@ -61,6 +63,7 @@ from portage.exception import (
1788 InvalidData,
1789 InvalidLocation,
1790 InvalidPackageName,
1791 + InvalidBinaryPackageFormat,
1792 FileNotFound,
1793 PermissionDenied,
1794 UnsupportedAPIException,
1795 @@ -1089,26 +1092,45 @@ class vardbapi(dbapi):
1796 "y" if include_unmodified_config else "n"
1797 )
1798 )
1799 -
1800 opts, args = parser.parse_known_args(opts_list)
1801
1802 - tar_cmd = ("tar", "-x", "--xattrs", "--xattrs-include=*", "-C", dest_dir)
1803 - pr, pw = os.pipe()
1804 - proc = await asyncio.create_subprocess_exec(*tar_cmd, stdin=pr)
1805 - os.close(pr)
1806 - with os.fdopen(pw, "wb", 0) as pw_file:
1807 + binpkg_format = settings.get(
1808 + "BINPKG_FORMAT", SUPPORTED_GENTOO_BINPKG_FORMATS[0]
1809 + )
1810 + if binpkg_format == "xpak":
1811 + tar_cmd = ("tar", "-x", "--xattrs", "--xattrs-include=*", "-C", dest_dir)
1812 + pr, pw = os.pipe()
1813 + proc = await asyncio.create_subprocess_exec(*tar_cmd, stdin=pr)
1814 + os.close(pr)
1815 + with os.fdopen(pw, "wb", 0) as pw_file:
1816 + excluded_config_files = await loop.run_in_executor(
1817 + ForkExecutor(loop=loop),
1818 + functools.partial(
1819 + self._dblink(cpv).quickpkg,
1820 + pw_file,
1821 + include_config=opts.include_config == "y",
1822 + include_unmodified_config=opts.include_unmodified_config == "y",
1823 + ),
1824 + )
1825 + await proc.wait()
1826 + if proc.returncode != os.EX_OK:
1827 + raise PortageException("command failed: {}".format(tar_cmd))
1828 + elif binpkg_format == "gpkg":
1829 + gpkg_tmp_fd, gpkg_tmp = tempfile.mkstemp(suffix=".gpkg.tar")
1830 + os.close(gpkg_tmp_fd)
1831 excluded_config_files = await loop.run_in_executor(
1832 ForkExecutor(loop=loop),
1833 functools.partial(
1834 self._dblink(cpv).quickpkg,
1835 - pw_file,
1836 + gpkg_tmp,
1837 include_config=opts.include_config == "y",
1838 include_unmodified_config=opts.include_unmodified_config == "y",
1839 ),
1840 )
1841 - await proc.wait()
1842 - if proc.returncode != os.EX_OK:
1843 - raise PortageException("command failed: {}".format(tar_cmd))
1844 + portage.gpkg.gpkg(settings, cpv, gpkg_tmp).decompress(dest_dir)
1845 + os.remove(gpkg_tmp)
1846 + else:
1847 + raise InvalidBinaryPackageFormat(binpkg_format)
1848
1849 if excluded_config_files:
1850 log_lines = [
1851 @@ -2107,7 +2129,11 @@ class dblink:
1852 return pkgfiles
1853
1854 def quickpkg(
1855 - self, output_file, include_config=False, include_unmodified_config=False
1856 + self,
1857 + output_file,
1858 + metadata=None,
1859 + include_config=False,
1860 + include_unmodified_config=False,
1861 ):
1862 """
1863 Create a tar file appropriate for use by quickpkg.
1864 @@ -2130,6 +2156,9 @@ class dblink:
1865 contents = self.getcontents()
1866 excluded_config_files = []
1867 protect = None
1868 + binpkg_format = settings.get(
1869 + "BINPKG_FORMAT", SUPPORTED_GENTOO_BINPKG_FORMATS[0]
1870 + )
1871
1872 if not include_config:
1873 confprot = ConfigProtect(
1874 @@ -2152,16 +2181,22 @@ class dblink:
1875 excluded_config_files.append(filename)
1876 return True
1877
1878 - # The tarfile module will write pax headers holding the
1879 - # xattrs only if PAX_FORMAT is specified here.
1880 - with tarfile.open(
1881 - fileobj=output_file,
1882 - mode="w|",
1883 - format=tarfile.PAX_FORMAT if xattrs else tarfile.DEFAULT_FORMAT,
1884 - ) as tar:
1885 - tar_contents(
1886 - contents, settings["ROOT"], tar, protect=protect, xattrs=xattrs
1887 - )
1888 + if binpkg_format == "xpak":
1889 + # The tarfile module will write pax headers holding the
1890 + # xattrs only if PAX_FORMAT is specified here.
1891 + with tarfile.open(
1892 + fileobj=output_file,
1893 + mode="w|",
1894 + format=tarfile.PAX_FORMAT if xattrs else tarfile.DEFAULT_FORMAT,
1895 + ) as tar:
1896 + tar_contents(
1897 + contents, settings["ROOT"], tar, protect=protect, xattrs=xattrs
1898 + )
1899 + elif binpkg_format == "gpkg":
1900 + gpkg_file = portage.gpkg.gpkg(settings, cpv, output_file)
1901 + gpkg_file._quickpkg(contents, metadata, settings["ROOT"], protect=protect)
1902 + else:
1903 + raise InvalidBinaryPackageFormat(binpkg_format)
1904
1905 return excluded_config_files
1906
1907
1908 diff --git a/lib/portage/exception.py b/lib/portage/exception.py
1909 index ec8ea1980..3df4ce8fd 100644
1910 --- a/lib/portage/exception.py
1911 +++ b/lib/portage/exception.py
1912 @@ -181,6 +181,22 @@ class InvalidPackageName(PortagePackageException):
1913 """Malformed package name"""
1914
1915
1916 +class InvalidBinaryPackageFormat(PortagePackageException):
1917 + """Invalid Binary Package Format"""
1918 +
1919 +
1920 +class InvalidCompressionMethod(PortagePackageException):
1921 + """Invalid or unsupported compression method"""
1922 +
1923 +
1924 +class CompressorNotFound(PortagePackageException):
1925 + """A required compressor binary was not available or executable"""
1926 +
1927 +
1928 +class CompressorOperationFailed(PortagePackageException):
1929 + """An error occurred during external operation"""
1930 +
1931 +
1932 class InvalidAtom(PortagePackageException):
1933 """Malformed atom spec"""
1934
1935 @@ -208,6 +224,10 @@ class UnsupportedAPIException(PortagePackageException):
1936 return _unicode_decode(msg, encoding=_encodings["content"], errors="replace")
1937
1938
1939 +class GPGException(PortageException):
1940 + """GPG operation failed"""
1941 +
1942 +
1943 class SignatureException(PortageException):
1944 """Signature was not present in the checked file"""
1945
1946
1947 diff --git a/lib/portage/gpg.py b/lib/portage/gpg.py
1948 new file mode 100644
1949 index 000000000..57be2ebc0
1950 --- /dev/null
1951 +++ b/lib/portage/gpg.py
1952 @@ -0,0 +1,106 @@
1953 +# Copyright 2001-2020 Gentoo Authors
1954 +# Distributed under the terms of the GNU General Public License v2
1955 +
1956 +import subprocess
1957 +import sys
1958 +import threading
1959 +import time
1960 +
1961 +from portage import os
1962 +from portage.const import SUPPORTED_GENTOO_BINPKG_FORMATS
1963 +from portage.exception import GPGException
1964 +from portage.output import colorize
1965 +from portage.util import shlex_split, varexpand, writemsg, writemsg_stdout
1966 +
1967 +
1968 +class GPG:
1969 + """
1970 + Unlock GPG, must call dircetly from main program for get correct TTY
1971 + """
1972 +
1973 + def __init__(self, settings):
1974 + """
1975 + Portage settings are needed to run GPG unlock command.
1976 + """
1977 + self.settings = settings
1978 + self.thread = None
1979 + self.GPG_signing_base_command = self.settings.get(
1980 + "BINPKG_GPG_SIGNING_BASE_COMMAND"
1981 + )
1982 + self.digest_algo = self.settings.get("BINPKG_GPG_SIGNING_DIGEST")
1983 + self.signing_gpg_home = self.settings.get("BINPKG_GPG_SIGNING_GPG_HOME")
1984 + self.signing_gpg_key = self.settings.get("BINPKG_GPG_SIGNING_KEY")
1985 + self.GPG_unlock_command = self.GPG_signing_base_command.replace(
1986 + "[PORTAGE_CONFIG]",
1987 + f"--homedir {self.signing_gpg_home} "
1988 + f"--digest-algo {self.digest_algo} "
1989 + f"--local-user {self.signing_gpg_key} "
1990 + "--output /dev/null /dev/null",
1991 + )
1992 +
1993 + if "gpg-keepalive" in self.settings.features:
1994 + self.keepalive = True
1995 + else:
1996 + self.keepalive = False
1997 +
1998 + def unlock(self):
1999 + """
2000 + Set GPG_TTY and run GPG unlock command.
2001 + If gpg-keepalive is set, start keepalive thread.
2002 + """
2003 + if self.GPG_unlock_command and (
2004 + self.settings.get("BINPKG_FORMAT", SUPPORTED_GENTOO_BINPKG_FORMATS[0])
2005 + == "gpkg"
2006 + ):
2007 + try:
2008 + os.environ["GPG_TTY"] = os.ttyname(sys.stdout.fileno())
2009 + except OSError as e:
2010 + # When run with no input/output tty, this will fail.
2011 + # However, if the password is given by command,
2012 + # GPG does not need to ask password, so can be ignored.
2013 + writemsg(colorize("WARN", str(e)) + "\n")
2014 +
2015 + cmd = shlex_split(varexpand(self.GPG_unlock_command, mydict=self.settings))
2016 + return_code = subprocess.Popen(cmd).wait()
2017 +
2018 + if return_code == os.EX_OK:
2019 + writemsg_stdout(colorize("GOOD", "unlocked") + "\n")
2020 + sys.stdout.flush()
2021 + else:
2022 + raise GPGException("GPG unlock failed")
2023 +
2024 + if self.keepalive:
2025 + self.GPG_unlock_command = shlex_split(
2026 + varexpand(self.GPG_unlock_command, mydict=self.settings)
2027 + )
2028 + self.thread = threading.Thread(target=self.gpg_keepalive, daemon=True)
2029 + self.thread.start()
2030 +
2031 + def stop(self):
2032 + """
2033 + Stop keepalive thread.
2034 + """
2035 + if self.thread is not None:
2036 + self.keepalive = False
2037 +
2038 + def gpg_keepalive(self):
2039 + """
2040 + Call GPG unlock command every 5 mins to avoid the passphrase expired.
2041 + """
2042 + count = 0
2043 + while self.keepalive:
2044 + if count < 5:
2045 + time.sleep(60)
2046 + count += 1
2047 + continue
2048 + else:
2049 + count = 0
2050 +
2051 + proc = subprocess.Popen(
2052 + self.GPG_unlock_command,
2053 + stdin=subprocess.DEVNULL,
2054 + stdout=subprocess.DEVNULL,
2055 + stderr=subprocess.STDOUT,
2056 + )
2057 + if proc.wait() != os.EX_OK:
2058 + raise GPGException("GPG keepalive failed")
2059
2060 diff --git a/lib/portage/gpkg.py b/lib/portage/gpkg.py
2061 new file mode 100644
2062 index 000000000..b642e74ec
2063 --- /dev/null
2064 +++ b/lib/portage/gpkg.py
2065 @@ -0,0 +1,2015 @@
2066 +# Copyright 2001-2020 Gentoo Authors
2067 +# Distributed under the terms of the GNU General Public License v2
2068 +
2069 +import tarfile
2070 +import io
2071 +import threading
2072 +import subprocess
2073 +import errno
2074 +import pwd
2075 +import grp
2076 +import stat
2077 +import sys
2078 +import tempfile
2079 +from copy import copy
2080 +from datetime import datetime
2081 +
2082 +from portage import checksum
2083 +from portage import os
2084 +from portage import shutil
2085 +from portage import normalize_path
2086 +from portage import _encodings
2087 +from portage import _unicode_decode
2088 +from portage import _unicode_encode
2089 +from portage.exception import (
2090 + FileNotFound,
2091 + InvalidBinaryPackageFormat,
2092 + InvalidCompressionMethod,
2093 + CompressorNotFound,
2094 + CompressorOperationFailed,
2095 + CommandNotFound,
2096 + GPGException,
2097 + DigestException,
2098 + MissingSignature,
2099 + InvalidSignature,
2100 +)
2101 +from portage.output import colorize
2102 +from portage.util._urlopen import urlopen
2103 +from portage.util import writemsg
2104 +from portage.util import shlex_split, varexpand
2105 +from portage.util.compression_probe import _compressors
2106 +from portage.process import find_binary
2107 +from portage.const import MANIFEST2_HASH_DEFAULTS, HASHING_BLOCKSIZE
2108 +
2109 +
2110 +class tar_stream_writer:
2111 + """
2112 + One-pass helper function that return a file-like object
2113 + for create a file inside of a tar container.
2114 +
2115 + This helper allowed streaming add a new file to tar
2116 + without prior knows the file size.
2117 +
2118 + With optional call and pipe data through external program,
2119 + the helper can transparently save compressed data.
2120 +
2121 + With optional checksum helper, this helper can create
2122 + corresponding checksum and GPG signature.
2123 +
2124 + Example:
2125 +
2126 + writer = tar_stream_writer(
2127 + file_tarinfo, # the file tarinfo that need to be added
2128 + container, # the outer container tarfile object
2129 + tarfile.USTAR_FORMAT, # the outer container format
2130 + ["gzip"], # compression command
2131 + checksum_helper # checksum helper
2132 + )
2133 +
2134 + writer.write(data)
2135 + writer.close()
2136 + """
2137 +
2138 + def __init__(
2139 + self,
2140 + tarinfo,
2141 + container,
2142 + tar_format,
2143 + cmd=None,
2144 + checksum_helper=None,
2145 + uid=None,
2146 + gid=None,
2147 + ):
2148 + """
2149 + tarinfo # the file tarinfo that need to be added
2150 + container # the outer container tarfile object
2151 + tar_format # the outer container format for create the tar header
2152 + cmd # subprocess.Popen format compression command
2153 + checksum_helper # checksum helper
2154 + uid # drop root user to uid
2155 + gid # drop root group to gid
2156 + """
2157 + self.checksum_helper = checksum_helper
2158 + self.closed = False
2159 + self.container = container
2160 + self.killed = False
2161 + self.tar_format = tar_format
2162 + self.tarinfo = tarinfo
2163 + self.uid = uid
2164 + self.gid = gid
2165 +
2166 + # Record container end position
2167 + self.container.fileobj.seek(0, io.SEEK_END)
2168 + self.begin_position = self.container.fileobj.tell()
2169 + self.end_position = 0
2170 + self.file_size = 0
2171 +
2172 + # Write tar header without size
2173 + tar_header = self.tarinfo.tobuf(
2174 + self.tar_format, self.container.encoding, self.container.errors
2175 + )
2176 + self.header_size = len(tar_header)
2177 + self.container.fileobj.write(tar_header)
2178 + self.container.fileobj.flush()
2179 + self.container.offset += self.header_size
2180 +
2181 + # Start external compressor if needed
2182 + if cmd is None:
2183 + self.proc = None
2184 + else:
2185 + if sys.hexversion >= 0x03090000:
2186 + self.proc = subprocess.Popen(
2187 + cmd,
2188 + stdin=subprocess.PIPE,
2189 + stdout=subprocess.PIPE,
2190 + stderr=subprocess.PIPE,
2191 + user=self.uid,
2192 + group=self.gid,
2193 + )
2194 + else:
2195 + self.proc = subprocess.Popen(
2196 + cmd,
2197 + stdin=subprocess.PIPE,
2198 + stdout=subprocess.PIPE,
2199 + stderr=subprocess.PIPE,
2200 + preexec_fn=self._drop_privileges,
2201 + )
2202 +
2203 + self.read_thread = threading.Thread(
2204 + target=self._cmd_read_thread, name="tar_stream_cmd_read", daemon=True
2205 + )
2206 + self.read_thread.start()
2207 +
2208 + def __del__(self):
2209 + self.close()
2210 +
2211 + def __enter__(self):
2212 + return self
2213 +
2214 + def __exit__(self, exc_type, exc_value, traceback):
2215 + self.close()
2216 +
2217 + def _drop_privileges(self):
2218 + if self.uid:
2219 + try:
2220 + os.setuid(self.uid)
2221 + except PermissionError:
2222 + writemsg(
2223 + colorize(
2224 + "BAD", f"!!! Drop root privileges to user {self.uid} failed."
2225 + )
2226 + )
2227 + raise
2228 +
2229 + if self.gid:
2230 + try:
2231 + os.setgid(self.gid)
2232 + except PermissionError:
2233 + writemsg(
2234 + colorize(
2235 + "BAD", f"!!! Drop root privileges to group {self.gid} failed."
2236 + )
2237 + )
2238 + raise
2239 +
2240 + def kill(self):
2241 + """
2242 + kill external program if any error happened in python
2243 + """
2244 + if self.proc is not None:
2245 + self.killed = True
2246 + self.proc.kill()
2247 + self.proc.stdin.close()
2248 + self.close()
2249 +
2250 + def _cmd_read_thread(self):
2251 + """
2252 + Use thread to avoid block.
2253 + Read stdout from external compressor, then write to the file
2254 + in container, and to checksum helper if needed.
2255 + """
2256 + while True:
2257 + try:
2258 + buffer = self.proc.stdout.read(HASHING_BLOCKSIZE)
2259 + if not buffer:
2260 + self.proc.stdout.close()
2261 + self.proc.stderr.close()
2262 + return
2263 + except BrokenPipeError:
2264 + self.proc.stdout.close()
2265 + if not self.killed:
2266 + # Do not raise error if killed by portage
2267 + raise CompressorOperationFailed("PIPE broken")
2268 +
2269 + self.container.fileobj.write(buffer)
2270 + if self.checksum_helper:
2271 + self.checksum_helper.update(buffer)
2272 +
2273 + def write(self, data):
2274 + """
2275 + Write data to tarfile or external compressor stdin
2276 + """
2277 + if self.closed:
2278 + raise OSError("writer closed")
2279 +
2280 + if self.proc:
2281 + # Write to external program
2282 + self.proc.stdin.write(data)
2283 + else:
2284 + # Write to container
2285 + self.container.fileobj.write(data)
2286 + if self.checksum_helper:
2287 + self.checksum_helper.update(data)
2288 +
2289 + def close(self):
2290 + """
2291 + Update the new file tar header when close
2292 + """
2293 + if self.closed:
2294 + return
2295 +
2296 + # Wait compressor exit
2297 + if self.proc is not None:
2298 + self.proc.stdin.close()
2299 + if self.proc.wait() != os.EX_OK:
2300 + raise CompressorOperationFailed("compression failed")
2301 + if self.read_thread.is_alive():
2302 + self.read_thread.join()
2303 +
2304 + # Get container end position and calculate file size
2305 + self.container.fileobj.seek(0, io.SEEK_END)
2306 + self.end_position = self.container.fileobj.tell()
2307 + self.file_size = self.end_position - self.begin_position - self.header_size
2308 + self.tarinfo.size = self.file_size
2309 +
2310 + # Tar block is 512, need padding \0
2311 + _, remainder = divmod(self.file_size, 512)
2312 + if remainder > 0:
2313 + padding_size = 512 - remainder
2314 + self.container.fileobj.write(b"\0" * padding_size)
2315 + self.container.offset += padding_size
2316 + self.container.fileobj.flush()
2317 +
2318 + # Update tar header
2319 + tar_header = self.tarinfo.tobuf(
2320 + self.tar_format, self.container.encoding, self.container.errors
2321 + )
2322 + self.container.fileobj.seek(self.begin_position)
2323 + self.container.fileobj.write(tar_header)
2324 + self.container.fileobj.seek(0, io.SEEK_END)
2325 + self.container.fileobj.flush()
2326 + self.container.offset = self.container.fileobj.tell()
2327 + self.closed = True
2328 +
2329 + # Add tarinfo to tarfile
2330 + self.container.members.append(self.tarinfo)
2331 +
2332 + if self.checksum_helper:
2333 + self.checksum_helper.finish()
2334 +
2335 + self.closed = True
2336 +
2337 +
2338 +class tar_stream_reader:
2339 + """
2340 + helper function that return a file-like object
2341 + for read a file inside of a tar container.
2342 +
2343 + This helper allowed transparently streaming read a compressed
2344 + file in tar.
2345 +
2346 + With optional call and pipe compressed data through external
2347 + program, and return the uncompressed data.
2348 +
2349 + reader = tar_stream_reader(
2350 + fileobj, # the fileobj from tarfile.extractfile(f)
2351 + ["gzip", "-d"], # decompression command
2352 + )
2353 +
2354 + reader.read()
2355 + reader.close()
2356 + """
2357 +
2358 + def __init__(self, fileobj, cmd=None, uid=None, gid=None):
2359 + """
2360 + fileobj should be a file-like object that have read().
2361 + cmd is optional external decompressor command.
2362 + """
2363 + self.closed = False
2364 + self.cmd = cmd
2365 + self.fileobj = fileobj
2366 + self.killed = False
2367 + self.uid = uid
2368 + self.gid = gid
2369 +
2370 + if cmd is None:
2371 + self.read_io = fileobj
2372 + self.proc = None
2373 + else:
2374 + # Start external decompressor
2375 + if sys.hexversion >= 0x03090000:
2376 + self.proc = subprocess.Popen(
2377 + cmd,
2378 + stdin=subprocess.PIPE,
2379 + stdout=subprocess.PIPE,
2380 + stderr=subprocess.PIPE,
2381 + user=self.uid,
2382 + group=self.gid,
2383 + )
2384 + else:
2385 + self.proc = subprocess.Popen(
2386 + cmd,
2387 + stdin=subprocess.PIPE,
2388 + stdout=subprocess.PIPE,
2389 + stderr=subprocess.PIPE,
2390 + preexec_fn=self._drop_privileges,
2391 + )
2392 + self.read_io = self.proc.stdout
2393 + # Start stdin block writing thread
2394 + self.thread = threading.Thread(
2395 + target=self._write_thread, name="tar_stream_stdin_writer", daemon=True
2396 + )
2397 + self.thread.start()
2398 +
2399 + def __del__(self):
2400 + try:
2401 + self.close()
2402 + except CompressorOperationFailed:
2403 + pass
2404 +
2405 + def __enter__(self):
2406 + return self
2407 +
2408 + def __exit__(self, exc_type, exc_value, traceback):
2409 + try:
2410 + self.close()
2411 + except CompressorOperationFailed:
2412 + pass
2413 +
2414 + def _write_thread(self):
2415 + """
2416 + writing thread to avoid full buffer blocking
2417 + """
2418 + try:
2419 + while True:
2420 + buffer = self.fileobj.read(HASHING_BLOCKSIZE)
2421 + if buffer:
2422 + try:
2423 + self.proc.stdin.write(buffer)
2424 + except ValueError:
2425 + if self.killed:
2426 + return
2427 + else:
2428 + raise
2429 + else:
2430 + self.proc.stdin.flush()
2431 + self.proc.stdin.close()
2432 + break
2433 + except BrokenPipeError:
2434 + if self.killed is False:
2435 + raise CompressorOperationFailed("PIPE broken")
2436 +
2437 + def _drop_privileges(self):
2438 + if self.uid:
2439 + try:
2440 + os.setuid(self.uid)
2441 + except PermissionError:
2442 + writemsg(
2443 + colorize(
2444 + "BAD", f"!!! Drop root privileges to user {self.uid} failed."
2445 + )
2446 + )
2447 + raise
2448 +
2449 + if self.gid:
2450 + try:
2451 + os.setgid(self.gid)
2452 + except PermissionError:
2453 + writemsg(
2454 + colorize(
2455 + "BAD", f"!!! Drop root privileges to group {self.gid} failed."
2456 + )
2457 + )
2458 + raise
2459 +
2460 + def kill(self):
2461 + """
2462 + kill external program if any error happened in python
2463 + """
2464 + if self.proc is not None:
2465 + self.killed = True
2466 + self.proc.kill()
2467 + self.proc.stdin.close()
2468 + self.close()
2469 +
2470 + def read(self, bufsize=-1):
2471 + """
2472 + return decompressor stdout data
2473 + """
2474 + if self.closed:
2475 + raise OSError("writer closed")
2476 + else:
2477 + return self.read_io.read(bufsize)
2478 +
2479 + def close(self):
2480 + """
2481 + wait external program complete and do clean up
2482 + """
2483 + if self.closed:
2484 + return
2485 +
2486 + self.closed = True
2487 +
2488 + if self.proc is not None:
2489 + self.thread.join()
2490 + try:
2491 + if self.proc.wait() != os.EX_OK:
2492 + if not self.proc.stderr.closed:
2493 + stderr = self.proc.stderr.read().decode()
2494 + if not self.killed:
2495 + writemsg(colorize("BAD", "!!!" + "\n" + stderr))
2496 + raise CompressorOperationFailed("decompression failed")
2497 + finally:
2498 + self.proc.stdout.close()
2499 + self.proc.stderr.close()
2500 +
2501 +
2502 +class checksum_helper:
2503 + """
2504 + Do checksum generation and GPG Signature generation and verification
2505 + """
2506 +
2507 + SIGNING = 0
2508 + VERIFY = 1
2509 +
2510 + def __init__(self, settings, gpg_operation=None, detached=True, signature=None):
2511 + """
2512 + settings # portage settings
2513 + gpg_operation # either SIGNING or VERIFY
2514 + signature # GPG signature string used for GPG verify only
2515 + """
2516 + self.settings = settings
2517 + self.gpg_operation = gpg_operation
2518 + self.gpg_proc = None
2519 + self.gpg_result = None
2520 + self.gpg_output = None
2521 + self.finished = False
2522 + self.sign_file_path = None
2523 +
2524 + if (gpg_operation == checksum_helper.VERIFY) and (os.getuid() == 0):
2525 + try:
2526 + drop_user = self.settings.get("GPG_VERIFY_USER_DROP", "nobody")
2527 + if drop_user == "":
2528 + self.uid = None
2529 + else:
2530 + self.uid = pwd.getpwnam(drop_user).pw_uid
2531 + except KeyError:
2532 + writemsg(colorize("BAD", f"!!! Failed to find user {drop_user}."))
2533 + raise
2534 +
2535 + try:
2536 + drop_group = self.settings.get("GPG_VERIFY_GROUP_DROP", "nogroup")
2537 + if drop_group == "":
2538 + self.gid = None
2539 + else:
2540 + self.gid = grp.getgrnam(drop_group).gr_gid
2541 + except KeyError:
2542 + writemsg(colorize("BAD", f"!!! Failed to find group {drop_group}."))
2543 + raise
2544 + else:
2545 + self.uid = None
2546 + self.gid = None
2547 +
2548 + # Initialize the hash libs
2549 + self.libs = {}
2550 + for hash_name in MANIFEST2_HASH_DEFAULTS:
2551 + self.libs[hash_name] = checksum.hashfunc_map[hash_name]._hashobject()
2552 +
2553 + # GPG
2554 + env = self.settings.environ()
2555 + if self.gpg_operation == checksum_helper.SIGNING:
2556 + gpg_signing_base_command = self.settings.get(
2557 + "BINPKG_GPG_SIGNING_BASE_COMMAND"
2558 + )
2559 + digest_algo = self.settings.get("BINPKG_GPG_SIGNING_DIGEST")
2560 + gpg_home = self.settings.get("BINPKG_GPG_SIGNING_GPG_HOME")
2561 + gpg_key = self.settings.get("BINPKG_GPG_SIGNING_KEY")
2562 +
2563 + if detached:
2564 + gpg_detached = "--detach-sig"
2565 + else:
2566 + gpg_detached = "--clear-sign"
2567 +
2568 + if gpg_signing_base_command:
2569 + gpg_signing_command = gpg_signing_base_command.replace(
2570 + "[PORTAGE_CONFIG]",
2571 + f"--homedir {gpg_home} "
2572 + f"--digest-algo {digest_algo} "
2573 + f"--local-user {gpg_key} "
2574 + f"{gpg_detached} "
2575 + "--batch --no-tty",
2576 + )
2577 +
2578 + gpg_signing_command = shlex_split(
2579 + varexpand(gpg_signing_command, mydict=self.settings)
2580 + )
2581 + gpg_signing_command = [x for x in gpg_signing_command if x != ""]
2582 + try:
2583 + env["GPG_TTY"] = os.ttyname(sys.stdout.fileno())
2584 + except OSError:
2585 + pass
2586 + else:
2587 + raise CommandNotFound("GPG signing command is not set")
2588 +
2589 + self.gpg_proc = subprocess.Popen(
2590 + gpg_signing_command,
2591 + stdin=subprocess.PIPE,
2592 + stdout=subprocess.PIPE,
2593 + stderr=subprocess.PIPE,
2594 + env=env,
2595 + )
2596 +
2597 + elif self.gpg_operation == checksum_helper.VERIFY:
2598 + if (signature is None) and (detached == True):
2599 + raise MissingSignature("No signature provided")
2600 +
2601 + gpg_verify_base_command = self.settings.get(
2602 + "BINPKG_GPG_VERIFY_BASE_COMMAND"
2603 + )
2604 + gpg_home = self.settings.get("BINPKG_GPG_VERIFY_GPG_HOME")
2605 +
2606 + if not gpg_verify_base_command:
2607 + raise CommandNotFound("GPG verify command is not set")
2608 +
2609 + gpg_verify_command = gpg_verify_base_command.replace(
2610 + "[PORTAGE_CONFIG]", f"--homedir {gpg_home} "
2611 + )
2612 +
2613 + if detached:
2614 + self.sign_file_fd, self.sign_file_path = tempfile.mkstemp(
2615 + ".sig", "portage-sign-"
2616 + )
2617 +
2618 + gpg_verify_command = gpg_verify_command.replace(
2619 + "[SIGNATURE]", f"{self.sign_file_path} -"
2620 + )
2621 +
2622 + # Create signature file and allow everyone read
2623 + with open(self.sign_file_fd, "wb") as sign:
2624 + sign.write(signature)
2625 + os.chmod(self.sign_file_path, 0o644)
2626 + else:
2627 + gpg_verify_command = gpg_verify_command.replace(
2628 + "[SIGNATURE]", "--output - -"
2629 + )
2630 +
2631 + gpg_verify_command = shlex_split(
2632 + varexpand(gpg_verify_command, mydict=self.settings)
2633 + )
2634 + gpg_verify_command = [x for x in gpg_verify_command if x != ""]
2635 +
2636 + if sys.hexversion >= 0x03090000:
2637 + self.gpg_proc = subprocess.Popen(
2638 + gpg_verify_command,
2639 + stdin=subprocess.PIPE,
2640 + stdout=subprocess.PIPE,
2641 + stderr=subprocess.PIPE,
2642 + env=env,
2643 + user=self.uid,
2644 + group=self.gid,
2645 + )
2646 +
2647 + else:
2648 + self.gpg_proc = subprocess.Popen(
2649 + gpg_verify_command,
2650 + stdin=subprocess.PIPE,
2651 + stdout=subprocess.PIPE,
2652 + stderr=subprocess.PIPE,
2653 + env=env,
2654 + preexec_fn=self._drop_privileges,
2655 + )
2656 +
2657 + def __del__(self):
2658 + self.finish()
2659 +
2660 + def _check_gpg_status(self, gpg_status):
2661 + """
2662 + Check GPG status log for extra info.
2663 + GPG will return OK even if the signature owner is not trusted.
2664 + """
2665 + good_signature = False
2666 + trust_signature = False
2667 +
2668 + for l in gpg_status.splitlines():
2669 + if l.startswith("[GNUPG:] GOODSIG"):
2670 + good_signature = True
2671 +
2672 + if l.startswith("[GNUPG:] TRUST_ULTIMATE") or l.startswith(
2673 + "[GNUPG:] TRUST_FULLY"
2674 + ):
2675 + trust_signature = True
2676 +
2677 + if (not good_signature) or (not trust_signature):
2678 + writemsg(colorize("BAD", "!!!" + "\n" + self.gpg_result.decode()))
2679 + raise InvalidSignature("GPG verify failed")
2680 +
2681 + def _drop_privileges(self):
2682 + if self.uid:
2683 + try:
2684 + os.setuid(self.uid)
2685 + except PermissionError:
2686 + writemsg(
2687 + colorize(
2688 + "BAD", f"!!! Drop root privileges to user {self.uid} failed."
2689 + )
2690 + )
2691 + raise
2692 +
2693 + if self.gid:
2694 + try:
2695 + os.setgid(self.gid)
2696 + except PermissionError:
2697 + writemsg(
2698 + colorize(
2699 + "BAD", f"!!! Drop root privileges to group {self.gid} failed."
2700 + )
2701 + )
2702 + raise
2703 +
2704 + def update(self, data):
2705 + """
2706 + Write data to hash libs and GPG stdin.
2707 + """
2708 + for c in self.libs:
2709 + self.libs[c].update(data)
2710 +
2711 + if self.gpg_proc is not None:
2712 + self.gpg_proc.stdin.write(data)
2713 +
2714 + def finish(self):
2715 + """
2716 + Tell GPG file is EOF, and get results, then do clean up.
2717 + """
2718 + if self.finished:
2719 + return
2720 +
2721 + if self.gpg_proc is not None:
2722 + # Tell GPG EOF
2723 + self.gpg_proc.stdin.close()
2724 +
2725 + return_code = self.gpg_proc.wait()
2726 +
2727 + if self.sign_file_path:
2728 + os.remove(self.sign_file_path)
2729 +
2730 + self.finished = True
2731 +
2732 + self.gpg_result = self.gpg_proc.stderr.read()
2733 + self.gpg_output = self.gpg_proc.stdout.read()
2734 + self.gpg_proc.stdout.close()
2735 + self.gpg_proc.stderr.close()
2736 +
2737 + if return_code == os.EX_OK:
2738 + if self.gpg_operation == checksum_helper.VERIFY:
2739 + self._check_gpg_status(self.gpg_result.decode())
2740 + else:
2741 + writemsg(colorize("BAD", "!!!" + "\n" + self.gpg_result.decode()))
2742 + if self.gpg_operation == checksum_helper.SIGNING:
2743 + writemsg(colorize("BAD", self.gpg_output.decode()))
2744 + raise GPGException("GPG signing failed")
2745 + elif self.gpg_operation == checksum_helper.VERIFY:
2746 + raise InvalidSignature("GPG verify failed")
2747 +
2748 +
2749 +class tar_safe_extract:
2750 + """
2751 + A safer version of tar extractall that doing sanity check.
2752 + Note that this does not solve all security problems.
2753 + """
2754 +
2755 + def __init__(self, tar: tarfile.TarFile, prefix: str = ""):
2756 + """
2757 + tar: an opened TarFile that ready to be read.
2758 + prefix: a optional prefix for an inner directory should be considered
2759 + as the root directory. e.g. "metadata" and "image".
2760 + """
2761 + self.tar = tar
2762 + self.prefix = prefix
2763 + self.closed = False
2764 + self.file_list = []
2765 +
2766 + def extractall(self, dest_dir: str):
2767 + """
2768 + Extract all files to a temporary directory in the dest_dir, and move
2769 + them to the dest_dir after sanity check.
2770 + """
2771 + if self.closed:
2772 + raise IOError("Tar file is closed.")
2773 + temp_dir = tempfile.TemporaryDirectory(dir=dest_dir)
2774 + try:
2775 + while True:
2776 + member = self.tar.next()
2777 + if member is None:
2778 + break
2779 + if (member.name in self.file_list) or (
2780 + os.path.join(".", member.name) in self.file_list
2781 + ):
2782 + writemsg(
2783 + colorize(
2784 + "BAD", f"Danger: duplicate files detected: {member.name}"
2785 + )
2786 + )
2787 + raise ValueError("Duplicate files detected.")
2788 + if member.name.startswith("/"):
2789 + writemsg(
2790 + colorize(
2791 + "BAD", f"Danger: absolute path detected: {member.name}"
2792 + )
2793 + )
2794 + raise ValueError("Absolute path detected.")
2795 + if member.name.startswith("../") or ("/../" in member.name):
2796 + writemsg(
2797 + colorize(
2798 + "BAD", f"Danger: path traversal detected: {member.name}"
2799 + )
2800 + )
2801 + raise ValueError("Path traversal detected.")
2802 + if member.isdev():
2803 + writemsg(
2804 + colorize("BAD", f"Danger: device file detected: {member.name}")
2805 + )
2806 + raise ValueError("Device file detected.")
2807 + if member.islnk() and (member.linkname not in self.file_list):
2808 + writemsg(
2809 + colorize(
2810 + "BAD", f"Danger: hardlink escape detected: {member.name}"
2811 + )
2812 + )
2813 + raise ValueError("Hardlink escape detected.")
2814 +
2815 + self.file_list.append(member.name)
2816 + self.tar.extract(member, path=temp_dir.name)
2817 +
2818 + data_dir = os.path.join(temp_dir.name, self.prefix)
2819 + for file in os.listdir(data_dir):
2820 + shutil.move(os.path.join(data_dir, file), os.path.join(dest_dir, file))
2821 + finally:
2822 + temp_dir.cleanup()
2823 + self.closed = True
2824 +
2825 +
2826 +class gpkg:
2827 + """
2828 + Gentoo binary package
2829 + https://www.gentoo.org/glep/glep-0078.html
2830 + """
2831 +
2832 + def __init__(self, settings, base_name=None, gpkg_file=None):
2833 + """
2834 + gpkg class handle all gpkg operations for one package.
2835 + base_name is the package basename.
2836 + gpkg_file should be exists file path for read or will create.
2837 + """
2838 + if sys.version_info.major < 3:
2839 + raise InvalidBinaryPackageFormat("GPKG not support Python 2")
2840 + self.settings = settings
2841 + self.gpkg_version = "gpkg-1"
2842 + if gpkg_file is None:
2843 + self.gpkg_file = None
2844 + else:
2845 + self.gpkg_file = _unicode_decode(
2846 + gpkg_file, encoding=_encodings["fs"], errors="strict"
2847 + )
2848 + self.base_name = base_name
2849 + self.checksums = []
2850 + self.manifest_old = []
2851 +
2852 + # Compression is the compression algorithm, if set to None will
2853 + # not use compression.
2854 + self.compression = self.settings.get("BINPKG_COMPRESS", None)
2855 + if self.compression in ["", "none"]:
2856 + self.compression = None
2857 +
2858 + # The create_signature is whether create signature for the package or not.
2859 + if "binpkg-signing" in self.settings.features:
2860 + self.create_signature = True
2861 + else:
2862 + self.create_signature = False
2863 +
2864 + # The request_signature is whether signature files are mandatory.
2865 + # If set true, any missing signature file will cause reject processing.
2866 + if "binpkg-request-signature" in self.settings.features:
2867 + self.request_signature = True
2868 + else:
2869 + self.request_signature = False
2870 +
2871 + # The verify_signature is whether verify package signature or not.
2872 + # In rare case user may want to ignore signature,
2873 + # E.g. package with expired signature.
2874 + if "binpkg-ignore-signature" in self.settings.features:
2875 + self.verify_signature = False
2876 + else:
2877 + self.verify_signature = True
2878 +
2879 + self.ext_list = {
2880 + "gzip": ".gz",
2881 + "bzip2": ".bz2",
2882 + "lz4": ".lz4",
2883 + "lzip": ".lz",
2884 + "lzop": ".lzo",
2885 + "xz": ".xz",
2886 + "zstd": ".zst",
2887 + }
2888 +
2889 + def unpack_metadata(self, dest_dir=None):
2890 + """
2891 + Unpack metadata to dest_dir.
2892 + If dest_dir is None, return files and values in dict.
2893 + The dict key will be UTF-8, not bytes.
2894 + """
2895 + self._verify_binpkg(metadata_only=True)
2896 +
2897 + with tarfile.open(self.gpkg_file, "r") as container:
2898 + metadata_tarinfo, metadata_comp = self._get_inner_tarinfo(
2899 + container, "metadata"
2900 + )
2901 +
2902 + with tar_stream_reader(
2903 + container.extractfile(metadata_tarinfo),
2904 + self._get_decompression_cmd(metadata_comp),
2905 + ) as metadata_reader:
2906 + metadata_tar = io.BytesIO(metadata_reader.read())
2907 +
2908 + with tarfile.open(mode="r:", fileobj=metadata_tar) as metadata:
2909 + if dest_dir is None:
2910 + metadata_ = {
2911 + os.path.relpath(k.name, "metadata"): metadata.extractfile(
2912 + k
2913 + ).read()
2914 + for k in metadata.getmembers()
2915 + }
2916 + else:
2917 + metadata_safe = tar_safe_extract(metadata, "metadata")
2918 + metadata_safe.extractall(dest_dir)
2919 + metadata_ = True
2920 + metadata_tar.close()
2921 + return metadata_
2922 +
2923 + def get_metadata(self, want=None):
2924 + """
2925 + get package metadata.
2926 + if want is list, return all want key-values in dict
2927 + if want is str, return the want key value
2928 + """
2929 + if want is None:
2930 + return self.unpack_metadata()
2931 + elif isinstance(want, str):
2932 + metadata = self.unpack_metadata()
2933 + metadata_want = metadata.get(want, None)
2934 + return metadata_want
2935 + else:
2936 + metadata = self.unpack_metadata()
2937 + metadata_want = {k: metadata.get(k, None) for k in want}
2938 + return metadata_want
2939 +
2940 + def get_metadata_url(self, url, want=None):
2941 + """
2942 + Return the requested metadata from url gpkg.
2943 + Default return all meta data.
2944 + Use 'want' to get specific name from metadata.
2945 + This method only support the correct package format.
2946 + Wrong files order or incorrect basename will be considered invalid
2947 + to reduce potential attacks.
2948 + Only signature will be check if the signature file is the next file.
2949 + Manifest will be ignored since it will be at the end of package.
2950 + """
2951 + # The init download file head size
2952 + init_size = 51200
2953 +
2954 + # Load remote container
2955 + container_file = io.BytesIO(
2956 + urlopen(url, headers={"Range": "bytes=0-" + str(init_size)}).read()
2957 + )
2958 +
2959 + # Check gpkg and metadata
2960 + with tarfile.open(mode="r", fileobj=container_file) as container:
2961 + if self.gpkg_version not in container.getnames():
2962 + raise InvalidBinaryPackageFormat("Invalid gpkg file.")
2963 +
2964 + metadata_tarinfo, metadata_comp = self._get_inner_tarinfo(
2965 + container, "metadata"
2966 + )
2967 +
2968 + # Extra 10240 bytes for signature
2969 + end_size = metadata_tarinfo.offset_data + metadata_tarinfo.size + 10240
2970 + _, remainder = divmod(end_size, 512)
2971 + end_size += 512 - remainder
2972 +
2973 + # If need more data
2974 + if end_size > 10000000:
2975 + raise InvalidBinaryPackageFormat("metadata too large " + str(end_size))
2976 + if end_size > init_size:
2977 + container_file.seek(0, io.SEEK_END)
2978 + container_file.write(
2979 + urlopen(
2980 + url,
2981 + headers={
2982 + "Range": "bytes=" + str(init_size + 1) + "-" + str(end_size)
2983 + },
2984 + ).read()
2985 + )
2986 +
2987 + container_file.seek(0)
2988 +
2989 + # Reload and process full metadata
2990 + with tarfile.open(mode="r", fileobj=container_file) as container:
2991 + metadata_tarinfo, metadata_comp = self._get_inner_tarinfo(
2992 + container, "metadata"
2993 + )
2994 +
2995 + # Verify metadata file signature if needed
2996 + # binpkg-ignore-signature can override this.
2997 + signature_filename = metadata_tarinfo.name + ".sig"
2998 + if signature_filename in container.getnames():
2999 + if self.request_signature and self.verify_signature:
3000 + metadata_signature = container.extractfile(
3001 + signature_filename
3002 + ).read()
3003 + checksum_info = checksum_helper(
3004 + self.settings,
3005 + gpg_operation=checksum_helper.VERIFY,
3006 + signature=metadata_signature,
3007 + )
3008 + checksum_info.update(container.extractfile(metadata_tarinfo).read())
3009 + checksum_info.finish()
3010 +
3011 + # Load metadata
3012 + with tar_stream_reader(
3013 + container.extractfile(metadata_tarinfo),
3014 + self._get_decompression_cmd(metadata_comp),
3015 + ) as metadata_reader:
3016 + metadata_file = io.BytesIO(metadata_reader.read())
3017 +
3018 + with tarfile.open(mode="r:", fileobj=metadata_file) as metadata:
3019 + if want is None:
3020 + metadata_ = {
3021 + os.path.relpath(k.name, "metadata"): metadata.extractfile(
3022 + k
3023 + ).read()
3024 + for k in metadata.getmembers()
3025 + }
3026 + else:
3027 + metadata_ = {
3028 + os.path.relpath(k.name, "metadata"): metadata.extractfile(
3029 + k
3030 + ).read()
3031 + for k in metadata.getmembers()
3032 + if k in want
3033 + }
3034 + metadata_file.close()
3035 + container_file.close()
3036 + return metadata_
3037 +
3038 + def compress(self, root_dir, metadata, clean=False):
3039 + """
3040 + Use initialized configuation create new gpkg file from root_dir.
3041 + Will overwrite any exists file.
3042 + metadata is a dict, the key will be file name, the value will be
3043 + the file contents.
3044 + """
3045 +
3046 + root_dir = normalize_path(
3047 + _unicode_decode(root_dir, encoding=_encodings["fs"], errors="strict")
3048 + )
3049 +
3050 + # Get pre image info
3051 + container_tar_format, image_tar_format = self._get_tar_format_from_stats(
3052 + *self._check_pre_image_files(root_dir)
3053 + )
3054 +
3055 + # Long CPV
3056 + if len(self.base_name) >= 154:
3057 + container_tar_format = tarfile.GNU_FORMAT
3058 +
3059 + # gpkg container
3060 + container = tarfile.TarFile(
3061 + name=self.gpkg_file, mode="w", format=container_tar_format
3062 + )
3063 +
3064 + # gpkg version
3065 + gpkg_version_file = tarfile.TarInfo(self.gpkg_version)
3066 + gpkg_version_file.mtime = datetime.utcnow().timestamp()
3067 + container.addfile(gpkg_version_file)
3068 +
3069 + compression_cmd = self._get_compression_cmd()
3070 +
3071 + # metadata
3072 + self._add_metadata(container, metadata, compression_cmd)
3073 +
3074 + # image
3075 + if self.create_signature:
3076 + checksum_info = checksum_helper(
3077 + self.settings, gpg_operation=checksum_helper.SIGNING
3078 + )
3079 + else:
3080 + checksum_info = checksum_helper(self.settings)
3081 +
3082 + image_tarinfo = self._create_tarinfo("image")
3083 + image_tarinfo.mtime = datetime.utcnow().timestamp()
3084 + with tar_stream_writer(
3085 + image_tarinfo, container, image_tar_format, compression_cmd, checksum_info
3086 + ) as image_writer:
3087 + with tarfile.open(
3088 + mode="w|", fileobj=image_writer, format=image_tar_format
3089 + ) as image_tar:
3090 + image_tar.add(root_dir, "image", recursive=True)
3091 +
3092 + image_tarinfo = container.getmember(image_tarinfo.name)
3093 + self._record_checksum(checksum_info, image_tarinfo)
3094 +
3095 + if self.create_signature:
3096 + self._add_signature(checksum_info, image_tarinfo, container)
3097 +
3098 + self._add_manifest(container)
3099 + container.close()
3100 +
3101 + def decompress(self, decompress_dir):
3102 + """
3103 + decompress current gpkg to decompress_dir
3104 + """
3105 + decompress_dir = normalize_path(
3106 + _unicode_decode(decompress_dir, encoding=_encodings["fs"], errors="strict")
3107 + )
3108 +
3109 + self._verify_binpkg()
3110 + os.makedirs(decompress_dir, mode=0o755, exist_ok=True)
3111 +
3112 + with tarfile.open(self.gpkg_file, "r") as container:
3113 + image_tarinfo, image_comp = self._get_inner_tarinfo(container, "image")
3114 +
3115 + with tar_stream_reader(
3116 + container.extractfile(image_tarinfo),
3117 + self._get_decompression_cmd(image_comp),
3118 + ) as image_tar:
3119 +
3120 + with tarfile.open(mode="r|", fileobj=image_tar) as image:
3121 + try:
3122 + image_safe = tar_safe_extract(image, "image")
3123 + image_safe.extractall(decompress_dir)
3124 + except Exception as ex:
3125 + writemsg(colorize("BAD", "!!!" + "Extract failed."))
3126 + raise
3127 + finally:
3128 + image_tar.kill()
3129 +
3130 + def update_metadata(self, metadata, newcpv=None):
3131 + """
3132 + Update metadata in the gpkg file.
3133 + """
3134 + self._verify_binpkg()
3135 + self.checksums = []
3136 + oldcpv = None
3137 +
3138 + if newcpv:
3139 + oldcpv = self.base_name
3140 +
3141 + with open(self.gpkg_file, "rb") as container:
3142 + container_tar_format = self._get_tar_format(container)
3143 + if container_tar_format is None:
3144 + raise InvalidBinaryPackageFormat("Cannot identify tar format")
3145 +
3146 + # container
3147 + tmp_gpkg_file_name = self.gpkg_file + "." + str(os.getpid())
3148 + with tarfile.TarFile(
3149 + name=tmp_gpkg_file_name, mode="w", format=container_tar_format
3150 + ) as container:
3151 + # gpkg version
3152 + gpkg_version_file = tarfile.TarInfo(self.gpkg_version)
3153 + gpkg_version_file.mtime = datetime.utcnow().timestamp()
3154 + container.addfile(gpkg_version_file)
3155 +
3156 + compression_cmd = self._get_compression_cmd()
3157 +
3158 + # metadata
3159 + if newcpv:
3160 + self.base_name = newcpv
3161 + self._add_metadata(container, metadata, compression_cmd)
3162 + self.base_name = oldcpv
3163 + else:
3164 + self._add_metadata(container, metadata, compression_cmd)
3165 +
3166 + # reuse other stuffs
3167 + with tarfile.open(self.gpkg_file, "r") as container_old:
3168 + manifest_old = self.manifest_old.copy()
3169 +
3170 + for m in manifest_old:
3171 + file_name_old = m[1]
3172 + if os.path.basename(file_name_old).startswith("metadata"):
3173 + continue
3174 + old_data_tarinfo = container_old.getmember(file_name_old)
3175 + new_data_tarinfo = copy(old_data_tarinfo)
3176 + if newcpv:
3177 + m[1] = m[1].replace(oldcpv, newcpv, 1)
3178 + new_data_tarinfo.name = new_data_tarinfo.name.replace(
3179 + oldcpv, newcpv, 1
3180 + )
3181 + container.addfile(
3182 + new_data_tarinfo, container_old.extractfile(old_data_tarinfo)
3183 + )
3184 + self.checksums.append(m)
3185 +
3186 + self._add_manifest(container)
3187 +
3188 + shutil.move(tmp_gpkg_file_name, self.gpkg_file)
3189 +
3190 + def _add_metadata(self, container, metadata, compression_cmd):
3191 + """
3192 + add metadata to container
3193 + """
3194 + if metadata is None:
3195 + metadata = {}
3196 + metadata_tarinfo = self._create_tarinfo("metadata")
3197 + metadata_tarinfo.mtime = datetime.utcnow().timestamp()
3198 +
3199 + if self.create_signature:
3200 + checksum_info = checksum_helper(
3201 + self.settings, gpg_operation=checksum_helper.SIGNING
3202 + )
3203 + else:
3204 + checksum_info = checksum_helper(self.settings)
3205 +
3206 + with tar_stream_writer(
3207 + metadata_tarinfo,
3208 + container,
3209 + tarfile.USTAR_FORMAT,
3210 + compression_cmd,
3211 + checksum_info,
3212 + ) as metadata_writer:
3213 + with tarfile.open(
3214 + mode="w|", fileobj=metadata_writer, format=tarfile.USTAR_FORMAT
3215 + ) as metadata_tar:
3216 +
3217 + for m in metadata:
3218 + m_info = tarfile.TarInfo(os.path.join("metadata", m))
3219 + m_info.mtime = datetime.utcnow().timestamp()
3220 +
3221 + if isinstance(metadata[m], bytes):
3222 + m_data = io.BytesIO(metadata[m])
3223 + else:
3224 + m_data = io.BytesIO(metadata[m].encode("UTF-8"))
3225 +
3226 + m_data.seek(0, io.SEEK_END)
3227 + m_info.size = m_data.tell()
3228 + m_data.seek(0)
3229 + metadata_tar.addfile(m_info, m_data)
3230 + m_data.close()
3231 +
3232 + metadata_tarinfo = container.getmember(metadata_tarinfo.name)
3233 + self._record_checksum(checksum_info, metadata_tarinfo)
3234 +
3235 + if self.create_signature:
3236 + self._add_signature(checksum_info, metadata_tarinfo, container)
3237 +
3238 + def _quickpkg(self, contents, metadata, root_dir, protect=None):
3239 + """
3240 + Similar to compress, but for quickpkg.
3241 + Will compress the given files to image with root,
3242 + ignoring all other files.
3243 + """
3244 +
3245 + protect_file = io.BytesIO(
3246 + b"# empty file because --include-config=n " + b"when `quickpkg` was used\n"
3247 + )
3248 + protect_file.seek(0, io.SEEK_END)
3249 + protect_file_size = protect_file.tell()
3250 +
3251 + root_dir = normalize_path(
3252 + _unicode_decode(root_dir, encoding=_encodings["fs"], errors="strict")
3253 + )
3254 +
3255 + # Get pre image info
3256 + container_tar_format, image_tar_format = self._get_tar_format_from_stats(
3257 + *self._check_pre_quickpkg_files(contents, root_dir)
3258 + )
3259 +
3260 + # Long CPV
3261 + if len(self.base_name) >= 154:
3262 + container_tar_format = tarfile.GNU_FORMAT
3263 +
3264 + # GPKG container
3265 + container = tarfile.TarFile(
3266 + name=self.gpkg_file, mode="w", format=container_tar_format
3267 + )
3268 +
3269 + # GPKG version
3270 + gpkg_version_file = tarfile.TarInfo(self.gpkg_version)
3271 + gpkg_version_file.mtime = datetime.utcnow().timestamp()
3272 + container.addfile(gpkg_version_file)
3273 +
3274 + compression_cmd = self._get_compression_cmd()
3275 + # Metadata
3276 + self._add_metadata(container, metadata, compression_cmd)
3277 +
3278 + # Image
3279 + if self.create_signature:
3280 + checksum_info = checksum_helper(
3281 + self.settings, gpg_operation=checksum_helper.SIGNING
3282 + )
3283 + else:
3284 + checksum_info = checksum_helper(self.settings)
3285 +
3286 + paths = list(contents)
3287 + paths.sort()
3288 + image_tarinfo = self._create_tarinfo("image")
3289 + image_tarinfo.mtime = datetime.utcnow().timestamp()
3290 + with tar_stream_writer(
3291 + image_tarinfo, container, image_tar_format, compression_cmd, checksum_info
3292 + ) as image_writer:
3293 + with tarfile.open(
3294 + mode="w|", fileobj=image_writer, format=image_tar_format
3295 + ) as image_tar:
3296 + if len(paths) == 0:
3297 + tarinfo = image_tar.tarinfo("image")
3298 + tarinfo.type = tarfile.DIRTYPE
3299 + tarinfo.size = 0
3300 + tarinfo.mode = 0o755
3301 + image_tar.addfile(tarinfo)
3302 +
3303 + for path in paths:
3304 + try:
3305 + lst = os.lstat(path)
3306 + except OSError as e:
3307 + if e.errno != errno.ENOENT:
3308 + raise
3309 + del e
3310 + continue
3311 + contents_type = contents[path][0]
3312 + if path.startswith(root_dir):
3313 + arcname = "image/" + path[len(root_dir) :]
3314 + else:
3315 + raise ValueError("invalid root argument: '%s'" % root_dir)
3316 + live_path = path
3317 + if (
3318 + "dir" == contents_type
3319 + and not stat.S_ISDIR(lst.st_mode)
3320 + and os.path.isdir(live_path)
3321 + ):
3322 + # Even though this was a directory in the original ${D}, it exists
3323 + # as a symlink to a directory in the live filesystem. It must be
3324 + # recorded as a real directory in the tar file to ensure that tar
3325 + # can properly extract it's children.
3326 + live_path = os.path.realpath(live_path)
3327 + lst = os.lstat(live_path)
3328 +
3329 + # Since os.lstat() inside TarFile.gettarinfo() can trigger a
3330 + # UnicodeEncodeError when python has something other than utf_8
3331 + # return from sys.getfilesystemencoding() (as in bug #388773),
3332 + # we implement the needed functionality here, using the result
3333 + # of our successful lstat call. An alternative to this would be
3334 + # to pass in the fileobj argument to TarFile.gettarinfo(), so
3335 + # that it could use fstat instead of lstat. However, that would
3336 + # have the unwanted effect of dereferencing symlinks.
3337 +
3338 + tarinfo = image_tar.tarinfo(arcname)
3339 + tarinfo.mode = lst.st_mode
3340 + tarinfo.uid = lst.st_uid
3341 + tarinfo.gid = lst.st_gid
3342 + tarinfo.size = 0
3343 + tarinfo.mtime = lst.st_mtime
3344 + tarinfo.linkname = ""
3345 + if stat.S_ISREG(lst.st_mode):
3346 + inode = (lst.st_ino, lst.st_dev)
3347 + if (
3348 + lst.st_nlink > 1
3349 + and inode in image_tar.inodes
3350 + and arcname != image_tar.inodes[inode]
3351 + ):
3352 + tarinfo.type = tarfile.LNKTYPE
3353 + tarinfo.linkname = image_tar.inodes[inode]
3354 + else:
3355 + image_tar.inodes[inode] = arcname
3356 + tarinfo.type = tarfile.REGTYPE
3357 + tarinfo.size = lst.st_size
3358 + elif stat.S_ISDIR(lst.st_mode):
3359 + tarinfo.type = tarfile.DIRTYPE
3360 + elif stat.S_ISLNK(lst.st_mode):
3361 + tarinfo.type = tarfile.SYMTYPE
3362 + tarinfo.linkname = os.readlink(live_path)
3363 + else:
3364 + continue
3365 + try:
3366 + tarinfo.uname = pwd.getpwuid(tarinfo.uid)[0]
3367 + except KeyError:
3368 + pass
3369 + try:
3370 + tarinfo.gname = grp.getgrgid(tarinfo.gid)[0]
3371 + except KeyError:
3372 + pass
3373 +
3374 + if stat.S_ISREG(lst.st_mode):
3375 + if protect and protect(path):
3376 + protect_file.seek(0)
3377 + tarinfo.size = protect_file_size
3378 + image_tar.addfile(tarinfo, protect_file)
3379 + else:
3380 + path_bytes = _unicode_encode(
3381 + path, encoding=_encodings["fs"], errors="strict"
3382 + )
3383 +
3384 + with open(path_bytes, "rb") as f:
3385 + image_tar.addfile(tarinfo, f)
3386 +
3387 + else:
3388 + image_tar.addfile(tarinfo)
3389 +
3390 + image_tarinfo = container.getmember(image_tarinfo.name)
3391 + self._record_checksum(checksum_info, image_tarinfo)
3392 +
3393 + if self.create_signature:
3394 + self._add_signature(checksum_info, image_tarinfo, container)
3395 +
3396 + self._add_manifest(container)
3397 + container.close()
3398 +
3399 + def _record_checksum(self, checksum_info, tarinfo):
3400 + """
3401 + Record checksum result for the given file.
3402 + Replace old checksum if already exists.
3403 + """
3404 + for c in self.checksums:
3405 + if c[1] == tarinfo.name:
3406 + self.checksums.remove(c)
3407 + break
3408 +
3409 + checksum_record = ["DATA", tarinfo.name, str(tarinfo.size)]
3410 +
3411 + for c in checksum_info.libs:
3412 + checksum_record.append(c)
3413 + checksum_record.append(checksum_info.libs[c].hexdigest())
3414 +
3415 + self.checksums.append(checksum_record)
3416 +
3417 + def _add_manifest(self, container):
3418 + """
3419 + Add Manifest to the container based on current checksums.
3420 + Creare GPG signatue if needed.
3421 + """
3422 + manifest = io.BytesIO()
3423 +
3424 + for m in self.checksums:
3425 + manifest.write((" ".join(m) + "\n").encode("UTF-8"))
3426 +
3427 + if self.create_signature:
3428 + checksum_info = checksum_helper(
3429 + self.settings, gpg_operation=checksum_helper.SIGNING, detached=False
3430 + )
3431 + checksum_info.update(manifest.getvalue())
3432 + checksum_info.finish()
3433 + manifest.seek(0)
3434 + manifest.write(checksum_info.gpg_output)
3435 +
3436 + manifest_tarinfo = tarfile.TarInfo("Manifest")
3437 + manifest_tarinfo.size = manifest.tell()
3438 + manifest_tarinfo.mtime = datetime.utcnow().timestamp()
3439 + manifest.seek(0)
3440 + container.addfile(manifest_tarinfo, manifest)
3441 + manifest.close()
3442 +
3443 + def _load_manifest(self, manifest_string):
3444 + """
3445 + Check, load, and return manifest in a list by files
3446 + """
3447 + manifest = []
3448 + manifest_filenames = []
3449 +
3450 + for manifest_record in manifest_string.splitlines():
3451 + if manifest_record == "":
3452 + continue
3453 + manifest_record = manifest_record.strip().split()
3454 +
3455 + if manifest_record[0] != "DATA":
3456 + raise DigestException("invalied Manifest")
3457 +
3458 + if manifest_record[1] in manifest_filenames:
3459 + raise DigestException("Manifest duplicate file exists")
3460 +
3461 + try:
3462 + int(manifest_record[2])
3463 + except ValueError:
3464 + raise DigestException("Manifest invalied file size")
3465 +
3466 + manifest.append(manifest_record)
3467 + manifest_filenames.append(manifest_record[1])
3468 +
3469 + return manifest
3470 +
3471 + def _add_signature(self, checksum_info, tarinfo, container, manifest=True):
3472 + """
3473 + Add GPG signature for the given tarinfo file.
3474 + manifest: add to manifest
3475 + """
3476 + if checksum_info.gpg_output is None:
3477 + raise GPGException("GPG signature is not exists")
3478 +
3479 + signature = io.BytesIO(checksum_info.gpg_output)
3480 + signature_tarinfo = tarfile.TarInfo(tarinfo.name + ".sig")
3481 + signature_tarinfo.size = len(signature.getvalue())
3482 + signature_tarinfo.mtime = datetime.utcnow().timestamp()
3483 + container.addfile(signature_tarinfo, signature)
3484 +
3485 + if manifest:
3486 + signature_checksum_info = checksum_helper(self.settings)
3487 + signature.seek(0)
3488 + signature_checksum_info.update(signature.read())
3489 + signature_checksum_info.finish()
3490 + self._record_checksum(signature_checksum_info, signature_tarinfo)
3491 +
3492 + signature.close()
3493 +
3494 + def _verify_binpkg(self, metadata_only=False):
3495 + """
3496 + Verify current GPKG file.
3497 + """
3498 + # Check file path
3499 + if self.gpkg_file is None:
3500 + raise FileNotFound("no gpkg file provided")
3501 +
3502 + # Check if is file
3503 + if not os.path.isfile(self.gpkg_file):
3504 + raise FileNotFound(f"File not found {self.gpkg_file}")
3505 +
3506 + # Check if is tar file
3507 + with open(self.gpkg_file, "rb") as container:
3508 + container_tar_format = self._get_tar_format(container)
3509 + if container_tar_format is None:
3510 + raise InvalidBinaryPackageFormat(
3511 + f"Cannot identify tar format: {self.gpkg_file}"
3512 + )
3513 +
3514 + # Check container
3515 + with tarfile.open(self.gpkg_file, "r") as container:
3516 + try:
3517 + container_files = container.getnames()
3518 + except tarfile.ReadError:
3519 + raise InvalidBinaryPackageFormat(
3520 + f"Cannot read tar file: {self.gpkg_file}"
3521 + )
3522 +
3523 + # Check gpkg header
3524 + if self.gpkg_version not in container_files:
3525 + raise InvalidBinaryPackageFormat(f"Invalid gpkg file: {self.gpkg_file}")
3526 +
3527 + # If any signature exists, we assume all files have signature.
3528 + if any(f.endswith(".sig") for f in container_files):
3529 + signature_exist = True
3530 + else:
3531 + signature_exist = False
3532 +
3533 + # Check if all files are unique to avoid same name attack
3534 + container_files_unique = []
3535 + for f in container_files:
3536 + if f in container_files_unique:
3537 + raise InvalidBinaryPackageFormat(
3538 + "Duplicate file %s exist, potential attack?" % f
3539 + )
3540 + container_files_unique.append(f)
3541 +
3542 + del container_files_unique
3543 +
3544 + # Add all files to check list
3545 + unverified_files = container_files.copy()
3546 + unverified_files.remove(self.gpkg_version)
3547 +
3548 + # Check Manifest file
3549 + if "Manifest" not in unverified_files:
3550 + raise MissingSignature(f"Manifest not found: {self.gpkg_file}")
3551 +
3552 + manifest_file = container.extractfile("Manifest")
3553 + manifest_data = manifest_file.read()
3554 + manifest_file.close()
3555 +
3556 + if b"-----BEGIN PGP SIGNATURE-----" in manifest_data:
3557 + signature_exist = True
3558 +
3559 + # Check Manifest signature if needed.
3560 + # binpkg-ignore-signature can override this.
3561 + if self.request_signature or signature_exist:
3562 + checksum_info = checksum_helper(
3563 + self.settings, gpg_operation=checksum_helper.VERIFY, detached=False
3564 + )
3565 +
3566 + try:
3567 + checksum_info.update(manifest_data)
3568 + checksum_info.finish()
3569 + except (InvalidSignature, MissingSignature):
3570 + if self.verify_signature:
3571 + raise
3572 +
3573 + manifest_data = checksum_info.gpg_output
3574 + unverified_files.remove("Manifest")
3575 + else:
3576 + unverified_files.remove("Manifest")
3577 +
3578 + # Load manifest and create manifest check list
3579 + manifest = self._load_manifest(manifest_data.decode("UTF-8"))
3580 + unverified_manifest = manifest.copy()
3581 +
3582 + # Check all remaining files
3583 + for f in unverified_files.copy():
3584 + if f.endswith(".sig"):
3585 + f_signature = None
3586 + else:
3587 + f_signature = f + ".sig"
3588 +
3589 + # Find current file manifest record
3590 + manifest_record = None
3591 + for m in manifest:
3592 + if m[1] == f:
3593 + manifest_record = m
3594 +
3595 + if manifest_record is None:
3596 + raise DigestException(f"{f} checksum not found in {self.gpkg_file}")
3597 +
3598 + if int(manifest_record[2]) != int(container.getmember(f).size):
3599 + raise DigestException(
3600 + f"{f} file size mismatched in {self.gpkg_file}"
3601 + )
3602 +
3603 + # Ignore image file and signature if not needed
3604 + if os.path.basename(f).startswith("image") and metadata_only:
3605 + unverified_files.remove(f)
3606 + unverified_manifest.remove(manifest_record)
3607 + continue
3608 +
3609 + # Verify current file signature if needed
3610 + # binpkg-ignore-signature can override this.
3611 + if (
3612 + (self.request_signature or signature_exist)
3613 + and self.verify_signature
3614 + and f_signature
3615 + ):
3616 + if f_signature in unverified_files:
3617 + signature_file = container.extractfile(f_signature)
3618 + signature = signature_file.read()
3619 + signature_file.close()
3620 + checksum_info = checksum_helper(
3621 + self.settings,
3622 + gpg_operation=checksum_helper.VERIFY,
3623 + signature=signature,
3624 + )
3625 + else:
3626 + raise MissingSignature(
3627 + f"{f} signature not found in {self.gpkg_file}"
3628 + )
3629 + else:
3630 + checksum_info = checksum_helper(self.settings)
3631 +
3632 + # Verify current file checksum
3633 + f_io = container.extractfile(f)
3634 + while True:
3635 + buffer = f_io.read(HASHING_BLOCKSIZE)
3636 + if buffer:
3637 + checksum_info.update(buffer)
3638 + else:
3639 + checksum_info.finish()
3640 + break
3641 + f_io.close()
3642 +
3643 + # At least one supported checksum must be checked
3644 + verified_hash_count = 0
3645 + for c in checksum_info.libs:
3646 + try:
3647 + if (
3648 + checksum_info.libs[c].hexdigest().lower()
3649 + == manifest_record[manifest_record.index(c) + 1].lower()
3650 + ):
3651 + verified_hash_count += 1
3652 + else:
3653 + raise DigestException(
3654 + f"{f} checksum mismatched in {self.gpkg_file}"
3655 + )
3656 + except KeyError:
3657 + # Checksum method not supported
3658 + pass
3659 +
3660 + if verified_hash_count < 1:
3661 + raise DigestException(
3662 + f"{f} no supported checksum found in {self.gpkg_file}"
3663 + )
3664 +
3665 + # Current file verified
3666 + unverified_files.remove(f)
3667 + unverified_manifest.remove(manifest_record)
3668 +
3669 + # Check if any file IN Manifest but NOT IN binary package
3670 + if len(unverified_manifest) != 0:
3671 + raise DigestException(
3672 + f"Missing files: {str(unverified_manifest)} in {self.gpkg_file}"
3673 + )
3674 +
3675 + # Check if any file NOT IN Manifest but IN binary package
3676 + if len(unverified_files) != 0:
3677 + raise DigestException(
3678 + f"Unknown files exists: {str(unverified_files)} in {self.gpkg_file}"
3679 + )
3680 +
3681 + # Save current Manifest for other operations.
3682 + self.manifest_old = manifest.copy()
3683 +
3684 + def _generate_metadata_from_dir(self, metadata_dir):
3685 + """
3686 + read all files in metadata_dir and return as dict
3687 + """
3688 + metadata = {}
3689 + metadata_dir = normalize_path(
3690 + _unicode_decode(metadata_dir, encoding=_encodings["fs"], errors="strict")
3691 + )
3692 + for parent, dirs, files in os.walk(metadata_dir):
3693 + for f in files:
3694 + try:
3695 + f = _unicode_decode(f, encoding=_encodings["fs"], errors="strict")
3696 + except UnicodeDecodeError:
3697 + continue
3698 + with open(os.path.join(parent, f), "rb") as metafile:
3699 + metadata[f] = metafile.read()
3700 + return metadata
3701 +
3702 + def _get_binary_cmd(self, compression, mode):
3703 + """
3704 + get command list form portage and try match compressor
3705 + """
3706 + if compression not in _compressors:
3707 + raise InvalidCompressionMethod(compression)
3708 +
3709 + compressor = _compressors[compression]
3710 + if mode not in compressor:
3711 + raise InvalidCompressionMethod("{}: {}".format(compression, mode))
3712 +
3713 + cmd = shlex_split(varexpand(compressor[mode], mydict=self.settings))
3714 + # Filter empty elements that make Popen fail
3715 + cmd = [x for x in cmd if x != ""]
3716 +
3717 + if (not cmd) and ((mode + "_alt") in compressor):
3718 + cmd = shlex_split(
3719 + varexpand(compressor[mode + "_alt"], mydict=self.settings)
3720 + )
3721 + cmd = [x for x in cmd if x != ""]
3722 +
3723 + if not cmd:
3724 + raise CompressorNotFound(compression)
3725 + if not find_binary(cmd[0]):
3726 + raise CompressorNotFound(cmd[0])
3727 +
3728 + return cmd
3729 +
3730 + def _get_compression_cmd(self, compression=None):
3731 + """
3732 + return compression command for Popen
3733 + """
3734 + if compression is None:
3735 + compression = self.compression
3736 + if compression is None:
3737 + return None
3738 + else:
3739 + return self._get_binary_cmd(compression, "compress")
3740 +
3741 + def _get_decompression_cmd(self, compression=None):
3742 + """
3743 + return decompression command for Popen
3744 + """
3745 + if compression is None:
3746 + compression = self.compression
3747 + if compression is None:
3748 + return None
3749 + else:
3750 + return self._get_binary_cmd(compression, "decompress")
3751 +
3752 + def _get_tar_format(self, fileobj):
3753 + """
3754 + Try to detect tar version
3755 + """
3756 + old_position = fileobj.tell()
3757 + fileobj.seek(0x101)
3758 + magic = fileobj.read(8)
3759 + fileobj.seek(0x9C)
3760 + typeflag = fileobj.read(1)
3761 + fileobj.seek(old_position)
3762 +
3763 + if magic == b"ustar \x00":
3764 + return tarfile.GNU_FORMAT
3765 + elif magic == b"ustar\x0000":
3766 + if typeflag == b"x" or typeflag == b"g":
3767 + return tarfile.PAX_FORMAT
3768 + else:
3769 + return tarfile.USTAR_FORMAT
3770 +
3771 + return None
3772 +
3773 + def _get_tar_format_from_stats(
3774 + self,
3775 + image_max_prefix_length,
3776 + image_max_name_length,
3777 + image_max_linkname_length,
3778 + image_max_file_size,
3779 + image_total_size,
3780 + ):
3781 + """
3782 + Choose the corresponding tar format according to
3783 + the image information
3784 + """
3785 + # Max possible size in UStar is 8 GiB (8589934591 bytes)
3786 + # stored in 11 octets
3787 + # Use 8000000000, just in case we need add something extra
3788 +
3789 + # Total size > 8 GiB, container need use GNU tar format
3790 + if image_total_size < 8000000000:
3791 + container_tar_format = tarfile.USTAR_FORMAT
3792 + else:
3793 + container_tar_format = tarfile.GNU_FORMAT
3794 +
3795 + # Image at least one file > 8 GiB, image need use GNU tar format
3796 + if image_max_file_size < 8000000000:
3797 + image_tar_format = tarfile.USTAR_FORMAT
3798 + else:
3799 + image_tar_format = tarfile.GNU_FORMAT
3800 +
3801 + # UStar support max 155 prefix length, 100 file name and 100 link name,
3802 + # ends with \x00. If any exceeded, failback to GNU format.
3803 + if image_max_prefix_length >= 155:
3804 + image_tar_format = tarfile.GNU_FORMAT
3805 +
3806 + if image_max_name_length >= 100:
3807 + image_tar_format = tarfile.GNU_FORMAT
3808 +
3809 + if image_max_linkname_length >= 100:
3810 + image_tar_format = tarfile.GNU_FORMAT
3811 + return container_tar_format, image_tar_format
3812 +
3813 + def _check_pre_image_files(self, root_dir, image_prefix="image"):
3814 + """
3815 + Check the pre image files size and path, return the longest
3816 + path length, largest single file size, and total files size.
3817 + """
3818 + image_prefix_length = len(image_prefix) + 1
3819 + root_dir = os.path.join(
3820 + normalize_path(
3821 + _unicode_decode(root_dir, encoding=_encodings["fs"], errors="strict")
3822 + ),
3823 + "",
3824 + )
3825 + root_dir_length = len(
3826 + _unicode_encode(root_dir, encoding=_encodings["fs"], errors="strict")
3827 + )
3828 +
3829 + image_max_prefix_length = 0
3830 + image_max_name_length = 0
3831 + image_max_link_length = 0
3832 + image_max_file_size = 0
3833 + image_total_size = 0
3834 +
3835 + for parent, dirs, files in os.walk(root_dir):
3836 + parent = _unicode_decode(parent, encoding=_encodings["fs"], errors="strict")
3837 + for d in dirs:
3838 + try:
3839 + d = _unicode_decode(d, encoding=_encodings["fs"], errors="strict")
3840 + except UnicodeDecodeError as err:
3841 + writemsg(colorize("BAD", "\n*** %s\n\n" % err), noiselevel=-1)
3842 + raise
3843 +
3844 + d = os.path.join(parent, d)
3845 + prefix_length = (
3846 + len(_unicode_encode(d, encoding=_encodings["fs"], errors="strict"))
3847 + - root_dir_length
3848 + + image_prefix_length
3849 + )
3850 +
3851 + if os.path.islink(d):
3852 + path_link = os.readlink(d)
3853 + path_link_length = len(
3854 + _unicode_encode(
3855 + path_link, encoding=_encodings["fs"], errors="strict"
3856 + )
3857 + )
3858 + image_max_link_length = max(image_max_link_length, path_link_length)
3859 +
3860 + image_max_prefix_length = max(image_max_prefix_length, prefix_length)
3861 +
3862 + for f in files:
3863 + try:
3864 + f = _unicode_decode(f, encoding=_encodings["fs"], errors="strict")
3865 + except UnicodeDecodeError as err:
3866 + writemsg(colorize("BAD", "\n*** %s\n\n" % err), noiselevel=-1)
3867 + raise
3868 +
3869 + filename_length = len(
3870 + _unicode_encode(f, encoding=_encodings["fs"], errors="strict")
3871 + )
3872 + image_max_name_length = max(image_max_name_length, filename_length)
3873 +
3874 + f = os.path.join(parent, f)
3875 + path_length = (
3876 + len(_unicode_encode(f, encoding=_encodings["fs"], errors="strict"))
3877 + - root_dir_length
3878 + + image_prefix_length
3879 + )
3880 +
3881 + file_stat = os.lstat(f)
3882 +
3883 + if os.path.islink(f):
3884 + path_link = os.readlink(f)
3885 + path_link_length = len(
3886 + _unicode_encode(
3887 + path_link, encoding=_encodings["fs"], errors="strict"
3888 + )
3889 + )
3890 + elif file_stat.st_nlink > 1:
3891 + # Hardlink exists
3892 + path_link_length = path_length
3893 + else:
3894 + path_link_length = 0
3895 +
3896 + image_max_link_length = max(image_max_link_length, path_link_length)
3897 +
3898 + try:
3899 + file_size = os.path.getsize(f)
3900 + except FileNotFoundError:
3901 + # Ignore file not found if symlink to non-existing file
3902 + if os.path.islink(f):
3903 + continue
3904 + else:
3905 + raise
3906 + image_total_size += file_size
3907 + image_max_file_size = max(image_max_file_size, file_size)
3908 +
3909 + return (
3910 + image_max_prefix_length,
3911 + image_max_name_length,
3912 + image_max_link_length,
3913 + image_max_file_size,
3914 + image_total_size,
3915 + )
3916 +
3917 + def _check_pre_quickpkg_files(self, contents, root):
3918 + """
3919 + Check the pre quickpkg files size and path, return the longest
3920 + path length, largest single file size, and total files size.
3921 + """
3922 + root_dir = os.path.join(
3923 + normalize_path(
3924 + _unicode_decode(root, encoding=_encodings["fs"], errors="strict")
3925 + ),
3926 + "",
3927 + )
3928 + root_dir_length = len(
3929 + _unicode_encode(root_dir, encoding=_encodings["fs"], errors="strict")
3930 + )
3931 +
3932 + image_max_prefix_length = 0
3933 + image_max_name_length = 0
3934 + image_max_link_length = 0
3935 + image_max_file_size = 0
3936 + image_total_size = 0
3937 +
3938 + paths = list(contents)
3939 + for path in paths:
3940 + try:
3941 + path = _unicode_decode(path, encoding=_encodings["fs"], errors="strict")
3942 + except UnicodeDecodeError as err:
3943 + writemsg(colorize("BAD", "\n*** %s\n\n" % err), noiselevel=-1)
3944 + raise
3945 +
3946 + d, f = os.path.split(path)
3947 +
3948 + prefix_length = (
3949 + len(_unicode_encode(d, encoding=_encodings["fs"], errors="strict"))
3950 + - root_dir_length
3951 + )
3952 + image_max_prefix_length = max(image_max_prefix_length, prefix_length)
3953 +
3954 + filename_length = len(
3955 + _unicode_encode(f, encoding=_encodings["fs"], errors="strict")
3956 + )
3957 + image_max_name_length = max(image_max_name_length, filename_length)
3958 +
3959 + path_length = (
3960 + len(_unicode_encode(path, encoding=_encodings["fs"], errors="strict"))
3961 + - root_dir_length
3962 + )
3963 +
3964 + file_stat = os.lstat(path)
3965 +
3966 + if os.path.islink(path):
3967 + path_link = os.readlink(path)
3968 + path_link_length = len(
3969 + _unicode_encode(
3970 + path_link, encoding=_encodings["fs"], errors="strict"
3971 + )
3972 + )
3973 + elif file_stat.st_nlink > 1:
3974 + # Hardlink exists
3975 + path_link_length = path_length
3976 + else:
3977 + path_link_length = 0
3978 +
3979 + image_max_link_length = max(image_max_link_length, path_link_length)
3980 +
3981 + if os.path.isfile(path):
3982 + try:
3983 + file_size = os.path.getsize(path)
3984 + except FileNotFoundError:
3985 + # Ignore file not found if symlink to non-existing file
3986 + if os.path.islink(path):
3987 + continue
3988 + else:
3989 + raise
3990 + image_total_size += file_size
3991 + if file_size > image_max_file_size:
3992 + image_max_file_size = file_size
3993 +
3994 + return (
3995 + image_max_prefix_length,
3996 + image_max_name_length,
3997 + image_max_link_length,
3998 + image_max_file_size,
3999 + image_total_size,
4000 + )
4001 +
4002 + def _create_tarinfo(self, file_name):
4003 + """
4004 + Create new tarinfo for the new file
4005 + """
4006 + if self.compression is None:
4007 + ext = ""
4008 + elif self.compression in self.ext_list:
4009 + ext = self.ext_list[self.compression]
4010 + else:
4011 + raise InvalidCompressionMethod(self.compression)
4012 +
4013 + data_tarinfo = tarfile.TarInfo(
4014 + os.path.join(self.base_name, file_name + ".tar" + ext)
4015 + )
4016 + return data_tarinfo
4017 +
4018 + def _extract_filename_compression(self, file_name):
4019 + """
4020 + Extract the file basename and compression method
4021 + """
4022 + file_name = os.path.basename(file_name)
4023 + if file_name.endswith(".tar"):
4024 + return file_name[:-4], None
4025 +
4026 + for compression in self.ext_list:
4027 + if file_name.endswith(".tar" + self.ext_list[compression]):
4028 + return (
4029 + file_name[: -len(".tar" + self.ext_list[compression])],
4030 + compression,
4031 + )
4032 +
4033 + raise InvalidCompressionMethod(file_name)
4034 +
4035 + def _get_inner_tarinfo(self, tar, file_name):
4036 + """
4037 + Get inner tarinfo from given container.
4038 + Will try get file_name from correct basename first,
4039 + if it fail, try any file that have same name as file_name, and
4040 + return the first one.
4041 + """
4042 + if self.gpkg_version not in tar.getnames():
4043 + raise InvalidBinaryPackageFormat("Invalid gpkg file.")
4044 +
4045 + # Try get file with correct basename
4046 + inner_tarinfo = None
4047 + if self.base_name is None:
4048 + base_name = ""
4049 + else:
4050 + base_name = self.base_name
4051 + all_files = tar.getmembers()
4052 + for f in all_files:
4053 + if os.path.dirname(f.name) == base_name:
4054 + try:
4055 + f_name, f_comp = self._extract_filename_compression(f.name)
4056 + except InvalidCompressionMethod:
4057 + continue
4058 +
4059 + if f_name == file_name:
4060 + return f, f_comp
4061 +
4062 + # If failed, try get any file name matched
4063 + if inner_tarinfo is None:
4064 + for f in all_files:
4065 + try:
4066 + f_name, f_comp = self._extract_filename_compression(f.name)
4067 + except InvalidCompressionMethod:
4068 + continue
4069 + if f_name == file_name:
4070 + if self.base_name is not None:
4071 + writemsg(
4072 + colorize(
4073 + "WARN", "Package basename mismatched, using " + f.name
4074 + )
4075 + )
4076 + self.base_name_alt = os.path.dirname(f.name)
4077 + return f, f_comp
4078 +
4079 + # Not found
4080 + raise FileNotFound(f"File Not found: {file_name}")
4081
4082 diff --git a/lib/portage/package/ebuild/_config/special_env_vars.py b/lib/portage/package/ebuild/_config/special_env_vars.py
4083 index 06ae3aa39..9b2d77aea 100644
4084 --- a/lib/portage/package/ebuild/_config/special_env_vars.py
4085 +++ b/lib/portage/package/ebuild/_config/special_env_vars.py
4086 @@ -86,6 +86,7 @@ environ_whitelist += [
4087 "ACCEPT_LICENSE",
4088 "BASH_ENV",
4089 "BASH_FUNC____in_portage_iuse%%",
4090 + "BINPKG_FORMAT",
4091 "BROOT",
4092 "BUILD_PREFIX",
4093 "COLUMNS",
4094
4095 diff --git a/lib/portage/package/ebuild/config.py b/lib/portage/package/ebuild/config.py
4096 index b4d6862a3..8fe51784e 100644
4097 --- a/lib/portage/package/ebuild/config.py
4098 +++ b/lib/portage/package/ebuild/config.py
4099 @@ -41,6 +41,7 @@ from portage.const import (
4100 PORTAGE_BASE_PATH,
4101 PRIVATE_PATH,
4102 PROFILE_PATH,
4103 + SUPPORTED_GENTOO_BINPKG_FORMATS,
4104 USER_CONFIG_PATH,
4105 USER_VIRTUALS_FILE,
4106 )
4107 @@ -1513,6 +1514,15 @@ class config:
4108 noiselevel=-1,
4109 )
4110
4111 + binpkg_format = self.get("BINPKG_FORMAT")
4112 + if binpkg_format:
4113 + if binpkg_format not in SUPPORTED_GENTOO_BINPKG_FORMATS:
4114 + writemsg(
4115 + "!!! BINPKG_FORMAT contains invalid or "
4116 + "unsupported format: %s" % binpkg_fotmat,
4117 + noiselevel=-1,
4118 + )
4119 +
4120 binpkg_compression = self.get("BINPKG_COMPRESS")
4121 if binpkg_compression:
4122 try:
4123
4124 diff --git a/lib/portage/package/ebuild/doebuild.py b/lib/portage/package/ebuild/doebuild.py
4125 index ac627f555..eb719b1ea 100644
4126 --- a/lib/portage/package/ebuild/doebuild.py
4127 +++ b/lib/portage/package/ebuild/doebuild.py
4128 @@ -66,6 +66,7 @@ from portage.const import (
4129 INVALID_ENV_FILE,
4130 MISC_SH_BINARY,
4131 PORTAGE_PYM_PACKAGES,
4132 + SUPPORTED_GENTOO_BINPKG_FORMATS,
4133 )
4134 from portage.data import portage_gid, portage_uid, secpass, uid, userpriv_groups
4135 from portage.dbapi.porttree import _parse_uri_map
4136 @@ -649,6 +650,18 @@ def doebuild_environment(
4137 mysettings["KV"] = ""
4138 mysettings.backup_changes("KV")
4139
4140 + binpkg_format = mysettings.get(
4141 + "BINPKG_FORMAT", SUPPORTED_GENTOO_BINPKG_FORMATS[0]
4142 + )
4143 + if binpkg_format not in portage.const.SUPPORTED_GENTOO_BINPKG_FORMATS:
4144 + writemsg(
4145 + "!!! BINPKG_FORMAT contains invalid or "
4146 + "unsupported format: %s" % binpkg_fotmat,
4147 + noiselevel=-1,
4148 + )
4149 + binpkg_format = "xpak"
4150 + mysettings["BINPKG_FORMAT"] = binpkg_format
4151 +
4152 binpkg_compression = mysettings.get("BINPKG_COMPRESS", "bzip2")
4153 try:
4154 compression = _compressors[binpkg_compression]
4155
4156 diff --git a/lib/portage/tests/.gnupg/openpgp-revocs.d/06B3A311BD775C280D22A9305D90EA06352177F6.rev b/lib/portage/tests/.gnupg/openpgp-revocs.d/06B3A311BD775C280D22A9305D90EA06352177F6.rev
4157 new file mode 100644
4158 index 000000000..a6752fd30
4159 --- /dev/null
4160 +++ b/lib/portage/tests/.gnupg/openpgp-revocs.d/06B3A311BD775C280D22A9305D90EA06352177F6.rev
4161 @@ -0,0 +1,37 @@
4162 +This is a revocation certificate for the OpenPGP key:
4163 +
4164 +pub rsa4096 2020-07-14 [S]
4165 + 06B3A311BD775C280D22A9305D90EA06352177F6
4166 +uid Gentoo Portage Test Trusted Key (Test Only, Do NOT Trust!!!) (Gentoo Test Key) <test@×××××××.org>
4167 +
4168 +A revocation certificate is a kind of "kill switch" to publicly
4169 +declare that a key shall not anymore be used. It is not possible
4170 +to retract such a revocation certificate once it has been published.
4171 +
4172 +Use it to revoke this key in case of a compromise or loss of
4173 +the secret key. However, if the secret key is still accessible,
4174 +it is better to generate a new revocation certificate and give
4175 +a reason for the revocation. For details see the description of
4176 +of the gpg command "--generate-revocation" in the GnuPG manual.
4177 +
4178 +To avoid an accidental use of this file, a colon has been inserted
4179 +before the 5 dashes below. Remove this colon with a text editor
4180 +before importing and publishing this revocation certificate.
4181 +
4182 +:-----BEGIN PGP PUBLIC KEY BLOCK-----
4183 +Comment: This is a revocation certificate
4184 +
4185 +iQI2BCABCAAgFiEEBrOjEb13XCgNIqkwXZDqBjUhd/YFAl8OFTwCHQAACgkQXZDq
4186 +BjUhd/aXCA/+OgzosMDaDe5DNwkSi2yKdC2X18v8JcaYnXBUR93nXA0LVN7iVWkR
4187 +WEH3NuVspQZ5vK+3AHTKabqZFC/buA5oQOH01Ncd4lQISfOOhFiBn5DIPX31BVT0
4188 +iPmVkcxHAD4031ptP4oat6EFclT13SRchtlnAO04JofeHnzQIw3SozQGzXpAA1g4
4189 +BogQ0HWA88HzuEYYE+e/yzZL4D496X1DTaXksg0Py5c4SS6u5pND6lcUtAGxAwa9
4190 +sJFPs+coeURaRV99CrJfdh4u2OkvINTfrKOS6NFBQq6HVH5mLsRXZlcE4Oo4d+fN
4191 +XoPrTZnRUqpJADUdjHFvO/lr0fArJTS5IQCVBNFeCMlvgmUPeKWJ1r6Uiwe/UHor
4192 +9OP/tK97EqpsaXmHbo0jOUkn5iiUwy784+JBSSu/Q2NxqcBr74aaRdfxvs62dmv7
4193 +droCDQi3ebqTdnlDSaeCIWHyVlSroOhZ+ZETVy193K1X7VXFX3hYKiJ3G8QZwy3e
4194 +AlsVGjIHWfC+K+enIn+uwSUvOWPN3upK8kqMRuXvAOppFCE4sTqNbxUnHHXaqo/r
4195 +s1q6zVsWVILBk97BHlJph2IaqhV7iIgPU97/r4U/BT11VqDFdVSHcXcs4PDNs5vh
4196 +6qttaDiyDqZjwMr+0iDoouHxFpqY8e+3M2gycUgGr2XV6ML0pXE6BqA=
4197 +=nIjC
4198 +-----END PGP PUBLIC KEY BLOCK-----
4199
4200 diff --git a/lib/portage/tests/.gnupg/openpgp-revocs.d/8DEDA2CDED49C8809287B89D8812797DDF1DD192.rev b/lib/portage/tests/.gnupg/openpgp-revocs.d/8DEDA2CDED49C8809287B89D8812797DDF1DD192.rev
4201 new file mode 100644
4202 index 000000000..456e0aa50
4203 --- /dev/null
4204 +++ b/lib/portage/tests/.gnupg/openpgp-revocs.d/8DEDA2CDED49C8809287B89D8812797DDF1DD192.rev
4205 @@ -0,0 +1,37 @@
4206 +This is a revocation certificate for the OpenPGP key:
4207 +
4208 +pub rsa4096 2020-07-14 [S]
4209 + 8DEDA2CDED49C8809287B89D8812797DDF1DD192
4210 +uid Gentoo Portage Test Untrusted Key (Test Only, Do NOT Trust!!!) (Gentoo Test Key) <test@×××××××.org>
4211 +
4212 +A revocation certificate is a kind of "kill switch" to publicly
4213 +declare that a key shall not anymore be used. It is not possible
4214 +to retract such a revocation certificate once it has been published.
4215 +
4216 +Use it to revoke this key in case of a compromise or loss of
4217 +the secret key. However, if the secret key is still accessible,
4218 +it is better to generate a new revocation certificate and give
4219 +a reason for the revocation. For details see the description of
4220 +of the gpg command "--generate-revocation" in the GnuPG manual.
4221 +
4222 +To avoid an accidental use of this file, a colon has been inserted
4223 +before the 5 dashes below. Remove this colon with a text editor
4224 +before importing and publishing this revocation certificate.
4225 +
4226 +:-----BEGIN PGP PUBLIC KEY BLOCK-----
4227 +Comment: This is a revocation certificate
4228 +
4229 +iQI2BCABCAAgFiEEje2ize1JyICSh7idiBJ5fd8d0ZIFAl8OFXUCHQAACgkQiBJ5
4230 +fd8d0ZKdwxAAhmkC0V+OLyOU9PCV6ogD9/3b3nVqNIreoc+gxHTLmEvxiMSItqmq
4231 +DkcW9RJKAduA/HiLZQ8Yzxw+ldC6kuWqYEjNpSM54VDkrgOePi8W1bVDTCoSp7bo
4232 +0JOG4frieqIxA6lhAA2UppH7EPRXoODPLYqooNxWAs3xxVrR6eGAb5l8NXzrymvN
4233 +acFfOZ0s5FgADQskQHWVq6TaJn9DrcZxd+b+plSwPYDXqzTChKQ5jw7uMAPUvDkG
4234 +JUWgoKiKSrK64bslUq8aEDEZQ4uxjyEi6G0vO/wPL/ysGhS7KkPgCZsEfNjWjajb
4235 +jAsdvl1raoHxK/O7llMNr9uRAZtC56pJ//SRDc3kylZrkAo0RNoXQFowT739HWei
4236 +2UkCFDfz488VKKrOI8TzTyUvLFEo14ZAXGg1wdHaGnbYMzxpKjP15alOFo6fKIcS
4237 +Kz1f/Mab4wf4Sg0XAjQ9pnai1/U9ZF3/NSnRtYgJkLCrIEtRLrgSHJsLDPxjCfGV
4238 +jWszAbIk167aA0yKsSmuwkpc5bZqqBaTo904r857fxyt5Les6SOHsV7iNXt7F+am
4239 +03Y6u6m2eROba7M67l115vTyYcw5EZVp5j0nI81PXsC9X2DD1ci5xrNmPyEeupC4
4240 +7y7mcGbUYPJAJHJ0kHG4ZYLnNMl42ZYr1ssEeasDwUsLWgVqvx9RkKI=
4241 +=kVUQ
4242 +-----END PGP PUBLIC KEY BLOCK-----
4243
4244 diff --git a/lib/portage/tests/.gnupg/private-keys-v1.d/273B030399E7BEA66A9AD42216DE7CA17BA5D42E.key b/lib/portage/tests/.gnupg/private-keys-v1.d/273B030399E7BEA66A9AD42216DE7CA17BA5D42E.key
4245 new file mode 100644
4246 index 000000000..0bd1026ad
4247 Binary files /dev/null and b/lib/portage/tests/.gnupg/private-keys-v1.d/273B030399E7BEA66A9AD42216DE7CA17BA5D42E.key differ
4248
4249 diff --git a/lib/portage/tests/.gnupg/private-keys-v1.d/C99796FB85B0C3DF03314A11B5850C51167D6282.key b/lib/portage/tests/.gnupg/private-keys-v1.d/C99796FB85B0C3DF03314A11B5850C51167D6282.key
4250 new file mode 100644
4251 index 000000000..8e29ef43c
4252 Binary files /dev/null and b/lib/portage/tests/.gnupg/private-keys-v1.d/C99796FB85B0C3DF03314A11B5850C51167D6282.key differ
4253
4254 diff --git a/lib/portage/tests/.gnupg/pubring.kbx b/lib/portage/tests/.gnupg/pubring.kbx
4255 new file mode 100644
4256 index 000000000..f6367f83b
4257 Binary files /dev/null and b/lib/portage/tests/.gnupg/pubring.kbx differ
4258
4259 diff --git a/lib/portage/tests/.gnupg/trustdb.gpg b/lib/portage/tests/.gnupg/trustdb.gpg
4260 new file mode 100644
4261 index 000000000..db5b1023b
4262 Binary files /dev/null and b/lib/portage/tests/.gnupg/trustdb.gpg differ
4263
4264 diff --git a/lib/portage/tests/__init__.py b/lib/portage/tests/__init__.py
4265 index 02d9c4932..f74f992d7 100644
4266 --- a/lib/portage/tests/__init__.py
4267 +++ b/lib/portage/tests/__init__.py
4268 @@ -14,6 +14,7 @@ import portage
4269 from portage import os
4270 from portage import _encodings
4271 from portage import _unicode_decode
4272 +from portage.output import colorize
4273 from portage.proxy.objectproxy import ObjectProxy
4274
4275
4276 @@ -184,19 +185,43 @@ class TextTestResult(_TextTestResult):
4277 self.todoed = []
4278 self.portage_skipped = []
4279
4280 + def addSuccess(self, test):
4281 + super(_TextTestResult, self).addSuccess(test)
4282 + if self.showAll:
4283 + self.stream.writeln(colorize("GOOD", "ok"))
4284 + elif self.dots:
4285 + self.stream.write(colorize("GOOD", "."))
4286 + self.stream.flush()
4287 +
4288 + def addError(self, test, err):
4289 + super(_TextTestResult, self).addError(test, err)
4290 + if self.showAll:
4291 + self.stream.writeln(colorize("BAD", "ERROR"))
4292 + elif self.dots:
4293 + self.stream.write(colorize("HILITE", "E"))
4294 + self.stream.flush()
4295 +
4296 + def addFailure(self, test, err):
4297 + super(_TextTestResult, self).addFailure(test, err)
4298 + if self.showAll:
4299 + self.stream.writeln(colorize("BAD", "FAIL"))
4300 + elif self.dots:
4301 + self.stream.write(colorize("BAD", "F"))
4302 + self.stream.flush()
4303 +
4304 def addTodo(self, test, info):
4305 self.todoed.append((test, info))
4306 if self.showAll:
4307 - self.stream.writeln("TODO")
4308 + self.stream.writeln(colorize("BRACKET", "TODO"))
4309 elif self.dots:
4310 - self.stream.write(".")
4311 + self.stream.write(colorize("BRACKET", "."))
4312
4313 def addPortageSkip(self, test, info):
4314 self.portage_skipped.append((test, info))
4315 if self.showAll:
4316 - self.stream.writeln("SKIP")
4317 + self.stream.writeln(colorize("WARN", "SKIP"))
4318 elif self.dots:
4319 - self.stream.write(".")
4320 + self.stream.write(colorize("WARN", "."))
4321
4322 def printErrors(self):
4323 if self.dots or self.showAll:
4324 @@ -331,7 +356,7 @@ class TextTestRunner(unittest.TextTestRunner):
4325 )
4326 self.stream.writeln()
4327 if not result.wasSuccessful():
4328 - self.stream.write("FAILED (")
4329 + self.stream.write(colorize("BAD", "FAILED") + " (")
4330 failed = len(result.failures)
4331 errored = len(result.errors)
4332 if failed:
4333 @@ -342,7 +367,7 @@ class TextTestRunner(unittest.TextTestRunner):
4334 self.stream.write("errors=%d" % errored)
4335 self.stream.writeln(")")
4336 else:
4337 - self.stream.writeln("OK")
4338 + self.stream.writeln(colorize("GOOD", "OK"))
4339 return result
4340
4341
4342
4343 diff --git a/lib/portage/tests/emerge/test_simple.py b/lib/portage/tests/emerge/test_simple.py
4344 index 3a8bf3764..6d0dae0dd 100644
4345 --- a/lib/portage/tests/emerge/test_simple.py
4346 +++ b/lib/portage/tests/emerge/test_simple.py
4347 @@ -3,6 +3,7 @@
4348
4349 import argparse
4350 import subprocess
4351 +import sys
4352
4353 import portage
4354 from portage import shutil, os
4355 @@ -11,6 +12,7 @@ from portage.const import (
4356 BINREPOS_CONF_FILE,
4357 PORTAGE_PYM_PATH,
4358 USER_CONFIG_PATH,
4359 + SUPPORTED_GENTOO_BINPKG_FORMATS,
4360 )
4361 from portage.cache.mappings import Mapping
4362 from portage.process import find_binary
4363 @@ -19,6 +21,7 @@ from portage.tests.resolver.ResolverPlayground import ResolverPlayground
4364 from portage.tests.util.test_socks5 import AsyncHTTPServer
4365 from portage.util import ensure_dirs, find_updated_config_files, shlex_split
4366 from portage.util.futures import asyncio
4367 +from portage.output import colorize
4368
4369
4370 class BinhostContentMap(Mapping):
4371 @@ -223,17 +226,28 @@ call_has_and_best_version() {
4372 ),
4373 )
4374
4375 - playground = ResolverPlayground(
4376 - ebuilds=ebuilds, installed=installed, debug=debug
4377 - )
4378 + for binpkg_format in SUPPORTED_GENTOO_BINPKG_FORMATS:
4379 + with self.subTest(binpkg_format=binpkg_format):
4380 + print(colorize("HILITE", binpkg_format), end=" ... ")
4381 + sys.stdout.flush()
4382 + playground = ResolverPlayground(
4383 + ebuilds=ebuilds,
4384 + installed=installed,
4385 + debug=debug,
4386 + user_config={
4387 + "make.conf": ('BINPKG_FORMAT="%s"' % binpkg_format,),
4388 + },
4389 + )
4390
4391 - loop = asyncio._wrap_loop()
4392 - loop.run_until_complete(
4393 - asyncio.ensure_future(
4394 - self._async_test_simple(playground, metadata_xml_files, loop=loop),
4395 - loop=loop,
4396 - )
4397 - )
4398 + loop = asyncio._wrap_loop()
4399 + loop.run_until_complete(
4400 + asyncio.ensure_future(
4401 + self._async_test_simple(
4402 + playground, metadata_xml_files, loop=loop
4403 + ),
4404 + loop=loop,
4405 + )
4406 + )
4407
4408 async def _async_test_simple(self, playground, metadata_xml_files, loop):
4409
4410 @@ -327,6 +341,15 @@ call_has_and_best_version() {
4411 path=binhost_remote_path,
4412 )
4413
4414 + binpkg_format = settings.get(
4415 + "BINPKG_FORMAT", SUPPORTED_GENTOO_BINPKG_FORMATS[0]
4416 + )
4417 + self.assertIn(binpkg_format, ("xpak", "gpkg"))
4418 + if binpkg_format == "xpak":
4419 + foo_filename = "foo-0-1.xpak"
4420 + elif binpkg_format == "gpkg":
4421 + foo_filename = "foo-0-1.gpkg.tar"
4422 +
4423 test_commands = ()
4424
4425 if hasattr(argparse.ArgumentParser, "parse_intermixed_args"):
4426 @@ -387,13 +410,13 @@ call_has_and_best_version() {
4427 rm_cmd + ("-rf", cachedir),
4428 emerge_cmd + ("--oneshot", "virtual/foo"),
4429 lambda: self.assertFalse(
4430 - os.path.exists(os.path.join(pkgdir, "virtual", "foo", "foo-0-1.xpak"))
4431 + os.path.exists(os.path.join(pkgdir, "virtual", "foo", foo_filename))
4432 ),
4433 ({"FEATURES": "unmerge-backup"},)
4434 + emerge_cmd
4435 + ("--unmerge", "virtual/foo"),
4436 lambda: self.assertTrue(
4437 - os.path.exists(os.path.join(pkgdir, "virtual", "foo", "foo-0-1.xpak"))
4438 + os.path.exists(os.path.join(pkgdir, "virtual", "foo", foo_filename))
4439 ),
4440 emerge_cmd + ("--pretend", "dev-libs/A"),
4441 ebuild_cmd + (test_ebuild, "manifest", "clean", "package", "merge"),
4442
4443 diff --git a/lib/portage/tests/gpkg/__init__.py b/lib/portage/tests/gpkg/__init__.py
4444 new file mode 100644
4445 index 000000000..532918b6a
4446 --- /dev/null
4447 +++ b/lib/portage/tests/gpkg/__init__.py
4448 @@ -0,0 +1,2 @@
4449 +# Copyright 2011 Gentoo Foundation
4450 +# Distributed under the terms of the GNU General Public License v2
4451
4452 diff --git a/lib/portage/tests/gpkg/__test__.py b/lib/portage/tests/gpkg/__test__.py
4453 new file mode 100644
4454 index 000000000..e69de29bb
4455
4456 diff --git a/lib/portage/tests/gpkg/test_gpkg_checksum.py b/lib/portage/tests/gpkg/test_gpkg_checksum.py
4457 new file mode 100644
4458 index 000000000..c78045c34
4459 --- /dev/null
4460 +++ b/lib/portage/tests/gpkg/test_gpkg_checksum.py
4461 @@ -0,0 +1,396 @@
4462 +# Copright Gentoo Foundation 2006-2020
4463 +# Portage Unit Testing Functionality
4464 +
4465 +import io
4466 +import sys
4467 +import tarfile
4468 +import tempfile
4469 +from os import urandom
4470 +
4471 +from portage import os
4472 +from portage import shutil
4473 +from portage.tests import TestCase
4474 +from portage.tests.resolver.ResolverPlayground import ResolverPlayground
4475 +from portage.gpkg import gpkg
4476 +from portage.exception import (
4477 + InvalidBinaryPackageFormat,
4478 + DigestException,
4479 + MissingSignature,
4480 +)
4481 +
4482 +
4483 +class test_gpkg_checksum_case(TestCase):
4484 + def test_gpkg_missing_header(self):
4485 + if sys.version_info.major < 3:
4486 + self.skipTest("Not support Python 2")
4487 +
4488 + playground = ResolverPlayground(
4489 + user_config={
4490 + "make.conf": (
4491 + 'FEATURES="${FEATURES} -binpkg-signing '
4492 + '-binpkg-request-signature -gpg-keepalive"',
4493 + ),
4494 + }
4495 + )
4496 + tmpdir = tempfile.mkdtemp()
4497 +
4498 + try:
4499 + settings = playground.settings
4500 + orig_full_path = os.path.join(tmpdir, "orig/")
4501 + os.makedirs(orig_full_path)
4502 +
4503 + data = urandom(1048576)
4504 + with open(os.path.join(orig_full_path, "data"), "wb") as f:
4505 + f.write(data)
4506 +
4507 + binpkg_1 = gpkg(settings, "test", os.path.join(tmpdir, "test-1.gpkg.tar"))
4508 + binpkg_1.compress(orig_full_path, {})
4509 +
4510 + with tarfile.open(os.path.join(tmpdir, "test-1.gpkg.tar"), "r") as tar_1:
4511 + with tarfile.open(
4512 + os.path.join(tmpdir, "test-2.gpkg.tar"), "w"
4513 + ) as tar_2:
4514 + for f in tar_1.getmembers():
4515 + if f.name != binpkg_1.gpkg_version:
4516 + tar_2.addfile(f, tar_1.extractfile(f))
4517 +
4518 + binpkg_2 = gpkg(settings, "test", os.path.join(tmpdir, "test-2.gpkg.tar"))
4519 +
4520 + self.assertRaises(
4521 + InvalidBinaryPackageFormat,
4522 + binpkg_2.decompress,
4523 + os.path.join(tmpdir, "test"),
4524 + )
4525 + finally:
4526 + shutil.rmtree(tmpdir)
4527 + playground.cleanup()
4528 +
4529 + def test_gpkg_missing_manifest(self):
4530 + if sys.version_info.major < 3:
4531 + self.skipTest("Not support Python 2")
4532 +
4533 + playground = ResolverPlayground(
4534 + user_config={
4535 + "make.conf": (
4536 + 'FEATURES="${FEATURES} -binpkg-signing '
4537 + '-binpkg-request-signature -gpg-keepalive"',
4538 + ),
4539 + }
4540 + )
4541 + tmpdir = tempfile.mkdtemp()
4542 +
4543 + try:
4544 + settings = playground.settings
4545 + orig_full_path = os.path.join(tmpdir, "orig/")
4546 + os.makedirs(orig_full_path)
4547 +
4548 + data = urandom(1048576)
4549 + with open(os.path.join(orig_full_path, "data"), "wb") as f:
4550 + f.write(data)
4551 +
4552 + binpkg_1 = gpkg(settings, "test", os.path.join(tmpdir, "test-1.gpkg.tar"))
4553 + binpkg_1.compress(orig_full_path, {})
4554 +
4555 + with tarfile.open(os.path.join(tmpdir, "test-1.gpkg.tar"), "r") as tar_1:
4556 + with tarfile.open(
4557 + os.path.join(tmpdir, "test-2.gpkg.tar"), "w"
4558 + ) as tar_2:
4559 + for f in tar_1.getmembers():
4560 + if f.name != "Manifest":
4561 + tar_2.addfile(f, tar_1.extractfile(f))
4562 +
4563 + binpkg_2 = gpkg(settings, "test", os.path.join(tmpdir, "test-2.gpkg.tar"))
4564 +
4565 + self.assertRaises(
4566 + MissingSignature, binpkg_2.decompress, os.path.join(tmpdir, "test")
4567 + )
4568 + finally:
4569 + shutil.rmtree(tmpdir)
4570 + playground.cleanup()
4571 +
4572 + def test_gpkg_missing_files(self):
4573 + if sys.version_info.major < 3:
4574 + self.skipTest("Not support Python 2")
4575 +
4576 + playground = ResolverPlayground(
4577 + user_config={
4578 + "make.conf": (
4579 + 'FEATURES="${FEATURES} -binpkg-signing '
4580 + '-binpkg-request-signature -gpg-keepalive"',
4581 + ),
4582 + }
4583 + )
4584 + tmpdir = tempfile.mkdtemp()
4585 +
4586 + try:
4587 + settings = playground.settings
4588 + orig_full_path = os.path.join(tmpdir, "orig/")
4589 + os.makedirs(orig_full_path)
4590 +
4591 + data = urandom(1048576)
4592 + with open(os.path.join(orig_full_path, "data"), "wb") as f:
4593 + f.write(data)
4594 +
4595 + data = urandom(1048576)
4596 + with open(os.path.join(orig_full_path, "data2"), "wb") as f:
4597 + f.write(data)
4598 +
4599 + binpkg_1 = gpkg(settings, "test", os.path.join(tmpdir, "test-1.gpkg.tar"))
4600 + binpkg_1.compress(orig_full_path, {})
4601 +
4602 + with tarfile.open(os.path.join(tmpdir, "test-1.gpkg.tar"), "r") as tar_1:
4603 + with tarfile.open(
4604 + os.path.join(tmpdir, "test-2.gpkg.tar"), "w"
4605 + ) as tar_2:
4606 + for f in tar_1.getmembers():
4607 + if "image.tar" not in f.name:
4608 + tar_2.addfile(f, tar_1.extractfile(f))
4609 +
4610 + binpkg_2 = gpkg(settings, "test", os.path.join(tmpdir, "test-2.gpkg.tar"))
4611 +
4612 + self.assertRaises(
4613 + DigestException, binpkg_2.decompress, os.path.join(tmpdir, "test")
4614 + )
4615 + finally:
4616 + shutil.rmtree(tmpdir)
4617 + playground.cleanup()
4618 +
4619 + def test_gpkg_extra_files(self):
4620 + if sys.version_info.major < 3:
4621 + self.skipTest("Not support Python 2")
4622 +
4623 + playground = ResolverPlayground(
4624 + user_config={
4625 + "make.conf": (
4626 + 'FEATURES="${FEATURES} -binpkg-signing '
4627 + '-binpkg-request-signature -gpg-keepalive"',
4628 + ),
4629 + }
4630 + )
4631 + tmpdir = tempfile.mkdtemp()
4632 +
4633 + try:
4634 + settings = playground.settings
4635 + orig_full_path = os.path.join(tmpdir, "orig/")
4636 + os.makedirs(orig_full_path)
4637 +
4638 + data = urandom(1048576)
4639 + with open(os.path.join(orig_full_path, "data"), "wb") as f:
4640 + f.write(data)
4641 +
4642 + binpkg_1 = gpkg(settings, "test", os.path.join(tmpdir, "test-1.gpkg.tar"))
4643 + binpkg_1.compress(orig_full_path, {})
4644 +
4645 + with tarfile.open(os.path.join(tmpdir, "test-1.gpkg.tar"), "r") as tar_1:
4646 + with tarfile.open(
4647 + os.path.join(tmpdir, "test-2.gpkg.tar"), "w"
4648 + ) as tar_2:
4649 + for f in tar_1.getmembers():
4650 + tar_2.addfile(f, tar_1.extractfile(f))
4651 + data_tarinfo = tarfile.TarInfo("data2")
4652 + data_tarinfo.size = len(data)
4653 + data2 = io.BytesIO(data)
4654 + tar_2.addfile(data_tarinfo, data2)
4655 + data2.close()
4656 +
4657 + binpkg_2 = gpkg(settings, "test", os.path.join(tmpdir, "test-2.gpkg.tar"))
4658 +
4659 + self.assertRaises(
4660 + DigestException, binpkg_2.decompress, os.path.join(tmpdir, "test")
4661 + )
4662 + finally:
4663 + shutil.rmtree(tmpdir)
4664 + playground.cleanup()
4665 +
4666 + def test_gpkg_incorrect_checksum(self):
4667 + if sys.version_info.major < 3:
4668 + self.skipTest("Not support Python 2")
4669 +
4670 + playground = ResolverPlayground(
4671 + user_config={
4672 + "make.conf": (
4673 + 'FEATURES="${FEATURES} -binpkg-signing '
4674 + '-binpkg-request-signature -gpg-keepalive"',
4675 + ),
4676 + }
4677 + )
4678 + tmpdir = tempfile.mkdtemp()
4679 +
4680 + try:
4681 + settings = playground.settings
4682 + orig_full_path = os.path.join(tmpdir, "orig/")
4683 + os.makedirs(orig_full_path)
4684 +
4685 + data = urandom(1048576)
4686 + with open(os.path.join(orig_full_path, "data"), "wb") as f:
4687 + f.write(data)
4688 +
4689 + binpkg_1 = gpkg(settings, "test", os.path.join(tmpdir, "test-1.gpkg.tar"))
4690 + binpkg_1.compress(orig_full_path, {})
4691 +
4692 + with tarfile.open(os.path.join(tmpdir, "test-1.gpkg.tar"), "r") as tar_1:
4693 + with tarfile.open(
4694 + os.path.join(tmpdir, "test-2.gpkg.tar"), "w"
4695 + ) as tar_2:
4696 + for f in tar_1.getmembers():
4697 + if f.name == "Manifest":
4698 + data = io.BytesIO(tar_1.extractfile(f).read())
4699 + data_view = data.getbuffer()
4700 + data_view[-16:] = b"20a6d80ab0320fh9"
4701 + del data_view
4702 + tar_2.addfile(f, data)
4703 + data.close()
4704 + else:
4705 + tar_2.addfile(f, tar_1.extractfile(f))
4706 +
4707 + binpkg_2 = gpkg(settings, "test", os.path.join(tmpdir, "test-2.gpkg.tar"))
4708 +
4709 + self.assertRaises(
4710 + DigestException, binpkg_2.decompress, os.path.join(tmpdir, "test")
4711 + )
4712 + finally:
4713 + shutil.rmtree(tmpdir)
4714 + playground.cleanup()
4715 +
4716 + def test_gpkg_duplicate_files(self):
4717 + if sys.version_info.major < 3:
4718 + self.skipTest("Not support Python 2")
4719 +
4720 + playground = ResolverPlayground(
4721 + user_config={
4722 + "make.conf": (
4723 + 'FEATURES="${FEATURES} -binpkg-signing '
4724 + '-binpkg-request-signature -gpg-keepalive"',
4725 + ),
4726 + }
4727 + )
4728 + tmpdir = tempfile.mkdtemp()
4729 +
4730 + try:
4731 + settings = playground.settings
4732 + orig_full_path = os.path.join(tmpdir, "orig/")
4733 + os.makedirs(orig_full_path)
4734 +
4735 + data = urandom(100)
4736 + with open(os.path.join(orig_full_path, "data"), "wb") as f:
4737 + f.write(data)
4738 +
4739 + binpkg_1 = gpkg(settings, "test", os.path.join(tmpdir, "test-1.gpkg.tar"))
4740 + binpkg_1.compress(orig_full_path, {})
4741 +
4742 + with tarfile.open(os.path.join(tmpdir, "test-1.gpkg.tar"), "r") as tar_1:
4743 + with tarfile.open(
4744 + os.path.join(tmpdir, "test-2.gpkg.tar"), "w"
4745 + ) as tar_2:
4746 + for f in tar_1.getmembers():
4747 + tar_2.addfile(f, tar_1.extractfile(f))
4748 + tar_2.addfile(f, tar_1.extractfile(f))
4749 +
4750 + binpkg_2 = gpkg(settings, "test", os.path.join(tmpdir, "test-2.gpkg.tar"))
4751 +
4752 + self.assertRaises(
4753 + InvalidBinaryPackageFormat,
4754 + binpkg_2.decompress,
4755 + os.path.join(tmpdir, "test"),
4756 + )
4757 + finally:
4758 + shutil.rmtree(tmpdir)
4759 + playground.cleanup()
4760 +
4761 + def test_gpkg_manifest_duplicate_files(self):
4762 + if sys.version_info.major < 3:
4763 + self.skipTest("Not support Python 2")
4764 +
4765 + playground = ResolverPlayground(
4766 + user_config={
4767 + "make.conf": (
4768 + 'FEATURES="${FEATURES} -binpkg-signing '
4769 + '-binpkg-request-signature -gpg-keepalive"',
4770 + ),
4771 + }
4772 + )
4773 + tmpdir = tempfile.mkdtemp()
4774 +
4775 + try:
4776 + settings = playground.settings
4777 + orig_full_path = os.path.join(tmpdir, "orig/")
4778 + os.makedirs(orig_full_path)
4779 +
4780 + data = urandom(100)
4781 + with open(os.path.join(orig_full_path, "data"), "wb") as f:
4782 + f.write(data)
4783 +
4784 + binpkg_1 = gpkg(settings, "test", os.path.join(tmpdir, "test-1.gpkg.tar"))
4785 + binpkg_1.compress(orig_full_path, {})
4786 +
4787 + with tarfile.open(os.path.join(tmpdir, "test-1.gpkg.tar"), "r") as tar_1:
4788 + with tarfile.open(
4789 + os.path.join(tmpdir, "test-2.gpkg.tar"), "w"
4790 + ) as tar_2:
4791 + for f in tar_1.getmembers():
4792 + if f.name == "Manifest":
4793 + manifest = tar_1.extractfile(f).read()
4794 + data = io.BytesIO(manifest)
4795 + data.seek(io.SEEK_END)
4796 + data.write(b"\n")
4797 + data.write(manifest)
4798 + f.size = data.tell()
4799 + data.seek(0)
4800 + tar_2.addfile(f, data)
4801 + data.close()
4802 + else:
4803 + tar_2.addfile(f, tar_1.extractfile(f))
4804 +
4805 + binpkg_2 = gpkg(settings, "test", os.path.join(tmpdir, "test-2.gpkg.tar"))
4806 +
4807 + self.assertRaises(
4808 + DigestException, binpkg_2.decompress, os.path.join(tmpdir, "test")
4809 + )
4810 + finally:
4811 + shutil.rmtree(tmpdir)
4812 + playground.cleanup()
4813 +
4814 + def test_gpkg_different_size_file(self):
4815 + if sys.version_info.major < 3:
4816 + self.skipTest("Not support Python 2")
4817 +
4818 + playground = ResolverPlayground(
4819 + user_config={
4820 + "make.conf": (
4821 + 'FEATURES="${FEATURES} -binpkg-signing '
4822 + '-binpkg-request-signature -gpg-keepalive"',
4823 + ),
4824 + }
4825 + )
4826 + tmpdir = tempfile.mkdtemp()
4827 +
4828 + try:
4829 + settings = playground.settings
4830 + orig_full_path = os.path.join(tmpdir, "orig/")
4831 + os.makedirs(orig_full_path)
4832 +
4833 + data = urandom(100)
4834 + with open(os.path.join(orig_full_path, "data"), "wb") as f:
4835 + f.write(data)
4836 +
4837 + binpkg_1 = gpkg(settings, "test", os.path.join(tmpdir, "test-1.gpkg.tar"))
4838 + binpkg_1.compress(orig_full_path, {})
4839 +
4840 + with tarfile.open(os.path.join(tmpdir, "test-1.gpkg.tar"), "r") as tar_1:
4841 + with tarfile.open(
4842 + os.path.join(tmpdir, "test-2.gpkg.tar"), "w"
4843 + ) as tar_2:
4844 + for f in tar_1.getmembers():
4845 + tar_2.addfile(f, tar_1.extractfile(f))
4846 + tar_2.addfile(f, tar_1.extractfile(f))
4847 +
4848 + binpkg_2 = gpkg(settings, "test", os.path.join(tmpdir, "test-2.gpkg.tar"))
4849 +
4850 + self.assertRaises(
4851 + InvalidBinaryPackageFormat,
4852 + binpkg_2.decompress,
4853 + os.path.join(tmpdir, "test"),
4854 + )
4855 + finally:
4856 + shutil.rmtree(tmpdir)
4857 + playground.cleanup()
4858
4859 diff --git a/lib/portage/tests/gpkg/test_gpkg_gpg.py b/lib/portage/tests/gpkg/test_gpkg_gpg.py
4860 new file mode 100644
4861 index 000000000..906608dff
4862 --- /dev/null
4863 +++ b/lib/portage/tests/gpkg/test_gpkg_gpg.py
4864 @@ -0,0 +1,398 @@
4865 +# Copright Gentoo Foundation 2006-2020
4866 +# Portage Unit Testing Functionality
4867 +
4868 +import io
4869 +import sys
4870 +import tarfile
4871 +import tempfile
4872 +from os import urandom
4873 +
4874 +from portage import os
4875 +from portage import shutil
4876 +from portage.tests import TestCase
4877 +from portage.tests.resolver.ResolverPlayground import ResolverPlayground
4878 +from portage.gpkg import gpkg
4879 +from portage.gpg import GPG
4880 +from portage.exception import MissingSignature, InvalidSignature
4881 +
4882 +
4883 +class test_gpkg_gpg_case(TestCase):
4884 + def test_gpkg_missing_manifest_signature(self):
4885 + if sys.version_info.major < 3:
4886 + self.skipTest("Not support Python 2")
4887 +
4888 + playground = ResolverPlayground(
4889 + user_config={
4890 + "make.conf": (
4891 + 'FEATURES="${FEATURES} binpkg-signing ' 'binpkg-request-signature"',
4892 + 'BINPKG_FORMAT="gpkg"',
4893 + ),
4894 + }
4895 + )
4896 + tmpdir = tempfile.mkdtemp()
4897 +
4898 + try:
4899 + settings = playground.settings
4900 + gpg = GPG(settings)
4901 + gpg.unlock()
4902 + orig_full_path = os.path.join(tmpdir, "orig/")
4903 + os.makedirs(orig_full_path)
4904 +
4905 + data = urandom(1048576)
4906 + with open(os.path.join(orig_full_path, "data"), "wb") as f:
4907 + f.write(data)
4908 +
4909 + binpkg_1 = gpkg(settings, "test", os.path.join(tmpdir, "test-1.gpkg.tar"))
4910 + binpkg_1.compress(orig_full_path, {})
4911 +
4912 + with tarfile.open(os.path.join(tmpdir, "test-1.gpkg.tar"), "r") as tar_1:
4913 + with tarfile.open(
4914 + os.path.join(tmpdir, "test-2.gpkg.tar"), "w"
4915 + ) as tar_2:
4916 + for f in tar_1.getmembers():
4917 + if f.name == "Manifest":
4918 + manifest = tar_1.extractfile(f).read().decode("UTF-8")
4919 + manifest = manifest.replace(
4920 + "-----BEGIN PGP SIGNATURE-----", ""
4921 + )
4922 + manifest = manifest.replace(
4923 + "-----END PGP SIGNATURE-----", ""
4924 + )
4925 + manifest_data = io.BytesIO(manifest.encode("UTF-8"))
4926 + manifest_data.seek(0, io.SEEK_END)
4927 + f.size = manifest_data.tell()
4928 + manifest_data.seek(0)
4929 + tar_2.addfile(f, manifest_data)
4930 + else:
4931 + tar_2.addfile(f, tar_1.extractfile(f))
4932 +
4933 + binpkg_2 = gpkg(settings, "test", os.path.join(tmpdir, "test-2.gpkg.tar"))
4934 +
4935 + self.assertRaises(
4936 + InvalidSignature, binpkg_2.decompress, os.path.join(tmpdir, "test")
4937 + )
4938 + finally:
4939 + shutil.rmtree(tmpdir)
4940 + playground.cleanup()
4941 +
4942 + def test_gpkg_missing_signature(self):
4943 + if sys.version_info.major < 3:
4944 + self.skipTest("Not support Python 2")
4945 +
4946 + playground = ResolverPlayground(
4947 + user_config={
4948 + "make.conf": (
4949 + 'FEATURES="${FEATURES} binpkg-signing ' 'binpkg-request-signature"',
4950 + 'BINPKG_FORMAT="gpkg"',
4951 + ),
4952 + }
4953 + )
4954 + tmpdir = tempfile.mkdtemp()
4955 +
4956 + try:
4957 + settings = playground.settings
4958 + gpg = GPG(settings)
4959 + gpg.unlock()
4960 + orig_full_path = os.path.join(tmpdir, "orig/")
4961 + os.makedirs(orig_full_path)
4962 +
4963 + data = urandom(1048576)
4964 + with open(os.path.join(orig_full_path, "data"), "wb") as f:
4965 + f.write(data)
4966 +
4967 + binpkg_1 = gpkg(settings, "test", os.path.join(tmpdir, "test-1.gpkg.tar"))
4968 + binpkg_1.compress(orig_full_path, {})
4969 +
4970 + with tarfile.open(os.path.join(tmpdir, "test-1.gpkg.tar"), "r") as tar_1:
4971 + with tarfile.open(
4972 + os.path.join(tmpdir, "test-2.gpkg.tar"), "w"
4973 + ) as tar_2:
4974 + for f in tar_1.getmembers():
4975 + if f.name.endswith(".sig"):
4976 + pass
4977 + else:
4978 + tar_2.addfile(f, tar_1.extractfile(f))
4979 +
4980 + binpkg_2 = gpkg(settings, "test", os.path.join(tmpdir, "test-2.gpkg.tar"))
4981 + self.assertRaises(
4982 + MissingSignature, binpkg_2.decompress, os.path.join(tmpdir, "test")
4983 + )
4984 +
4985 + finally:
4986 + shutil.rmtree(tmpdir)
4987 + playground.cleanup()
4988 +
4989 + def test_gpkg_ignore_signature(self):
4990 + if sys.version_info.major < 3:
4991 + self.skipTest("Not support Python 2")
4992 +
4993 + playground = ResolverPlayground(
4994 + user_config={
4995 + "make.conf": (
4996 + 'FEATURES="${FEATURES} binpkg-signing ' 'binpkg-ignore-signature"',
4997 + 'BINPKG_FORMAT="gpkg"',
4998 + ),
4999 + }
5000 + )
5001 + tmpdir = tempfile.mkdtemp()
5002 +
5003 + try:
5004 + settings = playground.settings
5005 + gpg = GPG(settings)
5006 + gpg.unlock()
5007 + orig_full_path = os.path.join(tmpdir, "orig/")
5008 + os.makedirs(orig_full_path)
5009 +
5010 + data = urandom(1048576)
5011 + with open(os.path.join(orig_full_path, "data"), "wb") as f:
5012 + f.write(data)
5013 +
5014 + binpkg_1 = gpkg(settings, "test", os.path.join(tmpdir, "test-1.gpkg.tar"))
5015 + binpkg_1.compress(orig_full_path, {})
5016 +
5017 + with tarfile.open(os.path.join(tmpdir, "test-1.gpkg.tar"), "r") as tar_1:
5018 + with tarfile.open(
5019 + os.path.join(tmpdir, "test-2.gpkg.tar"), "w"
5020 + ) as tar_2:
5021 + for f in tar_1.getmembers():
5022 + if f.name == "Manifest.sig":
5023 + pass
5024 + else:
5025 + tar_2.addfile(f, tar_1.extractfile(f))
5026 +
5027 + binpkg_2 = gpkg(settings, "test", os.path.join(tmpdir, "test-2.gpkg.tar"))
5028 + binpkg_2.decompress(os.path.join(tmpdir, "test"))
5029 + finally:
5030 + shutil.rmtree(tmpdir)
5031 + playground.cleanup()
5032 +
5033 + def test_gpkg_auto_use_signature(self):
5034 + if sys.version_info.major < 3:
5035 + self.skipTest("Not support Python 2")
5036 +
5037 + playground = ResolverPlayground(
5038 + user_config={
5039 + "make.conf": (
5040 + 'FEATURES="${FEATURES} binpkg-signing '
5041 + '-binpkg-request-signature"',
5042 + 'BINPKG_FORMAT="gpkg"',
5043 + ),
5044 + }
5045 + )
5046 + tmpdir = tempfile.mkdtemp()
5047 +
5048 + try:
5049 + settings = playground.settings
5050 + gpg = GPG(settings)
5051 + gpg.unlock()
5052 + orig_full_path = os.path.join(tmpdir, "orig/")
5053 + os.makedirs(orig_full_path)
5054 +
5055 + data = urandom(1048576)
5056 + with open(os.path.join(orig_full_path, "data"), "wb") as f:
5057 + f.write(data)
5058 +
5059 + binpkg_1 = gpkg(settings, "test", os.path.join(tmpdir, "test-1.gpkg.tar"))
5060 + binpkg_1.compress(orig_full_path, {})
5061 +
5062 + with tarfile.open(os.path.join(tmpdir, "test-1.gpkg.tar"), "r") as tar_1:
5063 + with tarfile.open(
5064 + os.path.join(tmpdir, "test-2.gpkg.tar"), "w"
5065 + ) as tar_2:
5066 + for f in tar_1.getmembers():
5067 + if f.name.endswith(".sig"):
5068 + pass
5069 + else:
5070 + tar_2.addfile(f, tar_1.extractfile(f))
5071 +
5072 + binpkg_2 = gpkg(settings, "test", os.path.join(tmpdir, "test-2.gpkg.tar"))
5073 + self.assertRaises(
5074 + MissingSignature, binpkg_2.decompress, os.path.join(tmpdir, "test")
5075 + )
5076 + finally:
5077 + shutil.rmtree(tmpdir)
5078 + playground.cleanup()
5079 +
5080 + def test_gpkg_invalid_signature(self):
5081 + if sys.version_info.major < 3:
5082 + self.skipTest("Not support Python 2")
5083 +
5084 + playground = ResolverPlayground(
5085 + user_config={
5086 + "make.conf": (
5087 + 'FEATURES="${FEATURES} binpkg-signing ' 'binpkg-request-signature"',
5088 + 'BINPKG_FORMAT="gpkg"',
5089 + ),
5090 + }
5091 + )
5092 + tmpdir = tempfile.mkdtemp()
5093 +
5094 + try:
5095 + settings = playground.settings
5096 + gpg = GPG(settings)
5097 + gpg.unlock()
5098 + orig_full_path = os.path.join(tmpdir, "orig/")
5099 + os.makedirs(orig_full_path)
5100 +
5101 + data = urandom(1048576)
5102 + with open(os.path.join(orig_full_path, "data"), "wb") as f:
5103 + f.write(data)
5104 +
5105 + binpkg_1 = gpkg(settings, "test", os.path.join(tmpdir, "test-1.gpkg.tar"))
5106 + binpkg_1.compress(orig_full_path, {})
5107 +
5108 + with tarfile.open(os.path.join(tmpdir, "test-1.gpkg.tar"), "r") as tar_1:
5109 + with tarfile.open(
5110 + os.path.join(tmpdir, "test-2.gpkg.tar"), "w"
5111 + ) as tar_2:
5112 + for f in tar_1.getmembers():
5113 + if f.name == "Manifest":
5114 + sig = b"""
5115 +-----BEGIN PGP SIGNED MESSAGE-----
5116 +Hash: SHA512
5117 +
5118 +DATA test/image.tar.zst 1049649 BLAKE2B 3112adba9c09023962f26d9dcbf8e74107c05220f2f29aa2ce894f8a4104c3bb238f87095df73735befcf1e1f6039fc3abf4defa87e68ce80f33dd01e09c055a SHA512 9f584727f2e20a50a30e0077b94082c8c1f517ebfc9978eb3281887e24458108e73d1a2ce82eb0b59f5df7181597e4b0a297ae68bbfb36763aa052e6bdbf2c59
5119 +DATA test/image.tar.zst.sig 833 BLAKE2B 214724ae4ff9198879c8c960fd8167632e27982c2278bb873f195abe75b75afa1ebed4c37ec696f5f5bc35c3a1184b60e0b50d56695b072b254f730db01eddb5 SHA512 67316187da8bb6b7a5f9dc6a42ed5c7d72c6184483a97f23c0bebd8b187ac9268e0409eb233c935101606768718c99eaa5699037d6a68c2d88c9ed5331a3f73c
5120 +-----BEGIN PGP SIGNATURE-----
5121 +
5122 +iQIzBAEBCgAdFiEEBrOjEb13XCgNIqkwXZDqBjUhd/YFAmFazXEACgkQXZDqBjUh
5123 +d/YFZA//eiXkYAS2NKxim6Ppr1HcZdjU1f6H+zyQzC7OdPkAh7wsVXpSr1aq+giD
5124 +G4tNtI6nsFokpA5CMhDf+ffBofKmFY5plk9zyQHr43N/RS5G6pcb2LHk0mQqgIdB
5125 +EsZRRD75Na4uGDWjuNHRmsasPTsc9qyW7FLckjwUsVmk9foAoiLYYaTsilsEGqXD
5126 +Bl/Z6PaQXvdd8txbcP6dOXfhVT06b+RWcnHI06KQrmFkZjZQh/7bCIeCVwNbXr7d
5127 +Obo8SVzCrQbTONei57AkyuRfnPqBfP61k8rQtcDUmCckQQfyaRwoW2nDIewOPfIH
5128 +xfvM137to2GEI2RR1TpWmGfu3iQzgC71f4svdX9Tyi5N7aFmfud7LZs6/Un3IdVk
5129 +ZH9/AmRzeH6hKllqSv/6WuhjsTNvr0bOzGbskkhqlLga2tml08gHFYOMWRJb/bRz
5130 +N8FZMhHzFoc0hsG8SU9uC+OeW+y5NdqpbRnQwgABmAiKEpgAPnABTsr0HjyxvjY+
5131 +uCUdvMMHvnTxTjNEZ3Q+UQ2VsSoZzPbW9Y4PuM0XxxmTI8htdn4uIhy9dLNPsJmB
5132 +eTE8aov/1uKq9VMsYC8wcx5vLMaR7/O/9XstP+r6PaZwiLlyrKHGexV4O52sj6LC
5133 +qGAN3VUF+8EsdcsV781H0F86PANhyBgEYTGDrnItTGe3/vAPjCo=
5134 +=S/Vn
5135 +-----END PGP SIGNATURE-----
5136 +"""
5137 + data = io.BytesIO(sig)
5138 + f.size = len(sig)
5139 + tar_2.addfile(f, data)
5140 + data.close()
5141 + else:
5142 + tar_2.addfile(f, tar_1.extractfile(f))
5143 +
5144 + binpkg_2 = gpkg(settings, "test", os.path.join(tmpdir, "test-2.gpkg.tar"))
5145 + self.assertRaises(
5146 + InvalidSignature, binpkg_2.decompress, os.path.join(tmpdir, "test")
5147 + )
5148 + finally:
5149 + shutil.rmtree(tmpdir)
5150 + playground.cleanup()
5151 +
5152 + def test_gpkg_untrusted_signature(self):
5153 + if sys.version_info.major < 3:
5154 + self.skipTest("Not support Python 2")
5155 +
5156 + gpg_test_path = os.environ["PORTAGE_GNUPGHOME"]
5157 +
5158 + playground = ResolverPlayground(
5159 + user_config={
5160 + "make.conf": (
5161 + 'FEATURES="${FEATURES} binpkg-signing ' 'binpkg-request-signature"',
5162 + 'BINPKG_FORMAT="gpkg"',
5163 + f'BINPKG_GPG_SIGNING_BASE_COMMAND="flock {gpg_test_path}/portage-binpkg-gpg.lock /usr/bin/gpg --sign --armor --batch --no-tty --yes --pinentry-mode loopback --passphrase GentooTest [PORTAGE_CONFIG]"',
5164 + 'BINPKG_GPG_SIGNING_DIGEST="SHA512"',
5165 + f'BINPKG_GPG_SIGNING_GPG_HOME="{gpg_test_path}"',
5166 + 'BINPKG_GPG_SIGNING_KEY="0x8812797DDF1DD192"',
5167 + 'BINPKG_GPG_VERIFY_BASE_COMMAND="/usr/bin/gpg --verify --batch --no-tty --yes --no-auto-check-trustdb --status-fd 1 [PORTAGE_CONFIG] [SIGNATURE]"',
5168 + f'BINPKG_GPG_VERIFY_GPG_HOME="{gpg_test_path}"',
5169 + ),
5170 + }
5171 + )
5172 + tmpdir = tempfile.mkdtemp()
5173 +
5174 + try:
5175 + settings = playground.settings
5176 + gpg = GPG(settings)
5177 + gpg.unlock()
5178 + orig_full_path = os.path.join(tmpdir, "orig/")
5179 + os.makedirs(orig_full_path)
5180 +
5181 + data = urandom(1048576)
5182 + with open(os.path.join(orig_full_path, "data"), "wb") as f:
5183 + f.write(data)
5184 +
5185 + binpkg_1 = gpkg(settings, "test", os.path.join(tmpdir, "test-1.gpkg.tar"))
5186 + binpkg_1.compress(orig_full_path, {})
5187 +
5188 + binpkg_2 = gpkg(settings, "test", os.path.join(tmpdir, "test-1.gpkg.tar"))
5189 + self.assertRaises(
5190 + InvalidSignature, binpkg_2.decompress, os.path.join(tmpdir, "test")
5191 + )
5192 +
5193 + finally:
5194 + shutil.rmtree(tmpdir)
5195 + playground.cleanup()
5196 +
5197 + def test_gpkg_unknown_signature(self):
5198 + if sys.version_info.major < 3:
5199 + self.skipTest("Not support Python 2")
5200 +
5201 + playground = ResolverPlayground(
5202 + user_config={
5203 + "make.conf": (
5204 + 'FEATURES="${FEATURES} binpkg-signing ' 'binpkg-request-signature"',
5205 + 'BINPKG_FORMAT="gpkg"',
5206 + ),
5207 + }
5208 + )
5209 + tmpdir = tempfile.mkdtemp()
5210 +
5211 + try:
5212 + settings = playground.settings
5213 + gpg = GPG(settings)
5214 + gpg.unlock()
5215 + orig_full_path = os.path.join(tmpdir, "orig/")
5216 + os.makedirs(orig_full_path)
5217 +
5218 + data = urandom(1048576)
5219 + with open(os.path.join(orig_full_path, "data"), "wb") as f:
5220 + f.write(data)
5221 +
5222 + binpkg_1 = gpkg(settings, "test", os.path.join(tmpdir, "test-1.gpkg.tar"))
5223 + binpkg_1.compress(orig_full_path, {})
5224 +
5225 + with tarfile.open(os.path.join(tmpdir, "test-1.gpkg.tar"), "r") as tar_1:
5226 + with tarfile.open(
5227 + os.path.join(tmpdir, "test-2.gpkg.tar"), "w"
5228 + ) as tar_2:
5229 + for f in tar_1.getmembers():
5230 + if f.name == "Manifest":
5231 + sig = b"""
5232 +-----BEGIN PGP SIGNED MESSAGE-----
5233 +Hash: SHA256
5234 +
5235 +
5236 +DATA test/image.tar.zst 1049649 BLAKE2B 3112adba9c09023962f26d9dcbf8e74107c05220f2f29aa2ce894f8a4104c3bb238f87095df73735befcf1e1f6039fc3abf4defa87e68ce80f33dd01e09c055a SHA512 9f584727f2e20a50a30e0077b94082c8c1f517ebfc9978eb3281887e24458108e73d1a2ce82eb0b59f5df7181597e4b0a297ae68bbfb36763aa052e6bdbf2c59
5237 +DATA test/image.tar.zst.sig 833 BLAKE2B 214724ae4ff9198879c8c960fd8167632e27982c2278bb873f195abe75b75afa1ebed4c37ec696f5f5bc35c3a1184b60e0b50d56695b072b254f730db01eddb5 SHA512 67316187da8bb6b7a5f9dc6a42ed5c7d72c6184483a97f23c0bebd8b187ac9268e0409eb233c935101606768718c99eaa5699037d6a68c2d88c9ed5331a3f73c
5238 +-----BEGIN PGP SIGNATURE-----
5239 +
5240 +iNUEARYIAH0WIQSMe+CQzU+/D/DeMitA3PGOlxUHlQUCYVrQal8UgAAAAAAuAChp
5241 +c3N1ZXItZnByQG5vdGF0aW9ucy5vcGVucGdwLmZpZnRoaG9yc2VtYW4ubmV0OEM3
5242 +QkUwOTBDRDRGQkYwRkYwREUzMjJCNDBEQ0YxOEU5NzE1MDc5NQAKCRBA3PGOlxUH
5243 +lbmTAP4jdhMTW6g550/t0V7XcixqVtBockOTln8hZrZIQrjAJAD/caDkxgz5Xl8C
5244 +EP1pgSXXGtlUnv6akg/wueFJKEr9KQs=
5245 +=edEg
5246 +-----END PGP SIGNATURE-----
5247 +"""
5248 + data = io.BytesIO(sig)
5249 + f.size = len(sig)
5250 + tar_2.addfile(f, data)
5251 + data.close()
5252 + else:
5253 + tar_2.addfile(f, tar_1.extractfile(f))
5254 +
5255 + binpkg_2 = gpkg(settings, "test", os.path.join(tmpdir, "test-2.gpkg.tar"))
5256 + self.assertRaises(
5257 + InvalidSignature, binpkg_2.decompress, os.path.join(tmpdir, "test")
5258 + )
5259 +
5260 + finally:
5261 + shutil.rmtree(tmpdir)
5262 + playground.cleanup()
5263
5264 diff --git a/lib/portage/tests/gpkg/test_gpkg_metadata_update.py b/lib/portage/tests/gpkg/test_gpkg_metadata_update.py
5265 new file mode 100644
5266 index 000000000..2d5d35a98
5267 --- /dev/null
5268 +++ b/lib/portage/tests/gpkg/test_gpkg_metadata_update.py
5269 @@ -0,0 +1,59 @@
5270 +# Copright Gentoo Foundation 2006-2020
5271 +# Portage Unit Testing Functionality
5272 +
5273 +import tempfile
5274 +import sys
5275 +from os import urandom
5276 +
5277 +from portage import os
5278 +from portage import shutil
5279 +from portage.util._compare_files import compare_files
5280 +from portage.tests import TestCase
5281 +from portage.tests.resolver.ResolverPlayground import ResolverPlayground
5282 +from portage.gpkg import gpkg
5283 +
5284 +
5285 +class test_gpkg_metadata_case(TestCase):
5286 + def test_gpkg_update_metadata(self):
5287 + if sys.version_info.major < 3:
5288 + self.skipTest("Not support Python 2")
5289 + playground = ResolverPlayground(
5290 + user_config={
5291 + "make.conf": ('BINPKG_COMPRESS="gzip"',),
5292 + }
5293 + )
5294 + tmpdir = tempfile.mkdtemp()
5295 +
5296 + try:
5297 + settings = playground.settings
5298 + orig_full_path = os.path.join(tmpdir, "orig/")
5299 + os.makedirs(orig_full_path)
5300 + with open(os.path.join(orig_full_path, "test"), "wb") as test_file:
5301 + test_file.write(urandom(1048576))
5302 +
5303 + gpkg_file_loc = os.path.join(tmpdir, "test.gpkg.tar")
5304 + test_gpkg = gpkg(settings, "test", gpkg_file_loc)
5305 +
5306 + meta = {"test1": b"1234567890", "test2": b"abcdef"}
5307 +
5308 + test_gpkg.compress(os.path.join(tmpdir, "orig"), meta)
5309 +
5310 + meta_result = test_gpkg.get_metadata()
5311 + self.assertEqual(meta, meta_result)
5312 +
5313 + meta_new = {"test3": b"0987654321", "test4": b"XXXXXXXX"}
5314 + test_gpkg.update_metadata(meta_new)
5315 +
5316 + meta_result = test_gpkg.get_metadata()
5317 + self.assertEqual(meta_new, meta_result)
5318 +
5319 + test_gpkg.decompress(os.path.join(tmpdir, "test"))
5320 + r = compare_files(
5321 + os.path.join(tmpdir, "orig/" + "test"),
5322 + os.path.join(tmpdir, "test/" + "test"),
5323 + skipped_types=("atime", "mtime", "ctime"),
5324 + )
5325 + self.assertEqual(r, ())
5326 + finally:
5327 + shutil.rmtree(tmpdir)
5328 + playground.cleanup()
5329
5330 diff --git a/lib/portage/tests/gpkg/test_gpkg_metadata_url.py b/lib/portage/tests/gpkg/test_gpkg_metadata_url.py
5331 new file mode 100644
5332 index 000000000..08591715b
5333 --- /dev/null
5334 +++ b/lib/portage/tests/gpkg/test_gpkg_metadata_url.py
5335 @@ -0,0 +1,173 @@
5336 +# Copright Gentoo Foundation 2006-2020
5337 +# Portage Unit Testing Functionality
5338 +
5339 +import io
5340 +import random
5341 +import sys
5342 +import tarfile
5343 +import tempfile
5344 +from functools import partial
5345 +from os import urandom
5346 +
5347 +from portage.gpkg import gpkg
5348 +from portage import os
5349 +from portage import shutil
5350 +from portage.tests import TestCase
5351 +from portage.tests.resolver.ResolverPlayground import ResolverPlayground
5352 +from portage.exception import InvalidSignature
5353 +from portage.gpg import GPG
5354 +
5355 +
5356 +class test_gpkg_metadata_url_case(TestCase):
5357 + def httpd(self, directory, port):
5358 + try:
5359 + import http.server
5360 + import socketserver
5361 + except ImportError:
5362 + self.skipTest("http server not exits")
5363 +
5364 + Handler = partial(http.server.SimpleHTTPRequestHandler, directory=directory)
5365 +
5366 + with socketserver.TCPServer(("127.0.0.1", port), Handler) as httpd:
5367 + httpd.serve_forever()
5368 +
5369 + def start_http_server(self, directory, port):
5370 + try:
5371 + import threading
5372 + except ImportError:
5373 + self.skipTest("threading module not exists")
5374 +
5375 + server = threading.Thread(
5376 + target=self.httpd, args=(directory, port), daemon=True
5377 + )
5378 + server.start()
5379 + return server
5380 +
5381 + def test_gpkg_get_metadata_url(self):
5382 + if sys.version_info.major < 3:
5383 + self.skipTest("Not support Python 2")
5384 +
5385 + if sys.version_info.major == 3 and sys.version_info.minor <= 6:
5386 + self.skipTest("http server not support change root dir")
5387 +
5388 + playground = ResolverPlayground(
5389 + user_config={
5390 + "make.conf": (
5391 + 'BINPKG_COMPRESS="gzip"',
5392 + 'FEATURES="${FEATURES} -binpkg-signing '
5393 + '-binpkg-request-signature"',
5394 + ),
5395 + }
5396 + )
5397 + tmpdir = tempfile.mkdtemp()
5398 + try:
5399 + settings = playground.settings
5400 + for _ in range(0, 5):
5401 + port = random.randint(30000, 60000)
5402 + try:
5403 + server = self.start_http_server(tmpdir, port)
5404 + except OSError:
5405 + continue
5406 + break
5407 +
5408 + orig_full_path = os.path.join(tmpdir, "orig/")
5409 + os.makedirs(orig_full_path)
5410 +
5411 + with open(os.path.join(orig_full_path, "test"), "wb") as test_file:
5412 + test_file.write(urandom(1048576))
5413 +
5414 + gpkg_file_loc = os.path.join(tmpdir, "test.gpkg.tar")
5415 + test_gpkg = gpkg(settings, "test", gpkg_file_loc)
5416 +
5417 + meta = {
5418 + "test1": b"{abcdefghijklmnopqrstuvwxyz, 1234567890}",
5419 + "test2": urandom(102400),
5420 + }
5421 +
5422 + test_gpkg.compress(os.path.join(tmpdir, "orig"), meta)
5423 +
5424 + meta_from_url = test_gpkg.get_metadata_url(
5425 + "http://127.0.0.1:" + str(port) + "/test.gpkg.tar"
5426 + )
5427 +
5428 + self.assertEqual(meta, meta_from_url)
5429 + finally:
5430 + shutil.rmtree(tmpdir)
5431 + playground.cleanup()
5432 +
5433 + def test_gpkg_get_metadata_url_unknown_signature(self):
5434 + if sys.version_info.major < 3:
5435 + self.skipTest("Not support Python 2")
5436 +
5437 + if sys.version_info.major == 3 and sys.version_info.minor <= 6:
5438 + self.skipTest("http server not support change root dir")
5439 +
5440 + playground = ResolverPlayground(
5441 + user_config={
5442 + "make.conf": (
5443 + 'BINPKG_COMPRESS="gzip"',
5444 + 'FEATURES="${FEATURES} binpkg-signing ' 'binpkg-request-signature"',
5445 + ),
5446 + }
5447 + )
5448 + tmpdir = tempfile.mkdtemp()
5449 + try:
5450 + settings = playground.settings
5451 + gpg = GPG(settings)
5452 + gpg.unlock()
5453 +
5454 + for _ in range(0, 5):
5455 + port = random.randint(30000, 60000)
5456 + try:
5457 + server = self.start_http_server(tmpdir, port)
5458 + except OSError:
5459 + continue
5460 + break
5461 +
5462 + orig_full_path = os.path.join(tmpdir, "orig/")
5463 + os.makedirs(orig_full_path)
5464 +
5465 + with open(os.path.join(orig_full_path, "test"), "wb") as test_file:
5466 + test_file.write(urandom(1048576))
5467 +
5468 + gpkg_file_loc = os.path.join(tmpdir, "test-1.gpkg.tar")
5469 + test_gpkg = gpkg(settings, "test", gpkg_file_loc)
5470 +
5471 + meta = {
5472 + "test1": b"{abcdefghijklmnopqrstuvwxyz, 1234567890}",
5473 + "test2": urandom(102400),
5474 + }
5475 +
5476 + test_gpkg.compress(os.path.join(tmpdir, "orig"), meta)
5477 +
5478 + with tarfile.open(os.path.join(tmpdir, "test-1.gpkg.tar"), "r") as tar_1:
5479 + with tarfile.open(
5480 + os.path.join(tmpdir, "test-2.gpkg.tar"), "w"
5481 + ) as tar_2:
5482 + for f in tar_1.getmembers():
5483 + if f.name == "test/metadata.tar.gz":
5484 + sig = b"""
5485 +-----BEGIN PGP SIGNATURE-----
5486 +
5487 +iHUEABYIAB0WIQRVhCbPGi/rhGTq4nV+k2dcK9uyIgUCXw4ehAAKCRB+k2dcK9uy
5488 +IkCfAP49AOYjzuQPP0n5P0SGCINnAVEXN7QLQ4PurY/lt7cT2gEAq01stXjFhrz5
5489 +87Koh+ND2r5XfQsz3XeBqbb/BpmbEgo=
5490 +=sc5K
5491 +-----END PGP SIGNATURE-----
5492 +"""
5493 + data = io.BytesIO(sig)
5494 + f.size = len(sig)
5495 + tar_2.addfile(f, data)
5496 + data.close()
5497 + else:
5498 + tar_2.addfile(f, tar_1.extractfile(f))
5499 +
5500 + test_gpkg = gpkg(settings, "test")
5501 + self.assertRaises(
5502 + InvalidSignature,
5503 + test_gpkg.get_metadata_url,
5504 + "http://127.0.0.1:" + str(port) + "/test-2.gpkg.tar",
5505 + )
5506 + finally:
5507 + shutil.rmtree(tmpdir)
5508 + playground.cleanup()
5509
5510 diff --git a/lib/portage/tests/gpkg/test_gpkg_path.py b/lib/portage/tests/gpkg/test_gpkg_path.py
5511 new file mode 100644
5512 index 000000000..828233dbd
5513 --- /dev/null
5514 +++ b/lib/portage/tests/gpkg/test_gpkg_path.py
5515 @@ -0,0 +1,390 @@
5516 +# -*- coding: utf-8 -*-
5517 +# Copright Gentoo Foundation 2006
5518 +# Portage Unit Testing Functionality
5519 +
5520 +import tempfile
5521 +import tarfile
5522 +import io
5523 +import sys
5524 +from os import urandom
5525 +
5526 +from portage import os
5527 +from portage import shutil
5528 +from portage.util._compare_files import compare_files
5529 +from portage.tests import TestCase
5530 +from portage.tests.resolver.ResolverPlayground import ResolverPlayground
5531 +from portage.gpkg import gpkg
5532 +
5533 +
5534 +class test_gpkg_path_case(TestCase):
5535 + def test_gpkg_short_path(self):
5536 + if sys.version_info.major < 3:
5537 + self.skipTest("Not support Python 2")
5538 +
5539 + playground = ResolverPlayground(
5540 + user_config={
5541 + "make.conf": ('BINPKG_COMPRESS="none"',),
5542 + }
5543 + )
5544 + tmpdir = tempfile.mkdtemp()
5545 +
5546 + try:
5547 + settings = playground.settings
5548 + path_name = (
5549 + "aaaabbbb/ccccdddd/eeeeffff/gggghhhh/iiiijjjj/kkkkllll/"
5550 + "mmmmnnnn/oooopppp/qqqqrrrr/sssstttt/"
5551 + )
5552 + orig_full_path = os.path.join(tmpdir, "orig/" + path_name)
5553 + os.makedirs(orig_full_path)
5554 + with open(os.path.join(orig_full_path, "test"), "wb") as test_file:
5555 + test_file.write(urandom(1048576))
5556 +
5557 + gpkg_file_loc = os.path.join(tmpdir, "test.gpkg.tar")
5558 + test_gpkg = gpkg(settings, "test", gpkg_file_loc)
5559 +
5560 + check_result = test_gpkg._check_pre_image_files(
5561 + os.path.join(tmpdir, "orig")
5562 + )
5563 + self.assertEqual(check_result, (95, 4, 0, 1048576, 1048576))
5564 +
5565 + test_gpkg.compress(os.path.join(tmpdir, "orig"), {"meta": "test"})
5566 + with open(gpkg_file_loc, "rb") as container:
5567 + # container
5568 + self.assertEqual(
5569 + test_gpkg._get_tar_format(container), tarfile.USTAR_FORMAT
5570 + )
5571 +
5572 + with tarfile.open(gpkg_file_loc, "r") as container:
5573 + metadata = io.BytesIO(container.extractfile("test/metadata.tar").read())
5574 + self.assertEqual(
5575 + test_gpkg._get_tar_format(metadata), tarfile.USTAR_FORMAT
5576 + )
5577 + metadata.close()
5578 +
5579 + image = io.BytesIO(container.extractfile("test/image.tar").read())
5580 + self.assertEqual(test_gpkg._get_tar_format(image), tarfile.USTAR_FORMAT)
5581 + image.close()
5582 +
5583 + test_gpkg.decompress(os.path.join(tmpdir, "test"))
5584 + r = compare_files(
5585 + os.path.join(tmpdir, "orig/" + path_name + "test"),
5586 + os.path.join(tmpdir, "test/" + path_name + "test"),
5587 + skipped_types=("atime", "mtime", "ctime"),
5588 + )
5589 + self.assertEqual(r, ())
5590 + finally:
5591 + shutil.rmtree(tmpdir)
5592 + playground.cleanup()
5593 +
5594 + def test_gpkg_long_path(self):
5595 + if sys.version_info.major < 3:
5596 + self.skipTest("Not support Python 2")
5597 +
5598 + playground = ResolverPlayground(
5599 + user_config={
5600 + "make.conf": ('BINPKG_COMPRESS="none"',),
5601 + }
5602 + )
5603 + tmpdir = tempfile.mkdtemp()
5604 +
5605 + try:
5606 + settings = playground.settings
5607 +
5608 + path_name = (
5609 + "aaaabbbb/ccccdddd/eeeeffff/gggghhhh/iiiijjjj/kkkkllll/"
5610 + "mmmmnnnn/oooopppp/qqqqrrrr/sssstttt/uuuuvvvv/wwwwxxxx/"
5611 + "yyyyzzzz/00001111/22223333/44445555/66667777/88889999/"
5612 + "aaaabbbb/ccccdddd/eeeeffff/gggghhhh/iiiijjjj/kkkkllll/"
5613 + "mmmmnnnn/oooopppp/qqqqrrrr/sssstttt/uuuuvvvv/wwwwxxxx/"
5614 + "yyyyzzzz/00001111/22223333/44445555/66667777/88889999/"
5615 + )
5616 + orig_full_path = os.path.join(tmpdir, "orig/" + path_name)
5617 + os.makedirs(orig_full_path)
5618 + with open(os.path.join(orig_full_path, "test"), "wb") as test_file:
5619 + test_file.write(urandom(1048576))
5620 +
5621 + gpkg_file_loc = os.path.join(tmpdir, "test.gpkg.tar")
5622 + test_gpkg = gpkg(settings, "test", gpkg_file_loc)
5623 +
5624 + check_result = test_gpkg._check_pre_image_files(
5625 + os.path.join(tmpdir, "orig")
5626 + )
5627 + self.assertEqual(check_result, (329, 4, 0, 1048576, 1048576))
5628 +
5629 + test_gpkg.compress(os.path.join(tmpdir, "orig"), {"meta": "test"})
5630 + with open(gpkg_file_loc, "rb") as container:
5631 + # container
5632 + self.assertEqual(
5633 + test_gpkg._get_tar_format(container), tarfile.USTAR_FORMAT
5634 + )
5635 +
5636 + with tarfile.open(gpkg_file_loc, "r") as container:
5637 + metadata = io.BytesIO(container.extractfile("test/metadata.tar").read())
5638 + self.assertEqual(
5639 + test_gpkg._get_tar_format(metadata), tarfile.USTAR_FORMAT
5640 + )
5641 + metadata.close()
5642 +
5643 + image = io.BytesIO(container.extractfile("test/image.tar").read())
5644 + self.assertEqual(test_gpkg._get_tar_format(image), tarfile.GNU_FORMAT)
5645 + image.close()
5646 +
5647 + test_gpkg.decompress(os.path.join(tmpdir, "test"))
5648 + r = compare_files(
5649 + os.path.join(tmpdir, "orig/" + path_name + "test"),
5650 + os.path.join(tmpdir, "test/" + path_name + "test"),
5651 + skipped_types=("atime", "mtime", "ctime"),
5652 + )
5653 + self.assertEqual(r, ())
5654 + finally:
5655 + shutil.rmtree(tmpdir)
5656 + playground.cleanup()
5657 +
5658 + def test_gpkg_non_ascii_path(self):
5659 + if sys.version_info.major < 3:
5660 + self.skipTest("Not support Python 2")
5661 +
5662 + playground = ResolverPlayground(
5663 + user_config={
5664 + "make.conf": ('BINPKG_COMPRESS="none"',),
5665 + }
5666 + )
5667 + tmpdir = tempfile.mkdtemp()
5668 +
5669 + try:
5670 + settings = playground.settings
5671 +
5672 + path_name = "中文测试/日本語テスト/한국어시험/"
5673 + orig_full_path = os.path.join(tmpdir, "orig/" + path_name)
5674 + os.makedirs(orig_full_path)
5675 + with open(os.path.join(orig_full_path, "test"), "wb") as test_file:
5676 + test_file.write(urandom(1048576))
5677 +
5678 + gpkg_file_loc = os.path.join(tmpdir, "test.gpkg.tar")
5679 + test_gpkg = gpkg(settings, "test", gpkg_file_loc)
5680 +
5681 + check_result = test_gpkg._check_pre_image_files(
5682 + os.path.join(tmpdir, "orig")
5683 + )
5684 + self.assertEqual(check_result, (53, 4, 0, 1048576, 1048576))
5685 +
5686 + test_gpkg.compress(os.path.join(tmpdir, "orig"), {"meta": "test"})
5687 + with open(gpkg_file_loc, "rb") as container:
5688 + # container
5689 + self.assertEqual(
5690 + test_gpkg._get_tar_format(container), tarfile.USTAR_FORMAT
5691 + )
5692 +
5693 + with tarfile.open(gpkg_file_loc, "r") as container:
5694 + metadata = io.BytesIO(container.extractfile("test/metadata.tar").read())
5695 + self.assertEqual(
5696 + test_gpkg._get_tar_format(metadata), tarfile.USTAR_FORMAT
5697 + )
5698 + metadata.close()
5699 +
5700 + image = io.BytesIO(container.extractfile("test/image.tar").read())
5701 + self.assertEqual(test_gpkg._get_tar_format(image), tarfile.USTAR_FORMAT)
5702 + image.close()
5703 +
5704 + test_gpkg.decompress(os.path.join(tmpdir, "test"))
5705 + r = compare_files(
5706 + os.path.join(tmpdir, "orig/" + path_name + "test"),
5707 + os.path.join(tmpdir, "test/" + path_name + "test"),
5708 + skipped_types=("atime", "mtime", "ctime"),
5709 + )
5710 + self.assertEqual(r, ())
5711 + finally:
5712 + shutil.rmtree(tmpdir)
5713 + playground.cleanup()
5714 +
5715 + def test_gpkg_symlink_path(self):
5716 + if sys.version_info.major < 3:
5717 + self.skipTest("Not support Python 2")
5718 +
5719 + playground = ResolverPlayground(
5720 + user_config={
5721 + "make.conf": ('BINPKG_COMPRESS="none"',),
5722 + }
5723 + )
5724 + tmpdir = tempfile.mkdtemp()
5725 +
5726 + try:
5727 + settings = playground.settings
5728 +
5729 + orig_full_path = os.path.join(tmpdir, "orig/")
5730 + os.makedirs(orig_full_path)
5731 + os.symlink(
5732 + "aaaabbbb/ccccdddd/eeeeffff/gggghhhh/iiiijjjj/kkkkllll/"
5733 + "mmmmnnnn/oooopppp/qqqqrrrr/sssstttt/uuuuvvvv/wwwwxxxx/"
5734 + "yyyyzzzz/00001111/22223333/44445555/66667777/88889999/test",
5735 + os.path.join(orig_full_path, "a_long_symlink"),
5736 + )
5737 +
5738 + gpkg_file_loc = os.path.join(tmpdir, "test.gpkg.tar")
5739 + test_gpkg = gpkg(settings, "test", gpkg_file_loc)
5740 +
5741 + check_result = test_gpkg._check_pre_image_files(
5742 + os.path.join(tmpdir, "orig")
5743 + )
5744 + self.assertEqual(check_result, (0, 14, 166, 0, 0))
5745 +
5746 + test_gpkg.compress(os.path.join(tmpdir, "orig"), {"meta": "test"})
5747 + with open(gpkg_file_loc, "rb") as container:
5748 + # container
5749 + self.assertEqual(
5750 + test_gpkg._get_tar_format(container), tarfile.USTAR_FORMAT
5751 + )
5752 +
5753 + with tarfile.open(gpkg_file_loc, "r") as container:
5754 + metadata = io.BytesIO(container.extractfile("test/metadata.tar").read())
5755 + self.assertEqual(
5756 + test_gpkg._get_tar_format(metadata), tarfile.USTAR_FORMAT
5757 + )
5758 + metadata.close()
5759 +
5760 + image = io.BytesIO(container.extractfile("test/image.tar").read())
5761 + self.assertEqual(test_gpkg._get_tar_format(image), tarfile.GNU_FORMAT)
5762 + image.close()
5763 +
5764 + test_gpkg.decompress(os.path.join(tmpdir, "test"))
5765 + r = compare_files(
5766 + os.path.join(tmpdir, "orig/", "a_long_symlink"),
5767 + os.path.join(tmpdir, "test/", "a_long_symlink"),
5768 + skipped_types=("atime", "mtime", "ctime"),
5769 + )
5770 + self.assertEqual(r, ())
5771 + finally:
5772 + shutil.rmtree(tmpdir)
5773 + playground.cleanup()
5774 +
5775 + def test_gpkg_long_hardlink_path(self):
5776 + if sys.version_info.major < 3:
5777 + self.skipTest("Not support Python 2")
5778 +
5779 + playground = ResolverPlayground(
5780 + user_config={
5781 + "make.conf": ('BINPKG_COMPRESS="none"',),
5782 + }
5783 + )
5784 + tmpdir = tempfile.mkdtemp()
5785 +
5786 + try:
5787 + settings = playground.settings
5788 +
5789 + path_name = (
5790 + "aaaabbbb/ccccdddd/eeeeffff/gggghhhh/iiiijjjj/kkkkllll/"
5791 + "mmmmnnnn/oooopppp/qqqqrrrr/sssstttt/uuuuvvvv/wwwwxxxx/"
5792 + )
5793 + file_name = (
5794 + "test-A-B-C-D-E-F-G-H-I-J-K-L-M-N-O-P-Q-R-S-T-U-V-W-X-Y-Z"
5795 + "A-B-C-D-E-F-G-H-I-J-K-L-M-N-O-P-Q-R-S-T-U-V-W-X-Y-Z"
5796 + "A-B-C-D-E-F-G-H-I-J-K-L-M-N-O-P-Q-R-S-T-U-V-W-X-Y-Z"
5797 + )
5798 + orig_full_path = os.path.join(tmpdir, "orig", path_name)
5799 + os.makedirs(orig_full_path)
5800 + with open(os.path.join(orig_full_path, "test"), "wb") as test_file:
5801 + test_file.write(urandom(1048576))
5802 +
5803 + os.link(
5804 + os.path.join(orig_full_path, "test"),
5805 + os.path.join(orig_full_path, file_name),
5806 + )
5807 +
5808 + gpkg_file_loc = os.path.join(tmpdir, "test.gpkg.tar")
5809 + test_gpkg = gpkg(settings, "test", gpkg_file_loc)
5810 +
5811 + check_result = test_gpkg._check_pre_image_files(
5812 + os.path.join(tmpdir, "orig")
5813 + )
5814 + self.assertEqual(check_result, (113, 158, 272, 1048576, 2097152))
5815 +
5816 + test_gpkg.compress(os.path.join(tmpdir, "orig"), {"meta": "test"})
5817 + with open(gpkg_file_loc, "rb") as container:
5818 + # container
5819 + self.assertEqual(
5820 + test_gpkg._get_tar_format(container), tarfile.USTAR_FORMAT
5821 + )
5822 +
5823 + with tarfile.open(gpkg_file_loc, "r") as container:
5824 + metadata = io.BytesIO(container.extractfile("test/metadata.tar").read())
5825 + self.assertEqual(
5826 + test_gpkg._get_tar_format(metadata), tarfile.USTAR_FORMAT
5827 + )
5828 + metadata.close()
5829 +
5830 + image = io.BytesIO(container.extractfile("test/image.tar").read())
5831 + self.assertEqual(test_gpkg._get_tar_format(image), tarfile.GNU_FORMAT)
5832 + image.close()
5833 +
5834 + test_gpkg.decompress(os.path.join(tmpdir, "test"))
5835 + r = compare_files(
5836 + os.path.join(tmpdir, "orig", path_name, file_name),
5837 + os.path.join(tmpdir, "test", path_name, file_name),
5838 + skipped_types=("atime", "mtime", "ctime"),
5839 + )
5840 + self.assertEqual(r, ())
5841 + finally:
5842 + shutil.rmtree(tmpdir)
5843 +
5844 + def test_gpkg_long_filename(self):
5845 + if sys.version_info.major < 3:
5846 + self.skipTest("Not support Python 2")
5847 +
5848 + playground = ResolverPlayground(
5849 + user_config={
5850 + "make.conf": ('BINPKG_COMPRESS="none"',),
5851 + }
5852 + )
5853 + tmpdir = tempfile.mkdtemp()
5854 +
5855 + try:
5856 + settings = playground.settings
5857 + path_name = "aaaabbbb/ccccdddd/eeeeffff/gggghhhh/iiiijjjj/kkkkllll/"
5858 + file_name = (
5859 + "test1234567890"
5860 + "A-B-C-D-E-F-G-H-I-J-K-L-M-N-O-P-Q-R-S-T-U-V-W-X-Y-Z"
5861 + "A-B-C-D-E-F-G-H-I-J-K-L-M-N-O-P-Q-R-S-T-U-V-W-X-Y-Z"
5862 + "A-B-C-D-E-F-G-H-I-J-K-L-M-N-O-P-Q-R-S-T-U-V-W-X-Y-Z"
5863 + )
5864 +
5865 + orig_full_path = os.path.join(tmpdir, "orig/" + path_name)
5866 + os.makedirs(orig_full_path)
5867 + with open(os.path.join(orig_full_path, file_name), "wb") as test_file:
5868 + test_file.write(urandom(1048576))
5869 +
5870 + gpkg_file_loc = os.path.join(tmpdir, "test.gpkg.tar")
5871 + test_gpkg = gpkg(settings, "test", gpkg_file_loc)
5872 +
5873 + check_result = test_gpkg._check_pre_image_files(
5874 + os.path.join(tmpdir, "orig")
5875 + )
5876 + self.assertEqual(check_result, (59, 167, 0, 1048576, 1048576))
5877 +
5878 + test_gpkg.compress(os.path.join(tmpdir, "orig"), {"meta": "test"})
5879 + with open(gpkg_file_loc, "rb") as container:
5880 + # container
5881 + self.assertEqual(
5882 + test_gpkg._get_tar_format(container), tarfile.USTAR_FORMAT
5883 + )
5884 +
5885 + with tarfile.open(gpkg_file_loc, "r") as container:
5886 + metadata = io.BytesIO(container.extractfile("test/metadata.tar").read())
5887 + self.assertEqual(
5888 + test_gpkg._get_tar_format(metadata), tarfile.USTAR_FORMAT
5889 + )
5890 + metadata.close()
5891 +
5892 + image = io.BytesIO(container.extractfile("test/image.tar").read())
5893 + self.assertEqual(test_gpkg._get_tar_format(image), tarfile.GNU_FORMAT)
5894 + image.close()
5895 +
5896 + test_gpkg.decompress(os.path.join(tmpdir, "test"))
5897 + r = compare_files(
5898 + os.path.join(tmpdir, "orig", path_name, file_name),
5899 + os.path.join(tmpdir, "test", path_name, file_name),
5900 + skipped_types=("atime", "mtime", "ctime"),
5901 + )
5902 + self.assertEqual(r, ())
5903 + finally:
5904 + shutil.rmtree(tmpdir)
5905 + playground.cleanup()
5906
5907 diff --git a/lib/portage/tests/gpkg/test_gpkg_size.py b/lib/portage/tests/gpkg/test_gpkg_size.py
5908 new file mode 100644
5909 index 000000000..a38621e70
5910 --- /dev/null
5911 +++ b/lib/portage/tests/gpkg/test_gpkg_size.py
5912 @@ -0,0 +1,58 @@
5913 +# Copright Gentoo Foundation 2006-2020
5914 +# Portage Unit Testing Functionality
5915 +
5916 +import tempfile
5917 +import tarfile
5918 +import sys
5919 +
5920 +from portage import os, shutil
5921 +from portage.tests import TestCase
5922 +from portage.tests.resolver.ResolverPlayground import ResolverPlayground
5923 +from portage.gpkg import gpkg
5924 +
5925 +
5926 +class test_gpkg_large_size_case(TestCase):
5927 + def test_gpkg_large_size(self):
5928 + if sys.version_info.major < 3:
5929 + self.skipTest("Not support Python 2")
5930 +
5931 + playground = ResolverPlayground(
5932 + user_config={
5933 + "make.conf": ('BINPKG_COMPRESS="gzip"',),
5934 + }
5935 + )
5936 + tmpdir = tempfile.mkdtemp()
5937 +
5938 + try:
5939 + settings = playground.settings
5940 +
5941 + orig_full_path = os.path.join(tmpdir, "orig/")
5942 + os.makedirs(orig_full_path)
5943 + # Check if filesystem support sparse file
5944 + with open(os.path.join(orig_full_path, "test"), "wb") as test_file:
5945 + test_file.truncate(1048576)
5946 +
5947 + if os.stat(os.path.join(orig_full_path, "test")).st_blocks != 0:
5948 + self.skipTest("Filesystem does not support sparse file")
5949 +
5950 + with open(os.path.join(orig_full_path, "test"), "wb") as test_file:
5951 + test_file.truncate(10737418240)
5952 +
5953 + gpkg_file_loc = os.path.join(tmpdir, "test.gpkg.tar")
5954 + test_gpkg = gpkg(settings, "test", gpkg_file_loc)
5955 +
5956 + check_result = test_gpkg._check_pre_image_files(
5957 + os.path.join(tmpdir, "orig")
5958 + )
5959 + self.assertEqual(check_result, (0, 4, 0, 10737418240, 10737418240))
5960 +
5961 + test_gpkg.compress(os.path.join(tmpdir, "orig"), {"meta": "test"})
5962 +
5963 + with open(gpkg_file_loc, "rb") as container:
5964 + # container
5965 + self.assertEqual(
5966 + test_gpkg._get_tar_format(container), tarfile.GNU_FORMAT
5967 + )
5968 + finally:
5969 + shutil.rmtree(tmpdir)
5970 + playground.cleanup()
5971
5972 diff --git a/lib/portage/tests/gpkg/test_gpkg_stream.py b/lib/portage/tests/gpkg/test_gpkg_stream.py
5973 new file mode 100644
5974 index 000000000..66471e01b
5975 --- /dev/null
5976 +++ b/lib/portage/tests/gpkg/test_gpkg_stream.py
5977 @@ -0,0 +1,112 @@
5978 +# Copright Gentoo Foundation 2006-2020
5979 +# Portage Unit Testing Functionality
5980 +
5981 +import sys
5982 +import tempfile
5983 +import io
5984 +import tarfile
5985 +from os import urandom
5986 +
5987 +import portage.gpkg
5988 +from portage import os
5989 +from portage import shutil
5990 +from portage.tests import TestCase
5991 +from portage.exception import CompressorOperationFailed
5992 +
5993 +
5994 +class test_gpkg_stream_case(TestCase):
5995 + def test_gpkg_stream_reader(self):
5996 + if sys.version_info.major < 3:
5997 + self.skipTest("Not support Python 2")
5998 +
5999 + data = urandom(1048576)
6000 + data_io = io.BytesIO(data)
6001 + data_io.seek(0)
6002 + with portage.gpkg.tar_stream_reader(data_io, ["cat"]) as test_reader:
6003 + data2 = test_reader.read()
6004 + data_io.close()
6005 + self.assertEqual(data, data2)
6006 +
6007 + def test_gpkg_stream_reader_without_cmd(self):
6008 + if sys.version_info.major < 3:
6009 + self.skipTest("Not support Python 2")
6010 +
6011 + data = urandom(1048576)
6012 + data_io = io.BytesIO(data)
6013 + data_io.seek(0)
6014 + with portage.gpkg.tar_stream_reader(data_io) as test_reader:
6015 + data2 = test_reader.read()
6016 + data_io.close()
6017 + self.assertEqual(data, data2)
6018 +
6019 + def test_gpkg_stream_reader_kill(self):
6020 + if sys.version_info.major < 3:
6021 + self.skipTest("Not support Python 2")
6022 +
6023 + data = urandom(1048576)
6024 + data_io = io.BytesIO(data)
6025 + data_io.seek(0)
6026 + with portage.gpkg.tar_stream_reader(data_io, ["cat"]) as test_reader:
6027 + try:
6028 + test_reader.kill()
6029 + except CompressorOperationFailed:
6030 + pass
6031 + data_io.close()
6032 + self.assertNotEqual(test_reader.proc.poll(), None)
6033 +
6034 + def test_gpkg_stream_reader_kill_without_cmd(self):
6035 + if sys.version_info.major < 3:
6036 + self.skipTest("Not support Python 2")
6037 +
6038 + data = urandom(1048576)
6039 + data_io = io.BytesIO(data)
6040 + data_io.seek(0)
6041 + with portage.gpkg.tar_stream_reader(data_io) as test_reader:
6042 + test_reader.kill()
6043 + data_io.close()
6044 + self.assertEqual(test_reader.proc, None)
6045 +
6046 + def test_gpkg_stream_writer(self):
6047 + if sys.version_info.major < 3:
6048 + self.skipTest("Not support Python 2")
6049 +
6050 + tmpdir = tempfile.mkdtemp()
6051 + try:
6052 + gpkg_file_loc = os.path.join(tmpdir, "test.gpkg.tar")
6053 + data = urandom(1048576)
6054 + with tarfile.open(gpkg_file_loc, "w") as test_tar:
6055 + test_tarinfo = tarfile.TarInfo("test")
6056 + with portage.gpkg.tar_stream_writer(
6057 + test_tarinfo, test_tar, tarfile.USTAR_FORMAT, ["cat"]
6058 + ) as test_writer:
6059 + test_writer.write(data)
6060 +
6061 + with tarfile.open(gpkg_file_loc, "r") as test_tar:
6062 + test_tarinfo = test_tar.getmember("test")
6063 + data2 = test_tar.extractfile(test_tarinfo).read()
6064 + self.assertEqual(data, data2)
6065 + finally:
6066 + shutil.rmtree(tmpdir)
6067 +
6068 + def test_gpkg_stream_writer_without_cmd(self):
6069 + if sys.version_info.major < 3:
6070 + self.skipTest("Not support Python 2")
6071 +
6072 + tmpdir = tempfile.mkdtemp()
6073 +
6074 + try:
6075 + gpkg_file_loc = os.path.join(tmpdir, "test.gpkg.tar")
6076 + data = urandom(1048576)
6077 + with tarfile.open(gpkg_file_loc, "w") as test_tar:
6078 + test_tarinfo = tarfile.TarInfo("test")
6079 + with portage.gpkg.tar_stream_writer(
6080 + test_tarinfo, test_tar, tarfile.USTAR_FORMAT
6081 + ) as test_writer:
6082 + test_writer.write(data)
6083 +
6084 + with tarfile.open(gpkg_file_loc, "r") as test_tar:
6085 + test_tarinfo = test_tar.getmember("test")
6086 + data2 = test_tar.extractfile(test_tarinfo).read()
6087 + self.assertEqual(data, data2)
6088 + finally:
6089 + shutil.rmtree(tmpdir)
6090
6091 diff --git a/lib/portage/tests/resolver/ResolverPlayground.py b/lib/portage/tests/resolver/ResolverPlayground.py
6092 index fdd0714e6..fa8b0cc76 100644
6093 --- a/lib/portage/tests/resolver/ResolverPlayground.py
6094 +++ b/lib/portage/tests/resolver/ResolverPlayground.py
6095 @@ -13,6 +13,7 @@ from portage.const import (
6096 GLOBAL_CONFIG_PATH,
6097 PORTAGE_BIN_PATH,
6098 USER_CONFIG_PATH,
6099 + SUPPORTED_GENTOO_BINPKG_FORMATS,
6100 )
6101 from portage.process import find_binary
6102 from portage.dep import Atom, _repo_separator
6103 @@ -23,6 +24,8 @@ from portage._sets.base import InternalPackageSet
6104 from portage.tests import cnf_path
6105 from portage.util import ensure_dirs, normalize_path
6106 from portage.versions import catsplit
6107 +from portage.exception import InvalidBinaryPackageFormat
6108 +from portage.gpg import GPG
6109
6110 import _emerge
6111 from _emerge.actions import _calc_depclean
6112 @@ -161,6 +164,7 @@ class ResolverPlayground:
6113 "egrep",
6114 "env",
6115 "find",
6116 + "flock",
6117 "grep",
6118 "head",
6119 "install",
6120 @@ -225,7 +229,6 @@ class ResolverPlayground:
6121
6122 self._create_distfiles(distfiles)
6123 self._create_ebuilds(ebuilds)
6124 - self._create_binpkgs(binpkgs)
6125 self._create_installed(installed)
6126 self._create_profile(
6127 ebuilds, eclasses, installed, profile, repo_configs, user_config, sets
6128 @@ -234,6 +237,8 @@ class ResolverPlayground:
6129
6130 self.settings, self.trees = self._load_config()
6131
6132 + self.gpg = None
6133 + self._create_binpkgs(binpkgs)
6134 self._create_ebuild_manifests(ebuilds)
6135
6136 portage.util.noiselimit = 0
6137 @@ -341,6 +346,13 @@ class ResolverPlayground:
6138 # a dict.
6139 items = getattr(binpkgs, "items", None)
6140 items = items() if items is not None else binpkgs
6141 + binpkg_format = self.settings.get(
6142 + "BINPKG_FORMAT", SUPPORTED_GENTOO_BINPKG_FORMATS[0]
6143 + )
6144 + if binpkg_format == "gpkg":
6145 + if self.gpg is None:
6146 + self.gpg = GPG(self.settings)
6147 + self.gpg.unlock()
6148 for cpv, metadata in items:
6149 a = Atom("=" + cpv, allow_repo=True)
6150 repo = a.repo
6151 @@ -356,19 +368,38 @@ class ResolverPlayground:
6152 metadata["repository"] = repo
6153 metadata["CATEGORY"] = cat
6154 metadata["PF"] = pf
6155 + metadata["BINPKG_FORMAT"] = binpkg_format
6156
6157 repo_dir = self.pkgdir
6158 category_dir = os.path.join(repo_dir, cat)
6159 if "BUILD_ID" in metadata:
6160 - binpkg_path = os.path.join(
6161 - category_dir, pn, "%s-%s.xpak" % (pf, metadata["BUILD_ID"])
6162 - )
6163 + if binpkg_format == "xpak":
6164 + binpkg_path = os.path.join(
6165 + category_dir, pn, "%s-%s.xpak" % (pf, metadata["BUILD_ID"])
6166 + )
6167 + elif binpkg_format == "gpkg":
6168 + binpkg_path = os.path.join(
6169 + category_dir, pn, "%s-%s.gpkg.tar" % (pf, metadata["BUILD_ID"])
6170 + )
6171 + else:
6172 + raise InvalidBinaryPackageFormat(binpkg_format)
6173 else:
6174 - binpkg_path = os.path.join(category_dir, pf + ".tbz2")
6175 + if binpkg_format == "xpak":
6176 + binpkg_path = os.path.join(category_dir, pf + ".tbz2")
6177 + elif binpkg_format == "gpkg":
6178 + binpkg_path = os.path.join(category_dir, pf + ".gpkg.tar")
6179 + else:
6180 + raise InvalidBinaryPackageFormat(binpkg_format)
6181
6182 ensure_dirs(os.path.dirname(binpkg_path))
6183 - t = portage.xpak.tbz2(binpkg_path)
6184 - t.recompose_mem(portage.xpak.xpak_mem(metadata))
6185 + if binpkg_format == "xpak":
6186 + t = portage.xpak.tbz2(binpkg_path)
6187 + t.recompose_mem(portage.xpak.xpak_mem(metadata))
6188 + elif binpkg_format == "gpkg":
6189 + t = portage.gpkg.gpkg(self.settings, a.cpv, binpkg_path)
6190 + t.compress(os.path.dirname(binpkg_path), metadata)
6191 + else:
6192 + raise InvalidBinaryPackageFormat(binpkg_format)
6193
6194 def _create_installed(self, installed):
6195 for cpv in installed:
6196 @@ -545,11 +576,19 @@ class ResolverPlayground:
6197 sub_profile_dir, os.path.join(user_config_dir, "make.profile")
6198 )
6199
6200 + gpg_test_path = os.environ["PORTAGE_GNUPGHOME"]
6201 +
6202 make_conf = {
6203 "ACCEPT_KEYWORDS": "x86",
6204 + "BINPKG_GPG_SIGNING_BASE_COMMAND": f"flock {gpg_test_path}/portage-binpkg-gpg.lock /usr/bin/gpg --sign --armor --yes --pinentry-mode loopback --passphrase GentooTest [PORTAGE_CONFIG]",
6205 + "BINPKG_GPG_SIGNING_GPG_HOME": gpg_test_path,
6206 + "BINPKG_GPG_SIGNING_KEY": "0x5D90EA06352177F6",
6207 + "BINPKG_GPG_VERIFY_GPG_HOME": gpg_test_path,
6208 "CLEAN_DELAY": "0",
6209 "DISTDIR": self.distdir,
6210 "EMERGE_WARNING_DELAY": "0",
6211 + "FEATURES": "${FEATURES} binpkg-signing binpkg-request-signature "
6212 + "gpg-keepalive",
6213 "PKGDIR": self.pkgdir,
6214 "PORTAGE_INST_GID": str(portage.data.portage_gid),
6215 "PORTAGE_INST_UID": str(portage.data.portage_uid),
6216 @@ -742,6 +781,8 @@ class ResolverPlayground:
6217 return
6218
6219 def cleanup(self):
6220 + if self.gpg is not None:
6221 + self.gpg.stop()
6222 for eroot in self.trees:
6223 portdb = self.trees[eroot]["porttree"].dbapi
6224 portdb.close_caches()
6225
6226 diff --git a/lib/portage/tests/resolver/binpkg_multi_instance/test_build_id_profile_format.py b/lib/portage/tests/resolver/binpkg_multi_instance/test_build_id_profile_format.py
6227 index b311961d6..4dc83e843 100644
6228 --- a/lib/portage/tests/resolver/binpkg_multi_instance/test_build_id_profile_format.py
6229 +++ b/lib/portage/tests/resolver/binpkg_multi_instance/test_build_id_profile_format.py
6230 @@ -1,11 +1,16 @@
6231 # Copyright 2015-2021 Gentoo Authors
6232 # Distributed under the terms of the GNU General Public License v2
6233
6234 +from __future__ import print_function
6235 +import sys
6236 +
6237 +from portage.const import SUPPORTED_GENTOO_BINPKG_FORMATS
6238 from portage.tests import TestCase
6239 from portage.tests.resolver.ResolverPlayground import (
6240 ResolverPlayground,
6241 ResolverPlaygroundTestCase,
6242 )
6243 +from portage.output import colorize
6244
6245
6246 class BuildIdProfileFormatTestCase(TestCase):
6247 @@ -139,21 +144,30 @@ class BuildIdProfileFormatTestCase(TestCase):
6248 ),
6249 )
6250
6251 - playground = ResolverPlayground(
6252 - debug=False,
6253 - binpkgs=binpkgs,
6254 - ebuilds=ebuilds,
6255 - installed=installed,
6256 - repo_configs=repo_configs,
6257 - profile=profile,
6258 - user_config=user_config,
6259 - world=world,
6260 - )
6261 - try:
6262 - for test_case in test_cases:
6263 - playground.run_TestCase(test_case)
6264 - self.assertEqual(test_case.test_success, True, test_case.fail_msg)
6265 - finally:
6266 - # Disable debug so that cleanup works.
6267 - # playground.debug = False
6268 - playground.cleanup()
6269 + for binpkg_format in SUPPORTED_GENTOO_BINPKG_FORMATS:
6270 + with self.subTest(binpkg_format=binpkg_format):
6271 + print(colorize("HILITE", binpkg_format), end=" ... ")
6272 + sys.stdout.flush()
6273 + _user_config = user_config.copy()
6274 + _user_config["make.conf"] += ('BINPKG_FORMAT="%s"' % binpkg_format,)
6275 + playground = ResolverPlayground(
6276 + debug=False,
6277 + binpkgs=binpkgs,
6278 + ebuilds=ebuilds,
6279 + installed=installed,
6280 + repo_configs=repo_configs,
6281 + profile=profile,
6282 + user_config=_user_config,
6283 + world=world,
6284 + )
6285 +
6286 + try:
6287 + for test_case in test_cases:
6288 + playground.run_TestCase(test_case)
6289 + self.assertEqual(
6290 + test_case.test_success, True, test_case.fail_msg
6291 + )
6292 + finally:
6293 + # Disable debug so that cleanup works.
6294 + # playground.debug = False
6295 + playground.cleanup()
6296
6297 diff --git a/lib/portage/tests/resolver/binpkg_multi_instance/test_rebuilt_binaries.py b/lib/portage/tests/resolver/binpkg_multi_instance/test_rebuilt_binaries.py
6298 index c854604c1..854dee458 100644
6299 --- a/lib/portage/tests/resolver/binpkg_multi_instance/test_rebuilt_binaries.py
6300 +++ b/lib/portage/tests/resolver/binpkg_multi_instance/test_rebuilt_binaries.py
6301 @@ -1,11 +1,16 @@
6302 # Copyright 2015 Gentoo Foundation
6303 # Distributed under the terms of the GNU General Public License v2
6304
6305 +from __future__ import print_function
6306 +import sys
6307 +
6308 +from portage.const import SUPPORTED_GENTOO_BINPKG_FORMATS
6309 from portage.tests import TestCase
6310 from portage.tests.resolver.ResolverPlayground import (
6311 ResolverPlayground,
6312 ResolverPlaygroundTestCase,
6313 )
6314 +from portage.output import colorize
6315
6316
6317 class RebuiltBinariesCase(TestCase):
6318 @@ -99,18 +104,27 @@ class RebuiltBinariesCase(TestCase):
6319 ),
6320 )
6321
6322 - playground = ResolverPlayground(
6323 - debug=False,
6324 - binpkgs=binpkgs,
6325 - installed=installed,
6326 - user_config=user_config,
6327 - world=world,
6328 - )
6329 - try:
6330 - for test_case in test_cases:
6331 - playground.run_TestCase(test_case)
6332 - self.assertEqual(test_case.test_success, True, test_case.fail_msg)
6333 - finally:
6334 - # Disable debug so that cleanup works.
6335 - # playground.debug = False
6336 - playground.cleanup()
6337 + for binpkg_format in SUPPORTED_GENTOO_BINPKG_FORMATS:
6338 + with self.subTest(binpkg_format=binpkg_format):
6339 + print(colorize("HILITE", binpkg_format), end=" ... ")
6340 + sys.stdout.flush()
6341 + _user_config = user_config.copy()
6342 + _user_config["make.conf"] += ('BINPKG_FORMAT="%s"' % binpkg_format,)
6343 + playground = ResolverPlayground(
6344 + debug=False,
6345 + binpkgs=binpkgs,
6346 + installed=installed,
6347 + user_config=_user_config,
6348 + world=world,
6349 + )
6350 +
6351 + try:
6352 + for test_case in test_cases:
6353 + playground.run_TestCase(test_case)
6354 + self.assertEqual(
6355 + test_case.test_success, True, test_case.fail_msg
6356 + )
6357 + finally:
6358 + # Disable debug so that cleanup works.
6359 + # playground.debug = False
6360 + playground.cleanup()
6361
6362 diff --git a/lib/portage/tests/resolver/soname/test_autounmask.py b/lib/portage/tests/resolver/soname/test_autounmask.py
6363 index 3ee0be8d0..e324c9392 100644
6364 --- a/lib/portage/tests/resolver/soname/test_autounmask.py
6365 +++ b/lib/portage/tests/resolver/soname/test_autounmask.py
6366 @@ -1,11 +1,16 @@
6367 # Copyright 2015 Gentoo Foundation
6368 # Distributed under the terms of the GNU General Public License v2
6369
6370 +from __future__ import print_function
6371 +import sys
6372 +
6373 +from portage.const import SUPPORTED_GENTOO_BINPKG_FORMATS
6374 from portage.tests import TestCase
6375 from portage.tests.resolver.ResolverPlayground import (
6376 ResolverPlayground,
6377 ResolverPlaygroundTestCase,
6378 )
6379 +from portage.output import colorize
6380
6381
6382 class SonameAutoUnmaskTestCase(TestCase):
6383 @@ -85,13 +90,26 @@ class SonameAutoUnmaskTestCase(TestCase):
6384 ),
6385 )
6386
6387 - playground = ResolverPlayground(
6388 - binpkgs=binpkgs, installed=installed, world=world, debug=False
6389 - )
6390 - try:
6391 - for test_case in test_cases:
6392 - playground.run_TestCase(test_case)
6393 - self.assertEqual(test_case.test_success, True, test_case.fail_msg)
6394 - finally:
6395 - playground.debug = False
6396 - playground.cleanup()
6397 + for binpkg_format in SUPPORTED_GENTOO_BINPKG_FORMATS:
6398 + with self.subTest(binpkg_format=binpkg_format):
6399 + print(colorize("HILITE", binpkg_format), end=" ... ")
6400 + sys.stdout.flush()
6401 + playground = ResolverPlayground(
6402 + binpkgs=binpkgs,
6403 + installed=installed,
6404 + world=world,
6405 + debug=False,
6406 + user_config={
6407 + "make.conf": ('BINPKG_FORMAT="%s"' % binpkg_format,),
6408 + },
6409 + )
6410 +
6411 + try:
6412 + for test_case in test_cases:
6413 + playground.run_TestCase(test_case)
6414 + self.assertEqual(
6415 + test_case.test_success, True, test_case.fail_msg
6416 + )
6417 + finally:
6418 + playground.debug = False
6419 + playground.cleanup()
6420
6421 diff --git a/lib/portage/tests/resolver/soname/test_downgrade.py b/lib/portage/tests/resolver/soname/test_downgrade.py
6422 index b683745e0..c601b6381 100644
6423 --- a/lib/portage/tests/resolver/soname/test_downgrade.py
6424 +++ b/lib/portage/tests/resolver/soname/test_downgrade.py
6425 @@ -1,11 +1,16 @@
6426 # Copyright 2015 Gentoo Foundation
6427 # Distributed under the terms of the GNU General Public License v2
6428
6429 +from __future__ import print_function
6430 +import sys
6431 +
6432 +from portage.const import SUPPORTED_GENTOO_BINPKG_FORMATS
6433 from portage.tests import TestCase
6434 from portage.tests.resolver.ResolverPlayground import (
6435 ResolverPlayground,
6436 ResolverPlaygroundTestCase,
6437 )
6438 +from portage.output import colorize
6439
6440
6441 class SonameDowngradeTestCase(TestCase):
6442 @@ -125,22 +130,29 @@ class SonameDowngradeTestCase(TestCase):
6443 ),
6444 )
6445
6446 - playground = ResolverPlayground(
6447 - binpkgs=binpkgs,
6448 - ebuilds=ebuilds,
6449 - installed=installed,
6450 - user_config=user_config,
6451 - world=world,
6452 - debug=False,
6453 - )
6454 - try:
6455 - for test_case in test_cases:
6456 - playground.run_TestCase(test_case)
6457 - self.assertEqual(test_case.test_success, True, test_case.fail_msg)
6458 - finally:
6459 - # Disable debug so that cleanup works.
6460 - playground.debug = False
6461 - playground.cleanup()
6462 + for binpkg_format in SUPPORTED_GENTOO_BINPKG_FORMATS:
6463 + with self.subTest(binpkg_format=binpkg_format):
6464 + print(colorize("HILITE", binpkg_format), end=" ... ")
6465 + sys.stdout.flush()
6466 + user_config["make.conf"] = ('BINPKG_FORMAT="%s"' % binpkg_format,)
6467 + playground = ResolverPlayground(
6468 + binpkgs=binpkgs,
6469 + ebuilds=ebuilds,
6470 + installed=installed,
6471 + user_config=user_config,
6472 + world=world,
6473 + debug=False,
6474 + )
6475 + try:
6476 + for test_case in test_cases:
6477 + playground.run_TestCase(test_case)
6478 + self.assertEqual(
6479 + test_case.test_success, True, test_case.fail_msg
6480 + )
6481 + finally:
6482 + # Disable debug so that cleanup works.
6483 + playground.debug = False
6484 + playground.cleanup()
6485
6486 def testTwoSlots(self):
6487
6488 @@ -217,19 +229,27 @@ class SonameDowngradeTestCase(TestCase):
6489 ),
6490 )
6491
6492 - playground = ResolverPlayground(
6493 - ebuilds=ebuilds,
6494 - binpkgs=binpkgs,
6495 - installed=installed,
6496 - user_config=user_config,
6497 - world=world,
6498 - debug=False,
6499 - )
6500 - try:
6501 - for test_case in test_cases:
6502 - playground.run_TestCase(test_case)
6503 - self.assertEqual(test_case.test_success, True, test_case.fail_msg)
6504 - finally:
6505 - # Disable debug so that cleanup works.
6506 - playground.debug = False
6507 - playground.cleanup()
6508 + for binpkg_format in SUPPORTED_GENTOO_BINPKG_FORMATS:
6509 + with self.subTest(binpkg_format=binpkg_format):
6510 + print(colorize("HILITE", binpkg_format), end=" ... ")
6511 + sys.stdout.flush()
6512 + user_config["make.conf"] = ('BINPKG_FORMAT="%s"' % binpkg_format,)
6513 + playground = ResolverPlayground(
6514 + ebuilds=ebuilds,
6515 + binpkgs=binpkgs,
6516 + installed=installed,
6517 + user_config=user_config,
6518 + world=world,
6519 + debug=False,
6520 + )
6521 +
6522 + try:
6523 + for test_case in test_cases:
6524 + playground.run_TestCase(test_case)
6525 + self.assertEqual(
6526 + test_case.test_success, True, test_case.fail_msg
6527 + )
6528 + finally:
6529 + # Disable debug so that cleanup works.
6530 + playground.debug = False
6531 + playground.cleanup()
6532
6533 diff --git a/lib/portage/tests/resolver/soname/test_or_choices.py b/lib/portage/tests/resolver/soname/test_or_choices.py
6534 index c636726f3..dcdcf57e3 100644
6535 --- a/lib/portage/tests/resolver/soname/test_or_choices.py
6536 +++ b/lib/portage/tests/resolver/soname/test_or_choices.py
6537 @@ -1,11 +1,16 @@
6538 # Copyright 2015 Gentoo Foundation
6539 # Distributed under the terms of the GNU General Public License v2
6540
6541 +from __future__ import print_function
6542 +import sys
6543 +
6544 +from portage.const import SUPPORTED_GENTOO_BINPKG_FORMATS
6545 from portage.tests import TestCase
6546 from portage.tests.resolver.ResolverPlayground import (
6547 ResolverPlayground,
6548 ResolverPlaygroundTestCase,
6549 )
6550 +from portage.output import colorize
6551
6552
6553 class SonameOrChoicesTestCase(TestCase):
6554 @@ -83,14 +88,26 @@ class SonameOrChoicesTestCase(TestCase):
6555 ),
6556 )
6557
6558 - playground = ResolverPlayground(
6559 - debug=False, binpkgs=binpkgs, installed=installed, world=world
6560 - )
6561 - try:
6562 - for test_case in test_cases:
6563 - playground.run_TestCase(test_case)
6564 - self.assertEqual(test_case.test_success, True, test_case.fail_msg)
6565 - finally:
6566 - # Disable debug so that cleanup works.
6567 - playground.debug = False
6568 - playground.cleanup()
6569 + for binpkg_format in SUPPORTED_GENTOO_BINPKG_FORMATS:
6570 + with self.subTest(binpkg_format=binpkg_format):
6571 + print(colorize("HILITE", binpkg_format), end=" ... ")
6572 + sys.stdout.flush()
6573 + playground = ResolverPlayground(
6574 + debug=False,
6575 + binpkgs=binpkgs,
6576 + installed=installed,
6577 + world=world,
6578 + user_config={
6579 + "make.conf": ('BINPKG_FORMAT="%s"' % binpkg_format,),
6580 + },
6581 + )
6582 + try:
6583 + for test_case in test_cases:
6584 + playground.run_TestCase(test_case)
6585 + self.assertEqual(
6586 + test_case.test_success, True, test_case.fail_msg
6587 + )
6588 + finally:
6589 + # Disable debug so that cleanup works.
6590 + playground.debug = False
6591 + playground.cleanup()
6592
6593 diff --git a/lib/portage/tests/resolver/soname/test_reinstall.py b/lib/portage/tests/resolver/soname/test_reinstall.py
6594 index f4616f9dd..68c842af1 100644
6595 --- a/lib/portage/tests/resolver/soname/test_reinstall.py
6596 +++ b/lib/portage/tests/resolver/soname/test_reinstall.py
6597 @@ -1,11 +1,16 @@
6598 # Copyright 2015 Gentoo Foundation
6599 # Distributed under the terms of the GNU General Public License v2
6600
6601 +from __future__ import print_function
6602 +import sys
6603 +
6604 +from portage.const import SUPPORTED_GENTOO_BINPKG_FORMATS
6605 from portage.tests import TestCase
6606 from portage.tests.resolver.ResolverPlayground import (
6607 ResolverPlayground,
6608 ResolverPlaygroundTestCase,
6609 )
6610 +from portage.output import colorize
6611
6612
6613 class SonameReinstallTestCase(TestCase):
6614 @@ -72,14 +77,27 @@ class SonameReinstallTestCase(TestCase):
6615 ),
6616 )
6617
6618 - playground = ResolverPlayground(
6619 - debug=False, binpkgs=binpkgs, installed=installed, world=world
6620 - )
6621 - try:
6622 - for test_case in test_cases:
6623 - playground.run_TestCase(test_case)
6624 - self.assertEqual(test_case.test_success, True, test_case.fail_msg)
6625 - finally:
6626 - # Disable debug so that cleanup works.
6627 - playground.debug = False
6628 - playground.cleanup()
6629 + for binpkg_format in SUPPORTED_GENTOO_BINPKG_FORMATS:
6630 + with self.subTest(binpkg_format=binpkg_format):
6631 + print(colorize("HILITE", binpkg_format), end=" ... ")
6632 + sys.stdout.flush()
6633 + playground = ResolverPlayground(
6634 + debug=False,
6635 + binpkgs=binpkgs,
6636 + installed=installed,
6637 + world=world,
6638 + user_config={
6639 + "make.conf": ('BINPKG_FORMAT="%s"' % binpkg_format,),
6640 + },
6641 + )
6642 +
6643 + try:
6644 + for test_case in test_cases:
6645 + playground.run_TestCase(test_case)
6646 + self.assertEqual(
6647 + test_case.test_success, True, test_case.fail_msg
6648 + )
6649 + finally:
6650 + # Disable debug so that cleanup works.
6651 + playground.debug = False
6652 + playground.cleanup()
6653
6654 diff --git a/lib/portage/tests/resolver/soname/test_skip_update.py b/lib/portage/tests/resolver/soname/test_skip_update.py
6655 index 336bfac4f..a515a5252 100644
6656 --- a/lib/portage/tests/resolver/soname/test_skip_update.py
6657 +++ b/lib/portage/tests/resolver/soname/test_skip_update.py
6658 @@ -1,11 +1,16 @@
6659 # Copyright 2015 Gentoo Foundation
6660 # Distributed under the terms of the GNU General Public License v2
6661
6662 +from __future__ import print_function
6663 +import sys
6664 +
6665 +from portage.const import SUPPORTED_GENTOO_BINPKG_FORMATS
6666 from portage.tests import TestCase
6667 from portage.tests.resolver.ResolverPlayground import (
6668 ResolverPlayground,
6669 ResolverPlaygroundTestCase,
6670 )
6671 +from portage.output import colorize
6672
6673
6674 class SonameSkipUpdateTestCase(TestCase):
6675 @@ -71,14 +76,26 @@ class SonameSkipUpdateTestCase(TestCase):
6676 ),
6677 )
6678
6679 - playground = ResolverPlayground(
6680 - debug=False, binpkgs=binpkgs, installed=installed, world=world
6681 - )
6682 - try:
6683 - for test_case in test_cases:
6684 - playground.run_TestCase(test_case)
6685 - self.assertEqual(test_case.test_success, True, test_case.fail_msg)
6686 - finally:
6687 - # Disable debug so that cleanup works.
6688 - playground.debug = False
6689 - playground.cleanup()
6690 + for binpkg_format in SUPPORTED_GENTOO_BINPKG_FORMATS:
6691 + with self.subTest(binpkg_format=binpkg_format):
6692 + print(colorize("HILITE", binpkg_format), end=" ... ")
6693 + sys.stdout.flush()
6694 + playground = ResolverPlayground(
6695 + debug=False,
6696 + binpkgs=binpkgs,
6697 + installed=installed,
6698 + world=world,
6699 + user_config={
6700 + "make.conf": ('BINPKG_FORMAT="%s"' % binpkg_format,),
6701 + },
6702 + )
6703 + try:
6704 + for test_case in test_cases:
6705 + playground.run_TestCase(test_case)
6706 + self.assertEqual(
6707 + test_case.test_success, True, test_case.fail_msg
6708 + )
6709 + finally:
6710 + # Disable debug so that cleanup works.
6711 + playground.debug = False
6712 + playground.cleanup()
6713
6714 diff --git a/lib/portage/tests/resolver/soname/test_slot_conflict_reinstall.py b/lib/portage/tests/resolver/soname/test_slot_conflict_reinstall.py
6715 index 39430ae41..027cadc83 100644
6716 --- a/lib/portage/tests/resolver/soname/test_slot_conflict_reinstall.py
6717 +++ b/lib/portage/tests/resolver/soname/test_slot_conflict_reinstall.py
6718 @@ -1,11 +1,16 @@
6719 # Copyright 2015 Gentoo Foundation
6720 # Distributed under the terms of the GNU General Public License v2
6721
6722 +from __future__ import print_function
6723 +import sys
6724 +
6725 +from portage.const import SUPPORTED_GENTOO_BINPKG_FORMATS
6726 from portage.tests import TestCase
6727 from portage.tests.resolver.ResolverPlayground import (
6728 ResolverPlayground,
6729 ResolverPlaygroundTestCase,
6730 )
6731 +from portage.output import colorize
6732
6733
6734 class SonameSlotConflictReinstallTestCase(TestCase):
6735 @@ -80,16 +85,28 @@ class SonameSlotConflictReinstallTestCase(TestCase):
6736 ),
6737 )
6738
6739 - playground = ResolverPlayground(
6740 - binpkgs=binpkgs, installed=installed, world=world, debug=False
6741 - )
6742 - try:
6743 - for test_case in test_cases:
6744 - playground.run_TestCase(test_case)
6745 - self.assertEqual(test_case.test_success, True, test_case.fail_msg)
6746 - finally:
6747 - playground.debug = False
6748 - playground.cleanup()
6749 + for binpkg_format in SUPPORTED_GENTOO_BINPKG_FORMATS:
6750 + with self.subTest(binpkg_format=binpkg_format):
6751 + print(colorize("HILITE", binpkg_format), end=" ... ")
6752 + sys.stdout.flush()
6753 + playground = ResolverPlayground(
6754 + binpkgs=binpkgs,
6755 + installed=installed,
6756 + world=world,
6757 + debug=False,
6758 + user_config={
6759 + "make.conf": ('BINPKG_FORMAT="%s"' % binpkg_format,),
6760 + },
6761 + )
6762 + try:
6763 + for test_case in test_cases:
6764 + playground.run_TestCase(test_case)
6765 + self.assertEqual(
6766 + test_case.test_success, True, test_case.fail_msg
6767 + )
6768 + finally:
6769 + playground.debug = False
6770 + playground.cleanup()
6771
6772 def testSonameSlotConflictMassRebuild(self):
6773 """
6774 @@ -159,16 +176,29 @@ class SonameSlotConflictReinstallTestCase(TestCase):
6775
6776 world = []
6777
6778 - playground = ResolverPlayground(
6779 - binpkgs=binpkgs, installed=installed, world=world, debug=False
6780 - )
6781 - try:
6782 - for test_case in test_cases:
6783 - playground.run_TestCase(test_case)
6784 - self.assertEqual(test_case.test_success, True, test_case.fail_msg)
6785 - finally:
6786 - playground.debug = False
6787 - playground.cleanup()
6788 + for binpkg_format in SUPPORTED_GENTOO_BINPKG_FORMATS:
6789 + with self.subTest(binpkg_format=binpkg_format):
6790 + print(colorize("HILITE", binpkg_format), end=" ... ")
6791 + sys.stdout.flush()
6792 + playground = ResolverPlayground(
6793 + binpkgs=binpkgs,
6794 + installed=installed,
6795 + world=world,
6796 + debug=False,
6797 + user_config={
6798 + "make.conf": ('BINPKG_FORMAT="%s"' % binpkg_format,),
6799 + },
6800 + )
6801 +
6802 + try:
6803 + for test_case in test_cases:
6804 + playground.run_TestCase(test_case)
6805 + self.assertEqual(
6806 + test_case.test_success, True, test_case.fail_msg
6807 + )
6808 + finally:
6809 + playground.debug = False
6810 + playground.cleanup()
6811
6812 def testSonameSlotConflictForgottenChild(self):
6813 """
6814 @@ -242,16 +272,29 @@ class SonameSlotConflictReinstallTestCase(TestCase):
6815
6816 world = ["app-misc/A"]
6817
6818 - playground = ResolverPlayground(
6819 - binpkgs=binpkgs, installed=installed, world=world, debug=False
6820 - )
6821 - try:
6822 - for test_case in test_cases:
6823 - playground.run_TestCase(test_case)
6824 - self.assertEqual(test_case.test_success, True, test_case.fail_msg)
6825 - finally:
6826 - playground.debug = False
6827 - playground.cleanup()
6828 + for binpkg_format in SUPPORTED_GENTOO_BINPKG_FORMATS:
6829 + with self.subTest(binpkg_format=binpkg_format):
6830 + print(colorize("HILITE", binpkg_format), end=" ... ")
6831 + sys.stdout.flush()
6832 + playground = ResolverPlayground(
6833 + binpkgs=binpkgs,
6834 + installed=installed,
6835 + world=world,
6836 + debug=False,
6837 + user_config={
6838 + "make.conf": ('BINPKG_FORMAT="%s"' % binpkg_format,),
6839 + },
6840 + )
6841 +
6842 + try:
6843 + for test_case in test_cases:
6844 + playground.run_TestCase(test_case)
6845 + self.assertEqual(
6846 + test_case.test_success, True, test_case.fail_msg
6847 + )
6848 + finally:
6849 + playground.debug = False
6850 + playground.cleanup()
6851
6852 def testSonameSlotConflictMixedDependencies(self):
6853 """
6854 @@ -316,13 +359,25 @@ class SonameSlotConflictReinstallTestCase(TestCase):
6855
6856 world = []
6857
6858 - playground = ResolverPlayground(
6859 - binpkgs=binpkgs, installed=installed, world=world, debug=False
6860 - )
6861 - try:
6862 - for test_case in test_cases:
6863 - playground.run_TestCase(test_case)
6864 - self.assertEqual(test_case.test_success, True, test_case.fail_msg)
6865 - finally:
6866 - playground.debug = False
6867 - playground.cleanup()
6868 + for binpkg_format in SUPPORTED_GENTOO_BINPKG_FORMATS:
6869 + with self.subTest(binpkg_format=binpkg_format):
6870 + print(colorize("HILITE", binpkg_format), end=" ... ")
6871 + sys.stdout.flush()
6872 + playground = ResolverPlayground(
6873 + binpkgs=binpkgs,
6874 + installed=installed,
6875 + world=world,
6876 + debug=False,
6877 + user_config={
6878 + "make.conf": ('BINPKG_FORMAT="%s"' % binpkg_format,),
6879 + },
6880 + )
6881 + try:
6882 + for test_case in test_cases:
6883 + playground.run_TestCase(test_case)
6884 + self.assertEqual(
6885 + test_case.test_success, True, test_case.fail_msg
6886 + )
6887 + finally:
6888 + playground.debug = False
6889 + playground.cleanup()
6890
6891 diff --git a/lib/portage/tests/resolver/soname/test_slot_conflict_update.py b/lib/portage/tests/resolver/soname/test_slot_conflict_update.py
6892 index 0541a185e..dd763caef 100644
6893 --- a/lib/portage/tests/resolver/soname/test_slot_conflict_update.py
6894 +++ b/lib/portage/tests/resolver/soname/test_slot_conflict_update.py
6895 @@ -1,11 +1,16 @@
6896 # Copyright 2015 Gentoo Foundation
6897 # Distributed under the terms of the GNU General Public License v2
6898
6899 +from __future__ import print_function
6900 +import sys
6901 +
6902 +from portage.const import SUPPORTED_GENTOO_BINPKG_FORMATS
6903 from portage.tests import TestCase
6904 from portage.tests.resolver.ResolverPlayground import (
6905 ResolverPlayground,
6906 ResolverPlaygroundTestCase,
6907 )
6908 +from portage.output import colorize
6909
6910
6911 class SonameSlotConflictUpdateTestCase(TestCase):
6912 @@ -88,13 +93,26 @@ class SonameSlotConflictUpdateTestCase(TestCase):
6913 ),
6914 )
6915
6916 - playground = ResolverPlayground(
6917 - binpkgs=binpkgs, installed=installed, world=world, debug=False
6918 - )
6919 - try:
6920 - for test_case in test_cases:
6921 - playground.run_TestCase(test_case)
6922 - self.assertEqual(test_case.test_success, True, test_case.fail_msg)
6923 - finally:
6924 - playground.debug = False
6925 - playground.cleanup()
6926 + for binpkg_format in SUPPORTED_GENTOO_BINPKG_FORMATS:
6927 + with self.subTest(binpkg_format=binpkg_format):
6928 + print(colorize("HILITE", binpkg_format), end=" ... ")
6929 + sys.stdout.flush()
6930 + playground = ResolverPlayground(
6931 + binpkgs=binpkgs,
6932 + installed=installed,
6933 + world=world,
6934 + debug=False,
6935 + user_config={
6936 + "make.conf": ('BINPKG_FORMAT="%s"' % binpkg_format,),
6937 + },
6938 + )
6939 +
6940 + try:
6941 + for test_case in test_cases:
6942 + playground.run_TestCase(test_case)
6943 + self.assertEqual(
6944 + test_case.test_success, True, test_case.fail_msg
6945 + )
6946 + finally:
6947 + playground.debug = False
6948 + playground.cleanup()
6949
6950 diff --git a/lib/portage/tests/resolver/soname/test_soname_provided.py b/lib/portage/tests/resolver/soname/test_soname_provided.py
6951 index 3cd9f1423..6a9ee76ba 100644
6952 --- a/lib/portage/tests/resolver/soname/test_soname_provided.py
6953 +++ b/lib/portage/tests/resolver/soname/test_soname_provided.py
6954 @@ -1,11 +1,16 @@
6955 # Copyright 2015 Gentoo Foundation
6956 # Distributed under the terms of the GNU General Public License v2
6957
6958 +from __future__ import print_function
6959 +import sys
6960 +
6961 +from portage.const import SUPPORTED_GENTOO_BINPKG_FORMATS
6962 from portage.tests import TestCase
6963 from portage.tests.resolver.ResolverPlayground import (
6964 ResolverPlayground,
6965 ResolverPlaygroundTestCase,
6966 )
6967 +from portage.output import colorize
6968
6969
6970 class SonameProvidedTestCase(TestCase):
6971 @@ -62,18 +67,28 @@ class SonameProvidedTestCase(TestCase):
6972 ),
6973 )
6974
6975 - playground = ResolverPlayground(
6976 - binpkgs=binpkgs,
6977 - debug=False,
6978 - profile=profile,
6979 - installed=installed,
6980 - world=world,
6981 - )
6982 - try:
6983 - for test_case in test_cases:
6984 - playground.run_TestCase(test_case)
6985 - self.assertEqual(test_case.test_success, True, test_case.fail_msg)
6986 - finally:
6987 - # Disable debug so that cleanup works.
6988 - playground.debug = False
6989 - playground.cleanup()
6990 + for binpkg_format in SUPPORTED_GENTOO_BINPKG_FORMATS:
6991 + with self.subTest(binpkg_format=binpkg_format):
6992 + print(colorize("HILITE", binpkg_format), end=" ... ")
6993 + sys.stdout.flush()
6994 + playground = ResolverPlayground(
6995 + binpkgs=binpkgs,
6996 + debug=False,
6997 + profile=profile,
6998 + installed=installed,
6999 + world=world,
7000 + user_config={
7001 + "make.conf": ('BINPKG_FORMAT="%s"' % binpkg_format,),
7002 + },
7003 + )
7004 +
7005 + try:
7006 + for test_case in test_cases:
7007 + playground.run_TestCase(test_case)
7008 + self.assertEqual(
7009 + test_case.test_success, True, test_case.fail_msg
7010 + )
7011 + finally:
7012 + # Disable debug so that cleanup works.
7013 + playground.debug = False
7014 + playground.cleanup()
7015
7016 diff --git a/lib/portage/tests/resolver/soname/test_unsatisfiable.py b/lib/portage/tests/resolver/soname/test_unsatisfiable.py
7017 index a8d2e10db..75d50c10f 100644
7018 --- a/lib/portage/tests/resolver/soname/test_unsatisfiable.py
7019 +++ b/lib/portage/tests/resolver/soname/test_unsatisfiable.py
7020 @@ -1,11 +1,16 @@
7021 # Copyright 2015 Gentoo Foundation
7022 # Distributed under the terms of the GNU General Public License v2
7023
7024 +from __future__ import print_function
7025 +import sys
7026 +
7027 +from portage.const import SUPPORTED_GENTOO_BINPKG_FORMATS
7028 from portage.tests import TestCase
7029 from portage.tests.resolver.ResolverPlayground import (
7030 ResolverPlayground,
7031 ResolverPlaygroundTestCase,
7032 )
7033 +from portage.output import colorize
7034
7035
7036 class SonameUnsatisfiableTestCase(TestCase):
7037 @@ -57,14 +62,27 @@ class SonameUnsatisfiableTestCase(TestCase):
7038 ),
7039 )
7040
7041 - playground = ResolverPlayground(
7042 - binpkgs=binpkgs, debug=False, installed=installed, world=world
7043 - )
7044 - try:
7045 - for test_case in test_cases:
7046 - playground.run_TestCase(test_case)
7047 - self.assertEqual(test_case.test_success, True, test_case.fail_msg)
7048 - finally:
7049 - # Disable debug so that cleanup works.
7050 - playground.debug = False
7051 - playground.cleanup()
7052 + for binpkg_format in SUPPORTED_GENTOO_BINPKG_FORMATS:
7053 + with self.subTest(binpkg_format=binpkg_format):
7054 + print(colorize("HILITE", binpkg_format), end=" ... ")
7055 + sys.stdout.flush()
7056 + playground = ResolverPlayground(
7057 + binpkgs=binpkgs,
7058 + debug=False,
7059 + installed=installed,
7060 + world=world,
7061 + user_config={
7062 + "make.conf": ('BINPKG_FORMAT="%s"' % binpkg_format,),
7063 + },
7064 + )
7065 +
7066 + try:
7067 + for test_case in test_cases:
7068 + playground.run_TestCase(test_case)
7069 + self.assertEqual(
7070 + test_case.test_success, True, test_case.fail_msg
7071 + )
7072 + finally:
7073 + # Disable debug so that cleanup works.
7074 + playground.debug = False
7075 + playground.cleanup()
7076
7077 diff --git a/lib/portage/tests/resolver/soname/test_unsatisfied.py b/lib/portage/tests/resolver/soname/test_unsatisfied.py
7078 index 955d5d75b..2e0fe6e7f 100644
7079 --- a/lib/portage/tests/resolver/soname/test_unsatisfied.py
7080 +++ b/lib/portage/tests/resolver/soname/test_unsatisfied.py
7081 @@ -1,11 +1,16 @@
7082 # Copyright 2015 Gentoo Foundation
7083 # Distributed under the terms of the GNU General Public License v2
7084
7085 +from __future__ import print_function
7086 +import sys
7087 +
7088 +from portage.const import SUPPORTED_GENTOO_BINPKG_FORMATS
7089 from portage.tests import TestCase
7090 from portage.tests.resolver.ResolverPlayground import (
7091 ResolverPlayground,
7092 ResolverPlaygroundTestCase,
7093 )
7094 +from portage.output import colorize
7095
7096
7097 class SonameUnsatisfiedTestCase(TestCase):
7098 @@ -70,14 +75,27 @@ class SonameUnsatisfiedTestCase(TestCase):
7099 ),
7100 )
7101
7102 - playground = ResolverPlayground(
7103 - binpkgs=binpkgs, debug=False, installed=installed, world=world
7104 - )
7105 - try:
7106 - for test_case in test_cases:
7107 - playground.run_TestCase(test_case)
7108 - self.assertEqual(test_case.test_success, True, test_case.fail_msg)
7109 - finally:
7110 - # Disable debug so that cleanup works.
7111 - playground.debug = False
7112 - playground.cleanup()
7113 + for binpkg_format in SUPPORTED_GENTOO_BINPKG_FORMATS:
7114 + with self.subTest(binpkg_format=binpkg_format):
7115 + print(colorize("HILITE", binpkg_format), end=" ... ")
7116 + sys.stdout.flush()
7117 + playground = ResolverPlayground(
7118 + binpkgs=binpkgs,
7119 + debug=False,
7120 + installed=installed,
7121 + world=world,
7122 + user_config={
7123 + "make.conf": ('BINPKG_FORMAT="%s"' % binpkg_format,),
7124 + },
7125 + )
7126 +
7127 + try:
7128 + for test_case in test_cases:
7129 + playground.run_TestCase(test_case)
7130 + self.assertEqual(
7131 + test_case.test_success, True, test_case.fail_msg
7132 + )
7133 + finally:
7134 + # Disable debug so that cleanup works.
7135 + playground.debug = False
7136 + playground.cleanup()
7137
7138 diff --git a/lib/portage/tests/resolver/test_autounmask_binpkg_use.py b/lib/portage/tests/resolver/test_autounmask_binpkg_use.py
7139 index e2164f0b1..682732611 100644
7140 --- a/lib/portage/tests/resolver/test_autounmask_binpkg_use.py
7141 +++ b/lib/portage/tests/resolver/test_autounmask_binpkg_use.py
7142 @@ -1,11 +1,16 @@
7143 # Copyright 2017 Gentoo Foundation
7144 # Distributed under the terms of the GNU General Public License v2
7145
7146 +from __future__ import print_function
7147 +import sys
7148 +
7149 +from portage.const import SUPPORTED_GENTOO_BINPKG_FORMATS
7150 from portage.tests import TestCase
7151 from portage.tests.resolver.ResolverPlayground import (
7152 ResolverPlayground,
7153 ResolverPlaygroundTestCase,
7154 )
7155 +from portage.output import colorize
7156
7157
7158 class AutounmaskBinpkgUseTestCase(TestCase):
7159 @@ -55,13 +60,26 @@ class AutounmaskBinpkgUseTestCase(TestCase):
7160 ),
7161 )
7162
7163 - playground = ResolverPlayground(
7164 - ebuilds=ebuilds, binpkgs=binpkgs, installed=installed, debug=False
7165 - )
7166 - try:
7167 - for test_case in test_cases:
7168 - playground.run_TestCase(test_case)
7169 - self.assertEqual(test_case.test_success, True, test_case.fail_msg)
7170 - finally:
7171 - playground.debug = False
7172 - playground.cleanup()
7173 + for binpkg_format in SUPPORTED_GENTOO_BINPKG_FORMATS:
7174 + with self.subTest(binpkg_format=binpkg_format):
7175 + print(colorize("HILITE", binpkg_format), end=" ... ")
7176 + sys.stdout.flush()
7177 + playground = ResolverPlayground(
7178 + ebuilds=ebuilds,
7179 + binpkgs=binpkgs,
7180 + installed=installed,
7181 + debug=False,
7182 + user_config={
7183 + "make.conf": ('BINPKG_FORMAT="%s"' % binpkg_format,),
7184 + },
7185 + )
7186 +
7187 + try:
7188 + for test_case in test_cases:
7189 + playground.run_TestCase(test_case)
7190 + self.assertEqual(
7191 + test_case.test_success, True, test_case.fail_msg
7192 + )
7193 + finally:
7194 + playground.debug = False
7195 + playground.cleanup()
7196
7197 diff --git a/lib/portage/tests/resolver/test_bdeps.py b/lib/portage/tests/resolver/test_bdeps.py
7198 index ce1f8e0d5..a1b987eca 100644
7199 --- a/lib/portage/tests/resolver/test_bdeps.py
7200 +++ b/lib/portage/tests/resolver/test_bdeps.py
7201 @@ -1,11 +1,16 @@
7202 # Copyright 2017 Gentoo Foundation
7203 # Distributed under the terms of the GNU General Public License v2
7204
7205 +from __future__ import print_function
7206 +import sys
7207 +
7208 from portage.tests import TestCase
7209 from portage.tests.resolver.ResolverPlayground import (
7210 ResolverPlayground,
7211 ResolverPlaygroundTestCase,
7212 )
7213 +from portage.const import SUPPORTED_GENTOO_BINPKG_FORMATS
7214 +from portage.output import colorize
7215
7216
7217 class BdepsTestCase(TestCase):
7218 @@ -183,18 +188,27 @@ class BdepsTestCase(TestCase):
7219 ),
7220 )
7221
7222 - playground = ResolverPlayground(
7223 - debug=False,
7224 - ebuilds=ebuilds,
7225 - installed=installed,
7226 - binpkgs=binpkgs,
7227 - world=world,
7228 - )
7229 - try:
7230 - for test_case in test_cases:
7231 - playground.run_TestCase(test_case)
7232 - self.assertEqual(test_case.test_success, True, test_case.fail_msg)
7233 - finally:
7234 - # Disable debug so that cleanup works.
7235 - playground.debug = False
7236 - playground.cleanup()
7237 + for binpkg_format in SUPPORTED_GENTOO_BINPKG_FORMATS:
7238 + with self.subTest(binpkg_format=binpkg_format):
7239 + print(colorize("HILITE", binpkg_format), end=" ... ")
7240 + sys.stdout.flush()
7241 + playground = ResolverPlayground(
7242 + debug=False,
7243 + ebuilds=ebuilds,
7244 + installed=installed,
7245 + binpkgs=binpkgs,
7246 + world=world,
7247 + user_config={
7248 + "make.conf": ('BINPKG_FORMAT="%s"' % binpkg_format,),
7249 + },
7250 + )
7251 + try:
7252 + for test_case in test_cases:
7253 + playground.run_TestCase(test_case)
7254 + self.assertEqual(
7255 + test_case.test_success, True, test_case.fail_msg
7256 + )
7257 + finally:
7258 + # Disable debug so that cleanup works.
7259 + playground.debug = False
7260 + playground.cleanup()
7261
7262 diff --git a/lib/portage/tests/resolver/test_binary_pkg_ebuild_visibility.py b/lib/portage/tests/resolver/test_binary_pkg_ebuild_visibility.py
7263 index 9a6a5417a..10a292abd 100644
7264 --- a/lib/portage/tests/resolver/test_binary_pkg_ebuild_visibility.py
7265 +++ b/lib/portage/tests/resolver/test_binary_pkg_ebuild_visibility.py
7266 @@ -1,11 +1,16 @@
7267 # Copyright 2017 Gentoo Foundation
7268 # Distributed under the terms of the GNU General Public License v2
7269
7270 +from __future__ import print_function
7271 +import sys
7272 +
7273 +from portage.const import SUPPORTED_GENTOO_BINPKG_FORMATS
7274 from portage.tests import TestCase
7275 from portage.tests.resolver.ResolverPlayground import (
7276 ResolverPlayground,
7277 ResolverPlaygroundTestCase,
7278 )
7279 +from portage.output import colorize
7280
7281
7282 class BinaryPkgEbuildVisibilityTestCase(TestCase):
7283 @@ -124,12 +129,24 @@ class BinaryPkgEbuildVisibilityTestCase(TestCase):
7284 ),
7285 )
7286
7287 - playground = ResolverPlayground(
7288 - binpkgs=binpkgs, ebuilds=ebuilds, installed=installed, world=world
7289 - )
7290 - try:
7291 - for test_case in test_cases:
7292 - playground.run_TestCase(test_case)
7293 - self.assertEqual(test_case.test_success, True, test_case.fail_msg)
7294 - finally:
7295 - playground.cleanup()
7296 + for binpkg_format in SUPPORTED_GENTOO_BINPKG_FORMATS:
7297 + with self.subTest(binpkg_format=binpkg_format):
7298 + print(colorize("HILITE", binpkg_format), end=" ... ")
7299 + sys.stdout.flush()
7300 + playground = ResolverPlayground(
7301 + binpkgs=binpkgs,
7302 + ebuilds=ebuilds,
7303 + installed=installed,
7304 + world=world,
7305 + user_config={
7306 + "make.conf": ('BINPKG_FORMAT="%s"' % binpkg_format,),
7307 + },
7308 + )
7309 + try:
7310 + for test_case in test_cases:
7311 + playground.run_TestCase(test_case)
7312 + self.assertEqual(
7313 + test_case.test_success, True, test_case.fail_msg
7314 + )
7315 + finally:
7316 + playground.cleanup()
7317
7318 diff --git a/lib/portage/tests/resolver/test_changed_deps.py b/lib/portage/tests/resolver/test_changed_deps.py
7319 index c3a0e2f87..0c9b34d4b 100644
7320 --- a/lib/portage/tests/resolver/test_changed_deps.py
7321 +++ b/lib/portage/tests/resolver/test_changed_deps.py
7322 @@ -1,11 +1,16 @@
7323 # Copyright 2014 Gentoo Foundation
7324 # Distributed under the terms of the GNU General Public License v2
7325
7326 +from __future__ import print_function
7327 +import sys
7328 +
7329 +from portage.const import SUPPORTED_GENTOO_BINPKG_FORMATS
7330 from portage.tests import TestCase
7331 from portage.tests.resolver.ResolverPlayground import (
7332 ResolverPlayground,
7333 ResolverPlaygroundTestCase,
7334 )
7335 +from portage.output import colorize
7336
7337
7338 class ChangedDepsTestCase(TestCase):
7339 @@ -103,16 +108,26 @@ class ChangedDepsTestCase(TestCase):
7340 ),
7341 )
7342
7343 - playground = ResolverPlayground(
7344 - debug=False,
7345 - ebuilds=ebuilds,
7346 - binpkgs=binpkgs,
7347 - installed=installed,
7348 - world=world,
7349 - )
7350 - try:
7351 - for test_case in test_cases:
7352 - playground.run_TestCase(test_case)
7353 - self.assertEqual(test_case.test_success, True, test_case.fail_msg)
7354 - finally:
7355 - playground.cleanup()
7356 + for binpkg_format in SUPPORTED_GENTOO_BINPKG_FORMATS:
7357 + with self.subTest(binpkg_format=binpkg_format):
7358 + print(colorize("HILITE", binpkg_format), end=" ... ")
7359 + sys.stdout.flush()
7360 + playground = ResolverPlayground(
7361 + debug=False,
7362 + ebuilds=ebuilds,
7363 + binpkgs=binpkgs,
7364 + installed=installed,
7365 + world=world,
7366 + user_config={
7367 + "make.conf": ('BINPKG_FORMAT="%s"' % binpkg_format,),
7368 + },
7369 + )
7370 +
7371 + try:
7372 + for test_case in test_cases:
7373 + playground.run_TestCase(test_case)
7374 + self.assertEqual(
7375 + test_case.test_success, True, test_case.fail_msg
7376 + )
7377 + finally:
7378 + playground.cleanup()
7379
7380 diff --git a/lib/portage/tests/resolver/test_complete_if_new_subslot_without_revbump.py b/lib/portage/tests/resolver/test_complete_if_new_subslot_without_revbump.py
7381 index 3a5912606..6b061ca2a 100644
7382 --- a/lib/portage/tests/resolver/test_complete_if_new_subslot_without_revbump.py
7383 +++ b/lib/portage/tests/resolver/test_complete_if_new_subslot_without_revbump.py
7384 @@ -1,11 +1,16 @@
7385 # Copyright 2013 Gentoo Foundation
7386 # Distributed under the terms of the GNU General Public License v2
7387
7388 +from __future__ import print_function
7389 +import sys
7390 +
7391 +from portage.const import SUPPORTED_GENTOO_BINPKG_FORMATS
7392 from portage.tests import TestCase
7393 from portage.tests.resolver.ResolverPlayground import (
7394 ResolverPlayground,
7395 ResolverPlaygroundTestCase,
7396 )
7397 +from portage.output import colorize
7398
7399
7400 class CompeteIfNewSubSlotWithoutRevBumpTestCase(TestCase):
7401 @@ -55,16 +60,25 @@ class CompeteIfNewSubSlotWithoutRevBumpTestCase(TestCase):
7402 ),
7403 )
7404
7405 - playground = ResolverPlayground(
7406 - ebuilds=ebuilds,
7407 - binpkgs=binpkgs,
7408 - installed=installed,
7409 - world=world,
7410 - debug=False,
7411 - )
7412 - try:
7413 - for test_case in test_cases:
7414 - playground.run_TestCase(test_case)
7415 - self.assertEqual(test_case.test_success, True, test_case.fail_msg)
7416 - finally:
7417 - playground.cleanup()
7418 + for binpkg_format in SUPPORTED_GENTOO_BINPKG_FORMATS:
7419 + with self.subTest(binpkg_format=binpkg_format):
7420 + print(colorize("HILITE", binpkg_format), end=" ... ")
7421 + sys.stdout.flush()
7422 + playground = ResolverPlayground(
7423 + ebuilds=ebuilds,
7424 + binpkgs=binpkgs,
7425 + installed=installed,
7426 + world=world,
7427 + debug=False,
7428 + user_config={
7429 + "make.conf": ('BINPKG_FORMAT="%s"' % binpkg_format,),
7430 + },
7431 + )
7432 + try:
7433 + for test_case in test_cases:
7434 + playground.run_TestCase(test_case)
7435 + self.assertEqual(
7436 + test_case.test_success, True, test_case.fail_msg
7437 + )
7438 + finally:
7439 + playground.cleanup()
7440
7441 diff --git a/lib/portage/tests/resolver/test_disjunctive_depend_order.py b/lib/portage/tests/resolver/test_disjunctive_depend_order.py
7442 index e08a1d845..4471bc605 100644
7443 --- a/lib/portage/tests/resolver/test_disjunctive_depend_order.py
7444 +++ b/lib/portage/tests/resolver/test_disjunctive_depend_order.py
7445 @@ -1,11 +1,16 @@
7446 # Copyright 2017 Gentoo Foundation
7447 # Distributed under the terms of the GNU General Public License v2
7448
7449 +from __future__ import print_function
7450 +import sys
7451 +
7452 +from portage.const import SUPPORTED_GENTOO_BINPKG_FORMATS
7453 from portage.tests import TestCase
7454 from portage.tests.resolver.ResolverPlayground import (
7455 ResolverPlayground,
7456 ResolverPlaygroundTestCase,
7457 )
7458 +from portage.output import colorize
7459
7460
7461 class DisjunctiveDependOrderTestCase(TestCase):
7462 @@ -71,12 +76,25 @@ class DisjunctiveDependOrderTestCase(TestCase):
7463 ),
7464 )
7465
7466 - playground = ResolverPlayground(debug=False, binpkgs=binpkgs, ebuilds=ebuilds)
7467 + for binpkg_format in SUPPORTED_GENTOO_BINPKG_FORMATS:
7468 + with self.subTest(binpkg_format=binpkg_format):
7469 + print(colorize("HILITE", binpkg_format), end=" ... ")
7470 + sys.stdout.flush()
7471 + playground = ResolverPlayground(
7472 + debug=False,
7473 + binpkgs=binpkgs,
7474 + ebuilds=ebuilds,
7475 + user_config={
7476 + "make.conf": ('BINPKG_FORMAT="%s"' % binpkg_format,),
7477 + },
7478 + )
7479
7480 - try:
7481 - for test_case in test_cases:
7482 - playground.run_TestCase(test_case)
7483 - self.assertEqual(test_case.test_success, True, test_case.fail_msg)
7484 - finally:
7485 - playground.debug = False
7486 - playground.cleanup()
7487 + try:
7488 + for test_case in test_cases:
7489 + playground.run_TestCase(test_case)
7490 + self.assertEqual(
7491 + test_case.test_success, True, test_case.fail_msg
7492 + )
7493 + finally:
7494 + playground.debug = False
7495 + playground.cleanup()
7496
7497 diff --git a/lib/portage/tests/resolver/test_multirepo.py b/lib/portage/tests/resolver/test_multirepo.py
7498 index 3a8eaa3d6..8132439b5 100644
7499 --- a/lib/portage/tests/resolver/test_multirepo.py
7500 +++ b/lib/portage/tests/resolver/test_multirepo.py
7501 @@ -1,11 +1,16 @@
7502 # Copyright 2010-2020 Gentoo Authors
7503 # Distributed under the terms of the GNU General Public License v2
7504
7505 +from __future__ import print_function
7506 +import sys
7507 +
7508 +from portage.const import SUPPORTED_GENTOO_BINPKG_FORMATS
7509 from portage.tests import TestCase
7510 from portage.tests.resolver.ResolverPlayground import (
7511 ResolverPlayground,
7512 ResolverPlaygroundTestCase,
7513 )
7514 +from portage.output import colorize
7515
7516
7517 class MultirepoTestCase(TestCase):
7518 @@ -236,15 +241,28 @@ class MultirepoTestCase(TestCase):
7519 ),
7520 )
7521
7522 - playground = ResolverPlayground(
7523 - ebuilds=ebuilds, binpkgs=binpkgs, installed=installed, sets=sets
7524 - )
7525 - try:
7526 - for test_case in test_cases:
7527 - playground.run_TestCase(test_case)
7528 - self.assertEqual(test_case.test_success, True, test_case.fail_msg)
7529 - finally:
7530 - playground.cleanup()
7531 + for binpkg_format in SUPPORTED_GENTOO_BINPKG_FORMATS:
7532 + with self.subTest(binpkg_format=binpkg_format):
7533 + print(colorize("HILITE", binpkg_format), end=" ... ")
7534 + sys.stdout.flush()
7535 + playground = ResolverPlayground(
7536 + ebuilds=ebuilds,
7537 + binpkgs=binpkgs,
7538 + installed=installed,
7539 + sets=sets,
7540 + user_config={
7541 + "make.conf": ('BINPKG_FORMAT="%s"' % binpkg_format,),
7542 + },
7543 + )
7544 +
7545 + try:
7546 + for test_case in test_cases:
7547 + playground.run_TestCase(test_case)
7548 + self.assertEqual(
7549 + test_case.test_success, True, test_case.fail_msg
7550 + )
7551 + finally:
7552 + playground.cleanup()
7553
7554 def testMultirepoUserConfig(self):
7555 ebuilds = {
7556 @@ -382,12 +400,20 @@ class MultirepoTestCase(TestCase):
7557 ),
7558 )
7559
7560 - playground = ResolverPlayground(
7561 - ebuilds=ebuilds, installed=installed, user_config=user_config
7562 - )
7563 - try:
7564 - for test_case in test_cases:
7565 - playground.run_TestCase(test_case)
7566 - self.assertEqual(test_case.test_success, True, test_case.fail_msg)
7567 - finally:
7568 - playground.cleanup()
7569 + for binpkg_format in SUPPORTED_GENTOO_BINPKG_FORMATS:
7570 + with self.subTest(binpkg_format=binpkg_format):
7571 + print(colorize("HILITE", binpkg_format), end=" ... ")
7572 + sys.stdout.flush()
7573 + user_config["make.conf"] = ('BINPKG_FORMAT="%s"' % binpkg_format,)
7574 + playground = ResolverPlayground(
7575 + ebuilds=ebuilds, installed=installed, user_config=user_config
7576 + )
7577 +
7578 + try:
7579 + for test_case in test_cases:
7580 + playground.run_TestCase(test_case)
7581 + self.assertEqual(
7582 + test_case.test_success, True, test_case.fail_msg
7583 + )
7584 + finally:
7585 + playground.cleanup()
7586
7587 diff --git a/lib/portage/tests/resolver/test_regular_slot_change_without_revbump.py b/lib/portage/tests/resolver/test_regular_slot_change_without_revbump.py
7588 index bb1ce0e87..127a175d1 100644
7589 --- a/lib/portage/tests/resolver/test_regular_slot_change_without_revbump.py
7590 +++ b/lib/portage/tests/resolver/test_regular_slot_change_without_revbump.py
7591 @@ -1,11 +1,16 @@
7592 # Copyright 2013 Gentoo Foundation
7593 # Distributed under the terms of the GNU General Public License v2
7594
7595 +from __future__ import print_function
7596 +import sys
7597 +
7598 +from portage.const import SUPPORTED_GENTOO_BINPKG_FORMATS
7599 from portage.tests import TestCase
7600 from portage.tests.resolver.ResolverPlayground import (
7601 ResolverPlayground,
7602 ResolverPlaygroundTestCase,
7603 )
7604 +from portage.output import colorize
7605
7606
7607 class RegularSlotChangeWithoutRevBumpTestCase(TestCase):
7608 @@ -42,16 +47,26 @@ class RegularSlotChangeWithoutRevBumpTestCase(TestCase):
7609 ),
7610 )
7611
7612 - playground = ResolverPlayground(
7613 - ebuilds=ebuilds,
7614 - binpkgs=binpkgs,
7615 - installed=installed,
7616 - world=world,
7617 - debug=False,
7618 - )
7619 - try:
7620 - for test_case in test_cases:
7621 - playground.run_TestCase(test_case)
7622 - self.assertEqual(test_case.test_success, True, test_case.fail_msg)
7623 - finally:
7624 - playground.cleanup()
7625 + for binpkg_format in SUPPORTED_GENTOO_BINPKG_FORMATS:
7626 + with self.subTest(binpkg_format=binpkg_format):
7627 + print(colorize("HILITE", binpkg_format), end=" ... ")
7628 + sys.stdout.flush()
7629 + playground = ResolverPlayground(
7630 + ebuilds=ebuilds,
7631 + binpkgs=binpkgs,
7632 + installed=installed,
7633 + world=world,
7634 + debug=False,
7635 + user_config={
7636 + "make.conf": ('BINPKG_FORMAT="%s"' % binpkg_format,),
7637 + },
7638 + )
7639 +
7640 + try:
7641 + for test_case in test_cases:
7642 + playground.run_TestCase(test_case)
7643 + self.assertEqual(
7644 + test_case.test_success, True, test_case.fail_msg
7645 + )
7646 + finally:
7647 + playground.cleanup()
7648
7649 diff --git a/lib/portage/tests/resolver/test_simple.py b/lib/portage/tests/resolver/test_simple.py
7650 index 9bcf446be..b2bfd5fdf 100644
7651 --- a/lib/portage/tests/resolver/test_simple.py
7652 +++ b/lib/portage/tests/resolver/test_simple.py
7653 @@ -1,11 +1,16 @@
7654 # Copyright 2010-2020 Gentoo Authors
7655 # Distributed under the terms of the GNU General Public License v2
7656
7657 +from __future__ import print_function
7658 +import sys
7659 +
7660 +from portage.const import SUPPORTED_GENTOO_BINPKG_FORMATS
7661 from portage.tests import TestCase
7662 from portage.tests.resolver.ResolverPlayground import (
7663 ResolverPlayground,
7664 ResolverPlaygroundTestCase,
7665 )
7666 +from portage.output import colorize
7667
7668
7669 class SimpleResolverTestCase(TestCase):
7670 @@ -75,12 +80,23 @@ class SimpleResolverTestCase(TestCase):
7671 ),
7672 )
7673
7674 - playground = ResolverPlayground(
7675 - ebuilds=ebuilds, binpkgs=binpkgs, installed=installed
7676 - )
7677 - try:
7678 - for test_case in test_cases:
7679 - playground.run_TestCase(test_case)
7680 - self.assertEqual(test_case.test_success, True, test_case.fail_msg)
7681 - finally:
7682 - playground.cleanup()
7683 + for binpkg_format in SUPPORTED_GENTOO_BINPKG_FORMATS:
7684 + with self.subTest(binpkg_format=binpkg_format):
7685 + print(colorize("HILITE", binpkg_format), end=" ... ")
7686 + sys.stdout.flush()
7687 + playground = ResolverPlayground(
7688 + ebuilds=ebuilds,
7689 + binpkgs=binpkgs,
7690 + installed=installed,
7691 + user_config={
7692 + "make.conf": ('BINPKG_FORMAT="%s"' % binpkg_format,),
7693 + },
7694 + )
7695 + try:
7696 + for test_case in test_cases:
7697 + playground.run_TestCase(test_case)
7698 + self.assertEqual(
7699 + test_case.test_success, True, test_case.fail_msg
7700 + )
7701 + finally:
7702 + playground.cleanup()
7703
7704 diff --git a/lib/portage/tests/resolver/test_slot_abi.py b/lib/portage/tests/resolver/test_slot_abi.py
7705 index afab001df..2d99fb676 100644
7706 --- a/lib/portage/tests/resolver/test_slot_abi.py
7707 +++ b/lib/portage/tests/resolver/test_slot_abi.py
7708 @@ -1,11 +1,16 @@
7709 # Copyright 2012-2019 Gentoo Authors
7710 # Distributed under the terms of the GNU General Public License v2
7711
7712 +from __future__ import print_function
7713 +import sys
7714 +
7715 +from portage.const import SUPPORTED_GENTOO_BINPKG_FORMATS
7716 from portage.tests import TestCase
7717 from portage.tests.resolver.ResolverPlayground import (
7718 ResolverPlayground,
7719 ResolverPlaygroundTestCase,
7720 )
7721 +from portage.output import colorize
7722
7723
7724 class SlotAbiTestCase(TestCase):
7725 @@ -118,19 +123,29 @@ class SlotAbiTestCase(TestCase):
7726 ),
7727 )
7728
7729 - playground = ResolverPlayground(
7730 - ebuilds=ebuilds,
7731 - binpkgs=binpkgs,
7732 - installed=installed,
7733 - world=world,
7734 - debug=False,
7735 - )
7736 - try:
7737 - for test_case in test_cases:
7738 - playground.run_TestCase(test_case)
7739 - self.assertEqual(test_case.test_success, True, test_case.fail_msg)
7740 - finally:
7741 - playground.cleanup()
7742 + for binpkg_format in SUPPORTED_GENTOO_BINPKG_FORMATS:
7743 + with self.subTest(binpkg_format=binpkg_format):
7744 + print(colorize("HILITE", binpkg_format), end=" ... ")
7745 + sys.stdout.flush()
7746 + playground = ResolverPlayground(
7747 + ebuilds=ebuilds,
7748 + binpkgs=binpkgs,
7749 + installed=installed,
7750 + world=world,
7751 + debug=False,
7752 + user_config={
7753 + "make.conf": ('BINPKG_FORMAT="%s"' % binpkg_format,),
7754 + },
7755 + )
7756 +
7757 + try:
7758 + for test_case in test_cases:
7759 + playground.run_TestCase(test_case)
7760 + self.assertEqual(
7761 + test_case.test_success, True, test_case.fail_msg
7762 + )
7763 + finally:
7764 + playground.cleanup()
7765
7766 def testWholeSlot(self):
7767 ebuilds = {
7768 @@ -243,19 +258,29 @@ class SlotAbiTestCase(TestCase):
7769 ),
7770 )
7771
7772 - playground = ResolverPlayground(
7773 - ebuilds=ebuilds,
7774 - binpkgs=binpkgs,
7775 - installed=installed,
7776 - world=world,
7777 - debug=False,
7778 - )
7779 - try:
7780 - for test_case in test_cases:
7781 - playground.run_TestCase(test_case)
7782 - self.assertEqual(test_case.test_success, True, test_case.fail_msg)
7783 - finally:
7784 - playground.cleanup()
7785 + for binpkg_format in SUPPORTED_GENTOO_BINPKG_FORMATS:
7786 + with self.subTest(binpkg_format=binpkg_format):
7787 + print(colorize("HILITE", binpkg_format), end=" ... ")
7788 + sys.stdout.flush()
7789 + playground = ResolverPlayground(
7790 + ebuilds=ebuilds,
7791 + binpkgs=binpkgs,
7792 + installed=installed,
7793 + world=world,
7794 + debug=False,
7795 + user_config={
7796 + "make.conf": ('BINPKG_FORMAT="%s"' % binpkg_format,),
7797 + },
7798 + )
7799 +
7800 + try:
7801 + for test_case in test_cases:
7802 + playground.run_TestCase(test_case)
7803 + self.assertEqual(
7804 + test_case.test_success, True, test_case.fail_msg
7805 + )
7806 + finally:
7807 + playground.cleanup()
7808
7809 def testWholeSlotConditional(self):
7810 ebuilds = {
7811 @@ -447,16 +472,26 @@ class SlotAbiTestCase(TestCase):
7812 ),
7813 )
7814
7815 - playground = ResolverPlayground(
7816 - ebuilds=ebuilds,
7817 - binpkgs=binpkgs,
7818 - installed=installed,
7819 - world=world,
7820 - debug=False,
7821 - )
7822 - try:
7823 - for test_case in test_cases:
7824 - playground.run_TestCase(test_case)
7825 - self.assertEqual(test_case.test_success, True, test_case.fail_msg)
7826 - finally:
7827 - playground.cleanup()
7828 + for binpkg_format in SUPPORTED_GENTOO_BINPKG_FORMATS:
7829 + with self.subTest(binpkg_format=binpkg_format):
7830 + print(colorize("HILITE", binpkg_format), end=" ... ")
7831 + sys.stdout.flush()
7832 + playground = ResolverPlayground(
7833 + ebuilds=ebuilds,
7834 + binpkgs=binpkgs,
7835 + installed=installed,
7836 + world=world,
7837 + debug=False,
7838 + user_config={
7839 + "make.conf": ('BINPKG_FORMAT="%s"' % binpkg_format,),
7840 + },
7841 + )
7842 +
7843 + try:
7844 + for test_case in test_cases:
7845 + playground.run_TestCase(test_case)
7846 + self.assertEqual(
7847 + test_case.test_success, True, test_case.fail_msg
7848 + )
7849 + finally:
7850 + playground.cleanup()
7851
7852 diff --git a/lib/portage/tests/resolver/test_slot_abi_downgrade.py b/lib/portage/tests/resolver/test_slot_abi_downgrade.py
7853 index badd31b2d..15c6a7bfc 100644
7854 --- a/lib/portage/tests/resolver/test_slot_abi_downgrade.py
7855 +++ b/lib/portage/tests/resolver/test_slot_abi_downgrade.py
7856 @@ -1,11 +1,16 @@
7857 # Copyright 2012-2019 Gentoo Authors
7858 # Distributed under the terms of the GNU General Public License v2
7859
7860 +from __future__ import print_function
7861 +import sys
7862 +
7863 +from portage.const import SUPPORTED_GENTOO_BINPKG_FORMATS
7864 from portage.tests import TestCase
7865 from portage.tests.resolver.ResolverPlayground import (
7866 ResolverPlayground,
7867 ResolverPlaygroundTestCase,
7868 )
7869 +from portage.output import colorize
7870
7871
7872 class SlotAbiDowngradeTestCase(TestCase):
7873 @@ -96,19 +101,29 @@ class SlotAbiDowngradeTestCase(TestCase):
7874 ),
7875 )
7876
7877 - playground = ResolverPlayground(
7878 - ebuilds=ebuilds,
7879 - binpkgs=binpkgs,
7880 - installed=installed,
7881 - world=world,
7882 - debug=False,
7883 - )
7884 - try:
7885 - for test_case in test_cases:
7886 - playground.run_TestCase(test_case)
7887 - self.assertEqual(test_case.test_success, True, test_case.fail_msg)
7888 - finally:
7889 - playground.cleanup()
7890 + for binpkg_format in SUPPORTED_GENTOO_BINPKG_FORMATS:
7891 + with self.subTest(binpkg_format=binpkg_format):
7892 + print(colorize("HILITE", binpkg_format), end=" ... ")
7893 + sys.stdout.flush()
7894 + playground = ResolverPlayground(
7895 + ebuilds=ebuilds,
7896 + binpkgs=binpkgs,
7897 + installed=installed,
7898 + world=world,
7899 + debug=False,
7900 + user_config={
7901 + "make.conf": ('BINPKG_FORMAT="%s"' % binpkg_format,),
7902 + },
7903 + )
7904 +
7905 + try:
7906 + for test_case in test_cases:
7907 + playground.run_TestCase(test_case)
7908 + self.assertEqual(
7909 + test_case.test_success, True, test_case.fail_msg
7910 + )
7911 + finally:
7912 + playground.cleanup()
7913
7914 def testWholeSlotSubSlotMix(self):
7915 ebuilds = {
7916 @@ -197,16 +212,26 @@ class SlotAbiDowngradeTestCase(TestCase):
7917 ),
7918 )
7919
7920 - playground = ResolverPlayground(
7921 - ebuilds=ebuilds,
7922 - binpkgs=binpkgs,
7923 - installed=installed,
7924 - world=world,
7925 - debug=False,
7926 - )
7927 - try:
7928 - for test_case in test_cases:
7929 - playground.run_TestCase(test_case)
7930 - self.assertEqual(test_case.test_success, True, test_case.fail_msg)
7931 - finally:
7932 - playground.cleanup()
7933 + for binpkg_format in SUPPORTED_GENTOO_BINPKG_FORMATS:
7934 + with self.subTest(binpkg_format=binpkg_format):
7935 + print(colorize("HILITE", binpkg_format), end=" ... ")
7936 + sys.stdout.flush()
7937 + playground = ResolverPlayground(
7938 + ebuilds=ebuilds,
7939 + binpkgs=binpkgs,
7940 + installed=installed,
7941 + world=world,
7942 + debug=False,
7943 + user_config={
7944 + "make.conf": ('BINPKG_FORMAT="%s"' % binpkg_format,),
7945 + },
7946 + )
7947 +
7948 + try:
7949 + for test_case in test_cases:
7950 + playground.run_TestCase(test_case)
7951 + self.assertEqual(
7952 + test_case.test_success, True, test_case.fail_msg
7953 + )
7954 + finally:
7955 + playground.cleanup()
7956
7957 diff --git a/lib/portage/tests/resolver/test_slot_change_without_revbump.py b/lib/portage/tests/resolver/test_slot_change_without_revbump.py
7958 index 3dbd4f75e..c1c727caf 100644
7959 --- a/lib/portage/tests/resolver/test_slot_change_without_revbump.py
7960 +++ b/lib/portage/tests/resolver/test_slot_change_without_revbump.py
7961 @@ -1,11 +1,16 @@
7962 # Copyright 2013 Gentoo Foundation
7963 # Distributed under the terms of the GNU General Public License v2
7964
7965 +from __future__ import print_function
7966 +import sys
7967 +
7968 +from portage.const import SUPPORTED_GENTOO_BINPKG_FORMATS
7969 from portage.tests import TestCase
7970 from portage.tests.resolver.ResolverPlayground import (
7971 ResolverPlayground,
7972 ResolverPlaygroundTestCase,
7973 )
7974 +from portage.output import colorize
7975
7976
7977 class SlotChangeWithoutRevBumpTestCase(TestCase):
7978 @@ -71,16 +76,25 @@ class SlotChangeWithoutRevBumpTestCase(TestCase):
7979 ),
7980 )
7981
7982 - playground = ResolverPlayground(
7983 - ebuilds=ebuilds,
7984 - binpkgs=binpkgs,
7985 - installed=installed,
7986 - world=world,
7987 - debug=False,
7988 - )
7989 - try:
7990 - for test_case in test_cases:
7991 - playground.run_TestCase(test_case)
7992 - self.assertEqual(test_case.test_success, True, test_case.fail_msg)
7993 - finally:
7994 - playground.cleanup()
7995 + for binpkg_format in SUPPORTED_GENTOO_BINPKG_FORMATS:
7996 + with self.subTest(binpkg_format=binpkg_format):
7997 + print(colorize("HILITE", binpkg_format), end=" ... ")
7998 + sys.stdout.flush()
7999 + playground = ResolverPlayground(
8000 + ebuilds=ebuilds,
8001 + binpkgs=binpkgs,
8002 + installed=installed,
8003 + world=world,
8004 + debug=False,
8005 + user_config={
8006 + "make.conf": ('BINPKG_FORMAT="%s"' % binpkg_format,),
8007 + },
8008 + )
8009 + try:
8010 + for test_case in test_cases:
8011 + playground.run_TestCase(test_case)
8012 + self.assertEqual(
8013 + test_case.test_success, True, test_case.fail_msg
8014 + )
8015 + finally:
8016 + playground.cleanup()
8017
8018 diff --git a/lib/portage/tests/resolver/test_slot_operator_autounmask.py b/lib/portage/tests/resolver/test_slot_operator_autounmask.py
8019 index 7d6c3af26..b8b502a68 100644
8020 --- a/lib/portage/tests/resolver/test_slot_operator_autounmask.py
8021 +++ b/lib/portage/tests/resolver/test_slot_operator_autounmask.py
8022 @@ -1,11 +1,16 @@
8023 # Copyright 2013-2019 Gentoo Authors
8024 # Distributed under the terms of the GNU General Public License v2
8025
8026 +from __future__ import print_function
8027 +import sys
8028 +
8029 +from portage.const import SUPPORTED_GENTOO_BINPKG_FORMATS
8030 from portage.tests import TestCase
8031 from portage.tests.resolver.ResolverPlayground import (
8032 ResolverPlayground,
8033 ResolverPlaygroundTestCase,
8034 )
8035 +from portage.output import colorize
8036
8037
8038 class SlotOperatorAutoUnmaskTestCase(TestCase):
8039 @@ -109,16 +114,25 @@ class SlotOperatorAutoUnmaskTestCase(TestCase):
8040 ),
8041 )
8042
8043 - playground = ResolverPlayground(
8044 - ebuilds=ebuilds,
8045 - binpkgs=binpkgs,
8046 - installed=installed,
8047 - world=world,
8048 - debug=False,
8049 - )
8050 - try:
8051 - for test_case in test_cases:
8052 - playground.run_TestCase(test_case)
8053 - self.assertEqual(test_case.test_success, True, test_case.fail_msg)
8054 - finally:
8055 - playground.cleanup()
8056 + for binpkg_format in SUPPORTED_GENTOO_BINPKG_FORMATS:
8057 + with self.subTest(binpkg_format=binpkg_format):
8058 + print(colorize("HILITE", binpkg_format), end=" ... ")
8059 + sys.stdout.flush()
8060 + playground = ResolverPlayground(
8061 + ebuilds=ebuilds,
8062 + binpkgs=binpkgs,
8063 + installed=installed,
8064 + world=world,
8065 + debug=False,
8066 + user_config={
8067 + "make.conf": ('BINPKG_FORMAT="%s"' % binpkg_format,),
8068 + },
8069 + )
8070 + try:
8071 + for test_case in test_cases:
8072 + playground.run_TestCase(test_case)
8073 + self.assertEqual(
8074 + test_case.test_success, True, test_case.fail_msg
8075 + )
8076 + finally:
8077 + playground.cleanup()
8078
8079 diff --git a/lib/portage/tests/resolver/test_slot_operator_bdeps.py b/lib/portage/tests/resolver/test_slot_operator_bdeps.py
8080 index 0b1f426b7..6f0e5f7e1 100644
8081 --- a/lib/portage/tests/resolver/test_slot_operator_bdeps.py
8082 +++ b/lib/portage/tests/resolver/test_slot_operator_bdeps.py
8083 @@ -1,11 +1,13 @@
8084 # Copyright 2020 Gentoo Authors
8085 # Distributed under the terms of the GNU General Public License v2
8086
8087 +from portage.const import SUPPORTED_GENTOO_BINPKG_FORMATS
8088 from portage.tests import TestCase
8089 from portage.tests.resolver.ResolverPlayground import (
8090 ResolverPlayground,
8091 ResolverPlaygroundTestCase,
8092 )
8093 +from portage.output import colorize
8094
8095
8096 class SlotOperatorBdependTestCase(TestCase):
8097 @@ -88,20 +90,28 @@ class SlotOperatorBdependTestCase(TestCase):
8098 ),
8099 )
8100
8101 - playground = ResolverPlayground(
8102 - ebuilds=ebuilds,
8103 - binpkgs=binpkgs,
8104 - installed=installed,
8105 - world=world,
8106 - debug=False,
8107 - )
8108 - try:
8109 - for test_case in test_cases:
8110 - playground.run_TestCase(test_case)
8111 - self.assertEqual(test_case.test_success, True, test_case.fail_msg)
8112 - finally:
8113 - playground.debug = False
8114 - playground.cleanup()
8115 + for binpkg_format in SUPPORTED_GENTOO_BINPKG_FORMATS:
8116 + with self.subTest(binpkg_format=binpkg_format):
8117 + print(colorize("HILITE", binpkg_format), end=" ... ")
8118 + playground = ResolverPlayground(
8119 + ebuilds=ebuilds,
8120 + binpkgs=binpkgs,
8121 + installed=installed,
8122 + world=world,
8123 + debug=False,
8124 + user_config={
8125 + "make.conf": ('BINPKG_FORMAT="%s"' % binpkg_format,),
8126 + },
8127 + )
8128 + try:
8129 + for test_case in test_cases:
8130 + playground.run_TestCase(test_case)
8131 + self.assertEqual(
8132 + test_case.test_success, True, test_case.fail_msg
8133 + )
8134 + finally:
8135 + playground.debug = False
8136 + playground.cleanup()
8137
8138 def testSlotOperatorBdependAfterBreakage(self):
8139 """
8140 @@ -182,17 +192,25 @@ class SlotOperatorBdependTestCase(TestCase):
8141 ),
8142 )
8143
8144 - playground = ResolverPlayground(
8145 - ebuilds=ebuilds,
8146 - binpkgs=binpkgs,
8147 - installed=installed,
8148 - world=world,
8149 - debug=False,
8150 - )
8151 - try:
8152 - for test_case in test_cases:
8153 - playground.run_TestCase(test_case)
8154 - self.assertEqual(test_case.test_success, True, test_case.fail_msg)
8155 - finally:
8156 - playground.debug = False
8157 - playground.cleanup()
8158 + for binpkg_format in SUPPORTED_GENTOO_BINPKG_FORMATS:
8159 + with self.subTest(binpkg_format=binpkg_format):
8160 + print(colorize("HILITE", binpkg_format), end=" ... ")
8161 + playground = ResolverPlayground(
8162 + ebuilds=ebuilds,
8163 + binpkgs=binpkgs,
8164 + installed=installed,
8165 + world=world,
8166 + debug=False,
8167 + user_config={
8168 + "make.conf": ('BINPKG_FORMAT="%s"' % binpkg_format,),
8169 + },
8170 + )
8171 + try:
8172 + for test_case in test_cases:
8173 + playground.run_TestCase(test_case)
8174 + self.assertEqual(
8175 + test_case.test_success, True, test_case.fail_msg
8176 + )
8177 + finally:
8178 + playground.debug = False
8179 + playground.cleanup()
8180
8181 diff --git a/lib/portage/tests/resolver/test_slot_operator_rebuild.py b/lib/portage/tests/resolver/test_slot_operator_rebuild.py
8182 index 9e2325afb..9ad2bc7ab 100644
8183 --- a/lib/portage/tests/resolver/test_slot_operator_rebuild.py
8184 +++ b/lib/portage/tests/resolver/test_slot_operator_rebuild.py
8185 @@ -1,11 +1,16 @@
8186 # Copyright 2014-2018 Gentoo Foundation
8187 # Distributed under the terms of the GNU General Public License v2
8188
8189 +from __future__ import print_function
8190 +import sys
8191 +
8192 +from portage.const import SUPPORTED_GENTOO_BINPKG_FORMATS
8193 from portage.tests import TestCase
8194 from portage.tests.resolver.ResolverPlayground import (
8195 ResolverPlayground,
8196 ResolverPlaygroundTestCase,
8197 )
8198 +from portage.output import colorize
8199
8200
8201 class SlotOperatorRebuildTestCase(TestCase):
8202 @@ -72,16 +77,25 @@ class SlotOperatorRebuildTestCase(TestCase):
8203 ),
8204 )
8205
8206 - playground = ResolverPlayground(
8207 - ebuilds=ebuilds,
8208 - binpkgs=binpkgs,
8209 - installed=installed,
8210 - world=world,
8211 - debug=False,
8212 - )
8213 - try:
8214 - for test_case in test_cases:
8215 - playground.run_TestCase(test_case)
8216 - self.assertEqual(test_case.test_success, True, test_case.fail_msg)
8217 - finally:
8218 - playground.cleanup()
8219 + for binpkg_format in SUPPORTED_GENTOO_BINPKG_FORMATS:
8220 + with self.subTest(binpkg_format=binpkg_format):
8221 + print(colorize("HILITE", binpkg_format), end=" ... ")
8222 + sys.stdout.flush()
8223 + playground = ResolverPlayground(
8224 + ebuilds=ebuilds,
8225 + binpkgs=binpkgs,
8226 + installed=installed,
8227 + world=world,
8228 + debug=False,
8229 + user_config={
8230 + "make.conf": ('BINPKG_FORMAT="%s"' % binpkg_format,),
8231 + },
8232 + )
8233 + try:
8234 + for test_case in test_cases:
8235 + playground.run_TestCase(test_case)
8236 + self.assertEqual(
8237 + test_case.test_success, True, test_case.fail_msg
8238 + )
8239 + finally:
8240 + playground.cleanup()
8241
8242 diff --git a/lib/portage/tests/resolver/test_slot_operator_unsolved.py b/lib/portage/tests/resolver/test_slot_operator_unsolved.py
8243 index d43e8367d..945e34ccf 100644
8244 --- a/lib/portage/tests/resolver/test_slot_operator_unsolved.py
8245 +++ b/lib/portage/tests/resolver/test_slot_operator_unsolved.py
8246 @@ -1,11 +1,16 @@
8247 # Copyright 2013 Gentoo Foundation
8248 # Distributed under the terms of the GNU General Public License v2
8249
8250 +from __future__ import print_function
8251 +import sys
8252 +
8253 +from portage.const import SUPPORTED_GENTOO_BINPKG_FORMATS
8254 from portage.tests import TestCase
8255 from portage.tests.resolver.ResolverPlayground import (
8256 ResolverPlayground,
8257 ResolverPlaygroundTestCase,
8258 )
8259 +from portage.output import colorize
8260
8261
8262 class SlotOperatorUnsolvedTestCase(TestCase):
8263 @@ -71,17 +76,25 @@ class SlotOperatorUnsolvedTestCase(TestCase):
8264 ),
8265 )
8266
8267 - playground = ResolverPlayground(
8268 - ebuilds=ebuilds,
8269 - binpkgs=binpkgs,
8270 - installed=installed,
8271 - user_config=user_config,
8272 - world=world,
8273 - debug=False,
8274 - )
8275 - try:
8276 - for test_case in test_cases:
8277 - playground.run_TestCase(test_case)
8278 - self.assertEqual(test_case.test_success, True, test_case.fail_msg)
8279 - finally:
8280 - playground.cleanup()
8281 + for binpkg_format in SUPPORTED_GENTOO_BINPKG_FORMATS:
8282 + with self.subTest(binpkg_format=binpkg_format):
8283 + print(colorize("HILITE", binpkg_format), end=" ... ")
8284 + sys.stdout.flush()
8285 + _user_config = user_config.copy()
8286 + _user_config["make.conf"] += ('BINPKG_FORMAT="%s"' % binpkg_format,)
8287 + playground = ResolverPlayground(
8288 + ebuilds=ebuilds,
8289 + binpkgs=binpkgs,
8290 + installed=installed,
8291 + user_config=_user_config,
8292 + world=world,
8293 + debug=False,
8294 + )
8295 + try:
8296 + for test_case in test_cases:
8297 + playground.run_TestCase(test_case)
8298 + self.assertEqual(
8299 + test_case.test_success, True, test_case.fail_msg
8300 + )
8301 + finally:
8302 + playground.cleanup()
8303
8304 diff --git a/lib/portage/tests/resolver/test_useflags.py b/lib/portage/tests/resolver/test_useflags.py
8305 index 6d74807e5..340ac5de2 100644
8306 --- a/lib/portage/tests/resolver/test_useflags.py
8307 +++ b/lib/portage/tests/resolver/test_useflags.py
8308 @@ -1,11 +1,16 @@
8309 # Copyright 2014 Gentoo Foundation
8310 # Distributed under the terms of the GNU General Public License v2
8311
8312 +from __future__ import print_function
8313 +import sys
8314 +
8315 +from portage.const import SUPPORTED_GENTOO_BINPKG_FORMATS
8316 from portage.tests import TestCase
8317 from portage.tests.resolver.ResolverPlayground import (
8318 ResolverPlayground,
8319 ResolverPlaygroundTestCase,
8320 )
8321 +from portage.output import colorize
8322
8323
8324 class UseFlagsTestCase(TestCase):
8325 @@ -118,15 +123,23 @@ class UseFlagsTestCase(TestCase):
8326 ),
8327 )
8328
8329 - playground = ResolverPlayground(
8330 - ebuilds=ebuilds,
8331 - binpkgs=binpkgs,
8332 - installed=installed,
8333 - user_config=user_config,
8334 - )
8335 - try:
8336 - for test_case in test_cases:
8337 - playground.run_TestCase(test_case)
8338 - self.assertEqual(test_case.test_success, True, test_case.fail_msg)
8339 - finally:
8340 - playground.cleanup()
8341 + for binpkg_format in SUPPORTED_GENTOO_BINPKG_FORMATS:
8342 + with self.subTest(binpkg_format=binpkg_format):
8343 + print(colorize("HILITE", binpkg_format), end=" ... ")
8344 + sys.stdout.flush()
8345 + user_config["make.conf"] = ('BINPKG_FORMAT="%s"' % binpkg_format,)
8346 + playground = ResolverPlayground(
8347 + ebuilds=ebuilds,
8348 + binpkgs=binpkgs,
8349 + installed=installed,
8350 + user_config=user_config,
8351 + )
8352 +
8353 + try:
8354 + for test_case in test_cases:
8355 + playground.run_TestCase(test_case)
8356 + self.assertEqual(
8357 + test_case.test_success, True, test_case.fail_msg
8358 + )
8359 + finally:
8360 + playground.cleanup()
8361
8362 diff --git a/lib/portage/tests/runTests.py b/lib/portage/tests/runTests.py
8363 index 85b746092..167b40d5a 100755
8364 --- a/lib/portage/tests/runTests.py
8365 +++ b/lib/portage/tests/runTests.py
8366 @@ -9,7 +9,10 @@ import os.path as osp
8367 import platform
8368 import pwd
8369 import signal
8370 +import tempfile
8371 +import shutil
8372 import sys
8373 +from distutils.dir_util import copy_tree
8374
8375
8376 def debug_signal(signum, frame):
8377 @@ -63,8 +66,17 @@ if insert_bin_path:
8378 path.insert(0, PORTAGE_BIN_PATH)
8379 os.environ["PATH"] = ":".join(path)
8380
8381 +# Copy GPG test keys to temporary directory
8382 +gpg_path = tempfile.mkdtemp(prefix="gpg_")
8383 +
8384 +copy_tree(os.path.join(os.path.dirname(os.path.realpath(__file__)), ".gnupg"), gpg_path)
8385 +
8386 +os.chmod(gpg_path, 0o700)
8387 +os.environ["PORTAGE_GNUPGHOME"] = gpg_path
8388 +
8389 if __name__ == "__main__":
8390 try:
8391 sys.exit(tests.main())
8392 finally:
8393 global_event_loop().close()
8394 + shutil.rmtree(gpg_path, ignore_errors=True)
8395
8396 diff --git a/lib/portage/tests/update/test_move_ent.py b/lib/portage/tests/update/test_move_ent.py
8397 index ba5add989..cb9bb5243 100644
8398 --- a/lib/portage/tests/update/test_move_ent.py
8399 +++ b/lib/portage/tests/update/test_move_ent.py
8400 @@ -1,14 +1,18 @@
8401 # Copyright 2012-2021 Gentoo Authors
8402 # Distributed under the terms of the GNU General Public License v2
8403
8404 +from __future__ import print_function
8405 +import sys
8406 import textwrap
8407
8408 import portage
8409 from portage import os
8410 +from portage.const import SUPPORTED_GENTOO_BINPKG_FORMATS
8411 from portage.tests import TestCase
8412 from portage.tests.resolver.ResolverPlayground import ResolverPlayground
8413 from portage.util import ensure_dirs
8414 from portage._global_updates import _do_global_updates
8415 +from portage.output import colorize
8416
8417
8418 class MoveEntTestCase(TestCase):
8419 @@ -47,60 +51,73 @@ class MoveEntTestCase(TestCase):
8420 """
8421 )
8422
8423 - playground = ResolverPlayground(
8424 - binpkgs=binpkgs, ebuilds=ebuilds, installed=installed
8425 - )
8426 -
8427 - settings = playground.settings
8428 - trees = playground.trees
8429 - eroot = settings["EROOT"]
8430 - test_repo_location = settings.repositories["test_repo"].location
8431 - portdb = trees[eroot]["porttree"].dbapi
8432 - vardb = trees[eroot]["vartree"].dbapi
8433 - bindb = trees[eroot]["bintree"].dbapi
8434 -
8435 - updates_dir = os.path.join(test_repo_location, "profiles", "updates")
8436 -
8437 - try:
8438 - ensure_dirs(updates_dir)
8439 - with open(os.path.join(updates_dir, "1Q-2010"), "w") as f:
8440 - f.write(updates)
8441 -
8442 - # Create an empty updates directory, so that this
8443 - # repo doesn't inherit updates from the main repo.
8444 - ensure_dirs(
8445 - os.path.join(
8446 - portdb.getRepositoryPath("dont_apply_updates"),
8447 - "profiles",
8448 - "updates",
8449 + for binpkg_format in SUPPORTED_GENTOO_BINPKG_FORMATS:
8450 + with self.subTest(binpkg_format=binpkg_format):
8451 + print(colorize("HILITE", binpkg_format), end=" ... ")
8452 + sys.stdout.flush()
8453 + playground = ResolverPlayground(
8454 + binpkgs=binpkgs,
8455 + ebuilds=ebuilds,
8456 + installed=installed,
8457 + user_config={
8458 + "make.conf": ('BINPKG_FORMAT="%s"' % binpkg_format,),
8459 + },
8460 )
8461 - )
8462 -
8463 - global_noiselimit = portage.util.noiselimit
8464 - portage.util.noiselimit = -2
8465 - try:
8466 - _do_global_updates(trees, {})
8467 - finally:
8468 - portage.util.noiselimit = global_noiselimit
8469 -
8470 - # Workaround for cache validation not working
8471 - # correctly when filesystem has timestamp precision
8472 - # of 1 second.
8473 - vardb._clear_cache()
8474 -
8475 - # A -> A-moved
8476 - self.assertRaises(KeyError, vardb.aux_get, "dev-libs/A-1", ["EAPI"])
8477 - vardb.aux_get("dev-libs/A-moved-1", ["EAPI"])
8478 - # The original package should still exist because a binary
8479 - # package move is a copy on write operation.
8480 - bindb.aux_get("dev-libs/A-1", ["EAPI"])
8481 - bindb.aux_get("dev-libs/A-moved-1", ["EAPI"])
8482 -
8483 - # dont_apply_updates
8484 - self.assertRaises(KeyError, vardb.aux_get, "dev-libs/A-moved-2", ["EAPI"])
8485 - vardb.aux_get("dev-libs/A-2", ["EAPI"])
8486 - self.assertRaises(KeyError, bindb.aux_get, "dev-libs/A-moved-2", ["EAPI"])
8487 - bindb.aux_get("dev-libs/A-2", ["EAPI"])
8488 -
8489 - finally:
8490 - playground.cleanup()
8491 +
8492 + settings = playground.settings
8493 + trees = playground.trees
8494 + eroot = settings["EROOT"]
8495 + test_repo_location = settings.repositories["test_repo"].location
8496 + portdb = trees[eroot]["porttree"].dbapi
8497 + vardb = trees[eroot]["vartree"].dbapi
8498 + bindb = trees[eroot]["bintree"].dbapi
8499 +
8500 + updates_dir = os.path.join(test_repo_location, "profiles", "updates")
8501 +
8502 + try:
8503 + ensure_dirs(updates_dir)
8504 + with open(os.path.join(updates_dir, "1Q-2010"), "w") as f:
8505 + f.write(updates)
8506 +
8507 + # Create an empty updates directory, so that this
8508 + # repo doesn't inherit updates from the main repo.
8509 + ensure_dirs(
8510 + os.path.join(
8511 + portdb.getRepositoryPath("dont_apply_updates"),
8512 + "profiles",
8513 + "updates",
8514 + )
8515 + )
8516 +
8517 + global_noiselimit = portage.util.noiselimit
8518 + portage.util.noiselimit = -2
8519 + try:
8520 + _do_global_updates(trees, {})
8521 + finally:
8522 + portage.util.noiselimit = global_noiselimit
8523 +
8524 + # Workaround for cache validation not working
8525 + # correctly when filesystem has timestamp precision
8526 + # of 1 second.
8527 + vardb._clear_cache()
8528 +
8529 + # A -> A-moved
8530 + self.assertRaises(KeyError, vardb.aux_get, "dev-libs/A-1", ["EAPI"])
8531 + vardb.aux_get("dev-libs/A-moved-1", ["EAPI"])
8532 + # The original package should still exist because a binary
8533 + # package move is a copy on write operation.
8534 + bindb.aux_get("dev-libs/A-1", ["EAPI"])
8535 + bindb.aux_get("dev-libs/A-moved-1", ["EAPI"])
8536 +
8537 + # dont_apply_updates
8538 + self.assertRaises(
8539 + KeyError, vardb.aux_get, "dev-libs/A-moved-2", ["EAPI"]
8540 + )
8541 + vardb.aux_get("dev-libs/A-2", ["EAPI"])
8542 + self.assertRaises(
8543 + KeyError, bindb.aux_get, "dev-libs/A-moved-2", ["EAPI"]
8544 + )
8545 + bindb.aux_get("dev-libs/A-2", ["EAPI"])
8546 +
8547 + finally:
8548 + playground.cleanup()
8549
8550 diff --git a/lib/portage/tests/update/test_move_slot_ent.py b/lib/portage/tests/update/test_move_slot_ent.py
8551 index 87e38f97e..27d51fbb1 100644
8552 --- a/lib/portage/tests/update/test_move_slot_ent.py
8553 +++ b/lib/portage/tests/update/test_move_slot_ent.py
8554 @@ -1,14 +1,18 @@
8555 # Copyright 2012-2019 Gentoo Authors
8556 # Distributed under the terms of the GNU General Public License v2
8557
8558 +from __future__ import print_function
8559 +import sys
8560 import textwrap
8561
8562 import portage
8563 from portage import os
8564 +from portage.const import SUPPORTED_GENTOO_BINPKG_FORMATS
8565 from portage.tests import TestCase
8566 from portage.tests.resolver.ResolverPlayground import ResolverPlayground
8567 from portage.util import ensure_dirs
8568 from portage._global_updates import _do_global_updates
8569 +from portage.output import colorize
8570
8571
8572 class MoveSlotEntTestCase(TestCase):
8573 @@ -75,63 +79,80 @@ class MoveSlotEntTestCase(TestCase):
8574 """
8575 )
8576
8577 - playground = ResolverPlayground(
8578 - binpkgs=binpkgs, ebuilds=ebuilds, installed=installed
8579 - )
8580 -
8581 - settings = playground.settings
8582 - trees = playground.trees
8583 - eroot = settings["EROOT"]
8584 - test_repo_location = settings.repositories["test_repo"].location
8585 - portdb = trees[eroot]["porttree"].dbapi
8586 - vardb = trees[eroot]["vartree"].dbapi
8587 - bindb = trees[eroot]["bintree"].dbapi
8588 -
8589 - updates_dir = os.path.join(test_repo_location, "profiles", "updates")
8590 -
8591 - try:
8592 - ensure_dirs(updates_dir)
8593 - with open(os.path.join(updates_dir, "1Q-2010"), "w") as f:
8594 - f.write(updates)
8595 -
8596 - # Create an empty updates directory, so that this
8597 - # repo doesn't inherit updates from the main repo.
8598 - ensure_dirs(
8599 - os.path.join(
8600 - portdb.getRepositoryPath("dont_apply_updates"),
8601 - "profiles",
8602 - "updates",
8603 + for binpkg_format in SUPPORTED_GENTOO_BINPKG_FORMATS:
8604 + with self.subTest(binpkg_format=binpkg_format):
8605 + print(colorize("HILITE", binpkg_format), end=" ... ")
8606 + sys.stdout.flush()
8607 + playground = ResolverPlayground(
8608 + binpkgs=binpkgs,
8609 + ebuilds=ebuilds,
8610 + installed=installed,
8611 + user_config={
8612 + "make.conf": ('BINPKG_FORMAT="%s"' % binpkg_format,),
8613 + },
8614 )
8615 - )
8616 -
8617 - global_noiselimit = portage.util.noiselimit
8618 - portage.util.noiselimit = -2
8619 - try:
8620 - _do_global_updates(trees, {})
8621 - finally:
8622 - portage.util.noiselimit = global_noiselimit
8623 -
8624 - # Workaround for cache validation not working
8625 - # correctly when filesystem has timestamp precision
8626 - # of 1 second.
8627 - vardb._clear_cache()
8628 -
8629 - # 0/2.30 -> 2/2.30
8630 - self.assertEqual("2/2.30", vardb.aux_get("dev-libs/A-1", ["SLOT"])[0])
8631 - self.assertEqual("2/2.30", bindb.aux_get("dev-libs/A-1", ["SLOT"])[0])
8632 -
8633 - # 0 -> 1
8634 - self.assertEqual("1", vardb.aux_get("dev-libs/B-1", ["SLOT"])[0])
8635 - self.assertEqual("1", bindb.aux_get("dev-libs/B-1", ["SLOT"])[0])
8636 -
8637 - # 0/1 -> 1 (equivalent to 1/1)
8638 - self.assertEqual("1", vardb.aux_get("dev-libs/C-1", ["SLOT"])[0])
8639 - self.assertEqual("1", bindb.aux_get("dev-libs/C-1", ["SLOT"])[0])
8640 -
8641 - # dont_apply_updates
8642 - self.assertEqual("0/2.30", bindb.aux_get("dev-libs/A-2", ["SLOT"])[0])
8643 - self.assertEqual("0", bindb.aux_get("dev-libs/B-2", ["SLOT"])[0])
8644 - self.assertEqual("0/2.1", bindb.aux_get("dev-libs/C-2.1", ["SLOT"])[0])
8645 -
8646 - finally:
8647 - playground.cleanup()
8648 +
8649 + settings = playground.settings
8650 + trees = playground.trees
8651 + eroot = settings["EROOT"]
8652 + test_repo_location = settings.repositories["test_repo"].location
8653 + portdb = trees[eroot]["porttree"].dbapi
8654 + vardb = trees[eroot]["vartree"].dbapi
8655 + bindb = trees[eroot]["bintree"].dbapi
8656 +
8657 + updates_dir = os.path.join(test_repo_location, "profiles", "updates")
8658 +
8659 + try:
8660 + ensure_dirs(updates_dir)
8661 + with open(os.path.join(updates_dir, "1Q-2010"), "w") as f:
8662 + f.write(updates)
8663 +
8664 + # Create an empty updates directory, so that this
8665 + # repo doesn't inherit updates from the main repo.
8666 + ensure_dirs(
8667 + os.path.join(
8668 + portdb.getRepositoryPath("dont_apply_updates"),
8669 + "profiles",
8670 + "updates",
8671 + )
8672 + )
8673 +
8674 + global_noiselimit = portage.util.noiselimit
8675 + portage.util.noiselimit = -2
8676 + try:
8677 + _do_global_updates(trees, {})
8678 + finally:
8679 + portage.util.noiselimit = global_noiselimit
8680 +
8681 + # Workaround for cache validation not working
8682 + # correctly when filesystem has timestamp precision
8683 + # of 1 second.
8684 + vardb._clear_cache()
8685 +
8686 + # 0/2.30 -> 2/2.30
8687 + self.assertEqual(
8688 + "2/2.30", vardb.aux_get("dev-libs/A-1", ["SLOT"])[0]
8689 + )
8690 + self.assertEqual(
8691 + "2/2.30", bindb.aux_get("dev-libs/A-1", ["SLOT"])[0]
8692 + )
8693 +
8694 + # 0 -> 1
8695 + self.assertEqual("1", vardb.aux_get("dev-libs/B-1", ["SLOT"])[0])
8696 + self.assertEqual("1", bindb.aux_get("dev-libs/B-1", ["SLOT"])[0])
8697 +
8698 + # 0/1 -> 1 (equivalent to 1/1)
8699 + self.assertEqual("1", vardb.aux_get("dev-libs/C-1", ["SLOT"])[0])
8700 + self.assertEqual("1", bindb.aux_get("dev-libs/C-1", ["SLOT"])[0])
8701 +
8702 + # dont_apply_updates
8703 + self.assertEqual(
8704 + "0/2.30", bindb.aux_get("dev-libs/A-2", ["SLOT"])[0]
8705 + )
8706 + self.assertEqual("0", bindb.aux_get("dev-libs/B-2", ["SLOT"])[0])
8707 + self.assertEqual(
8708 + "0/2.1", bindb.aux_get("dev-libs/C-2.1", ["SLOT"])[0]
8709 + )
8710 +
8711 + finally:
8712 + playground.cleanup()
8713
8714 diff --git a/lib/portage/tests/update/test_update_dbentry.py b/lib/portage/tests/update/test_update_dbentry.py
8715 index bed0f4b7c..3b49cba9d 100644
8716 --- a/lib/portage/tests/update/test_update_dbentry.py
8717 +++ b/lib/portage/tests/update/test_update_dbentry.py
8718 @@ -1,11 +1,14 @@
8719 # Copyright 2012-2013 Gentoo Foundation
8720 # Distributed under the terms of the GNU General Public License v2
8721
8722 +from __future__ import print_function
8723 +import sys
8724 import re
8725 import textwrap
8726
8727 import portage
8728 from portage import os
8729 +from portage.const import SUPPORTED_GENTOO_BINPKG_FORMATS
8730 from portage.dep import Atom
8731 from portage.tests import TestCase
8732 from portage.tests.resolver.ResolverPlayground import ResolverPlayground
8733 @@ -13,6 +16,7 @@ from portage.update import update_dbentry
8734 from portage.util import ensure_dirs
8735 from portage.versions import _pkg_str
8736 from portage._global_updates import _do_global_updates
8737 +from portage.output import colorize
8738
8739
8740 class UpdateDbentryTestCase(TestCase):
8741 @@ -224,98 +228,110 @@ class UpdateDbentryTestCase(TestCase):
8742 """
8743 )
8744
8745 - playground = ResolverPlayground(
8746 - binpkgs=binpkgs, ebuilds=ebuilds, installed=installed, world=world
8747 - )
8748 + for binpkg_format in SUPPORTED_GENTOO_BINPKG_FORMATS:
8749 + with self.subTest(binpkg_format=binpkg_format):
8750 + print(colorize("HILITE", binpkg_format), end=" ... ")
8751 + sys.stdout.flush()
8752 + playground = ResolverPlayground(
8753 + binpkgs=binpkgs,
8754 + ebuilds=ebuilds,
8755 + installed=installed,
8756 + world=world,
8757 + user_config={
8758 + "make.conf": ('BINPKG_FORMAT="%s"' % binpkg_format,),
8759 + },
8760 + )
8761
8762 - settings = playground.settings
8763 - trees = playground.trees
8764 - eroot = settings["EROOT"]
8765 - test_repo_location = settings.repositories["test_repo"].location
8766 - portdb = trees[eroot]["porttree"].dbapi
8767 - vardb = trees[eroot]["vartree"].dbapi
8768 - bindb = trees[eroot]["bintree"].dbapi
8769 - setconfig = trees[eroot]["root_config"].setconfig
8770 - selected_set = setconfig.getSets()["selected"]
8771 + settings = playground.settings
8772 + trees = playground.trees
8773 + eroot = settings["EROOT"]
8774 + test_repo_location = settings.repositories["test_repo"].location
8775 + portdb = trees[eroot]["porttree"].dbapi
8776 + vardb = trees[eroot]["vartree"].dbapi
8777 + bindb = trees[eroot]["bintree"].dbapi
8778 + setconfig = trees[eroot]["root_config"].setconfig
8779 + selected_set = setconfig.getSets()["selected"]
8780
8781 - updates_dir = os.path.join(test_repo_location, "profiles", "updates")
8782 + updates_dir = os.path.join(test_repo_location, "profiles", "updates")
8783
8784 - try:
8785 - ensure_dirs(updates_dir)
8786 - with open(os.path.join(updates_dir, "1Q-2010"), "w") as f:
8787 - f.write(updates)
8788 + try:
8789 + ensure_dirs(updates_dir)
8790 + with open(os.path.join(updates_dir, "1Q-2010"), "w") as f:
8791 + f.write(updates)
8792
8793 - # Create an empty updates directory, so that this
8794 - # repo doesn't inherit updates from the main repo.
8795 - ensure_dirs(
8796 - os.path.join(
8797 - portdb.getRepositoryPath("dont_apply_updates"),
8798 - "profiles",
8799 - "updates",
8800 - )
8801 - )
8802 + # Create an empty updates directory, so that this
8803 + # repo doesn't inherit updates from the main repo.
8804 + ensure_dirs(
8805 + os.path.join(
8806 + portdb.getRepositoryPath("dont_apply_updates"),
8807 + "profiles",
8808 + "updates",
8809 + )
8810 + )
8811
8812 - global_noiselimit = portage.util.noiselimit
8813 - portage.util.noiselimit = -2
8814 - try:
8815 - _do_global_updates(trees, {})
8816 - finally:
8817 - portage.util.noiselimit = global_noiselimit
8818 + global_noiselimit = portage.util.noiselimit
8819 + portage.util.noiselimit = -2
8820 + try:
8821 + _do_global_updates(trees, {})
8822 + finally:
8823 + portage.util.noiselimit = global_noiselimit
8824
8825 - # Workaround for cache validation not working
8826 - # correctly when filesystem has timestamp precision
8827 - # of 1 second.
8828 - vardb._clear_cache()
8829 + # Workaround for cache validation not working
8830 + # correctly when filesystem has timestamp precision
8831 + # of 1 second.
8832 + vardb._clear_cache()
8833
8834 - # M -> M-moved
8835 - old_pattern = re.compile(r"\bdev-libs/M(\s|$)")
8836 - rdepend = vardb.aux_get("dev-libs/A-1", ["RDEPEND"])[0]
8837 - self.assertTrue(old_pattern.search(rdepend) is None)
8838 - self.assertTrue("dev-libs/M-moved" in rdepend)
8839 - rdepend = bindb.aux_get("dev-libs/A-1", ["RDEPEND"])[0]
8840 - self.assertTrue(old_pattern.search(rdepend) is None)
8841 - self.assertTrue("dev-libs/M-moved" in rdepend)
8842 - rdepend = vardb.aux_get("dev-libs/B-1", ["RDEPEND"])[0]
8843 - self.assertTrue(old_pattern.search(rdepend) is None)
8844 - self.assertTrue("dev-libs/M-moved" in rdepend)
8845 - rdepend = vardb.aux_get("dev-libs/B-1", ["RDEPEND"])[0]
8846 - self.assertTrue(old_pattern.search(rdepend) is None)
8847 - self.assertTrue("dev-libs/M-moved" in rdepend)
8848 + # M -> M-moved
8849 + old_pattern = re.compile(r"\bdev-libs/M(\s|$)")
8850 + rdepend = vardb.aux_get("dev-libs/A-1", ["RDEPEND"])[0]
8851 + self.assertTrue(old_pattern.search(rdepend) is None)
8852 + self.assertTrue("dev-libs/M-moved" in rdepend)
8853 + rdepend = bindb.aux_get("dev-libs/A-1", ["RDEPEND"])[0]
8854 + self.assertTrue(old_pattern.search(rdepend) is None)
8855 + self.assertTrue("dev-libs/M-moved" in rdepend)
8856 + rdepend = vardb.aux_get("dev-libs/B-1", ["RDEPEND"])[0]
8857 + self.assertTrue(old_pattern.search(rdepend) is None)
8858 + self.assertTrue("dev-libs/M-moved" in rdepend)
8859 + rdepend = vardb.aux_get("dev-libs/B-1", ["RDEPEND"])[0]
8860 + self.assertTrue(old_pattern.search(rdepend) is None)
8861 + self.assertTrue("dev-libs/M-moved" in rdepend)
8862
8863 - # EAPI 4-python/*-progress N -> N.moved
8864 - rdepend = vardb.aux_get("dev-libs/B-1", ["RDEPEND"])[0]
8865 - old_pattern = re.compile(r"\bdev-libs/N(\s|$)")
8866 - self.assertTrue(old_pattern.search(rdepend) is None)
8867 - self.assertTrue("dev-libs/N.moved" in rdepend)
8868 - rdepend = bindb.aux_get("dev-libs/B-1", ["RDEPEND"])[0]
8869 - self.assertTrue(old_pattern.search(rdepend) is None)
8870 - self.assertTrue("dev-libs/N.moved" in rdepend)
8871 - self.assertRaises(KeyError, vardb.aux_get, "dev-libs/N-2", ["EAPI"])
8872 - vardb.aux_get("dev-libs/N.moved-2", ["RDEPEND"])[0]
8873 + # EAPI 4-python/*-progress N -> N.moved
8874 + rdepend = vardb.aux_get("dev-libs/B-1", ["RDEPEND"])[0]
8875 + old_pattern = re.compile(r"\bdev-libs/N(\s|$)")
8876 + self.assertTrue(old_pattern.search(rdepend) is None)
8877 + self.assertTrue("dev-libs/N.moved" in rdepend)
8878 + rdepend = bindb.aux_get("dev-libs/B-1", ["RDEPEND"])[0]
8879 + self.assertTrue(old_pattern.search(rdepend) is None)
8880 + self.assertTrue("dev-libs/N.moved" in rdepend)
8881 + self.assertRaises(KeyError, vardb.aux_get, "dev-libs/N-2", ["EAPI"])
8882 + vardb.aux_get("dev-libs/N.moved-2", ["RDEPEND"])[0]
8883
8884 - # EAPI 4 does not allow dots in package names for N -> N.moved
8885 - rdepend = vardb.aux_get("dev-libs/A-1", ["RDEPEND"])[0]
8886 - self.assertTrue("dev-libs/N" in rdepend)
8887 - self.assertTrue("dev-libs/N.moved" not in rdepend)
8888 - rdepend = bindb.aux_get("dev-libs/A-1", ["RDEPEND"])[0]
8889 - self.assertTrue("dev-libs/N" in rdepend)
8890 - self.assertTrue("dev-libs/N.moved" not in rdepend)
8891 - vardb.aux_get("dev-libs/N-1", ["RDEPEND"])[0]
8892 - self.assertRaises(KeyError, vardb.aux_get, "dev-libs/N.moved-1", ["EAPI"])
8893 + # EAPI 4 does not allow dots in package names for N -> N.moved
8894 + rdepend = vardb.aux_get("dev-libs/A-1", ["RDEPEND"])[0]
8895 + self.assertTrue("dev-libs/N" in rdepend)
8896 + self.assertTrue("dev-libs/N.moved" not in rdepend)
8897 + rdepend = bindb.aux_get("dev-libs/A-1", ["RDEPEND"])[0]
8898 + self.assertTrue("dev-libs/N" in rdepend)
8899 + self.assertTrue("dev-libs/N.moved" not in rdepend)
8900 + vardb.aux_get("dev-libs/N-1", ["RDEPEND"])[0]
8901 + self.assertRaises(
8902 + KeyError, vardb.aux_get, "dev-libs/N.moved-1", ["EAPI"]
8903 + )
8904
8905 - # dont_apply_updates
8906 - rdepend = vardb.aux_get("dev-libs/A-2", ["RDEPEND"])[0]
8907 - self.assertTrue("dev-libs/M" in rdepend)
8908 - self.assertTrue("dev-libs/M-moved" not in rdepend)
8909 - rdepend = bindb.aux_get("dev-libs/A-2", ["RDEPEND"])[0]
8910 - self.assertTrue("dev-libs/M" in rdepend)
8911 - self.assertTrue("dev-libs/M-moved" not in rdepend)
8912 + # dont_apply_updates
8913 + rdepend = vardb.aux_get("dev-libs/A-2", ["RDEPEND"])[0]
8914 + self.assertTrue("dev-libs/M" in rdepend)
8915 + self.assertTrue("dev-libs/M-moved" not in rdepend)
8916 + rdepend = bindb.aux_get("dev-libs/A-2", ["RDEPEND"])[0]
8917 + self.assertTrue("dev-libs/M" in rdepend)
8918 + self.assertTrue("dev-libs/M-moved" not in rdepend)
8919
8920 - selected_set.load()
8921 - self.assertTrue("dev-libs/M" not in selected_set)
8922 - self.assertTrue("dev-libs/M-moved" in selected_set)
8923 - self.assertTrue("dev-libs/N" not in selected_set)
8924 - self.assertTrue("dev-libs/N.moved" in selected_set)
8925 + selected_set.load()
8926 + self.assertTrue("dev-libs/M" not in selected_set)
8927 + self.assertTrue("dev-libs/M-moved" in selected_set)
8928 + self.assertTrue("dev-libs/N" not in selected_set)
8929 + self.assertTrue("dev-libs/N.moved" in selected_set)
8930
8931 - finally:
8932 - playground.cleanup()
8933 + finally:
8934 + playground.cleanup()
8935
8936 diff --git a/lib/portage/util/_urlopen.py b/lib/portage/util/_urlopen.py
8937 index 70440c3e1..22f0e08df 100644
8938 --- a/lib/portage/util/_urlopen.py
8939 +++ b/lib/portage/util/_urlopen.py
8940 @@ -26,7 +26,7 @@ def have_pep_476():
8941 return hasattr(__import__("ssl"), "_create_unverified_context")
8942
8943
8944 -def urlopen(url, if_modified_since=None, proxies=None):
8945 +def urlopen(url, if_modified_since=None, headers={}, proxies=None):
8946 parse_result = urllib_parse.urlparse(url)
8947 if parse_result.scheme not in ("http", "https"):
8948 return _urlopen(url)
8949 @@ -45,6 +45,8 @@ def urlopen(url, if_modified_since=None, proxies=None):
8950 password_manager = urllib_request.HTTPPasswordMgrWithDefaultRealm()
8951 request = urllib_request.Request(url)
8952 request.add_header("User-Agent", "Gentoo Portage")
8953 + for key in headers:
8954 + request.add_header(key, headers[key])
8955 if if_modified_since:
8956 request.add_header("If-Modified-Since", _timestamp_to_http(if_modified_since))
8957 if parse_result.username is not None:
8958
8959 diff --git a/lib/portage/versions.py b/lib/portage/versions.py
8960 index fe1ff6ce0..086ada555 100644
8961 --- a/lib/portage/versions.py
8962 +++ b/lib/portage/versions.py
8963 @@ -529,6 +529,22 @@ class _pkg_str(str):
8964 self.__dict__["_stable"] = stable
8965 return stable
8966
8967 + @property
8968 + def binpkg_format(self):
8969 + """
8970 + Returns the BINPKG_FORMAT metadata. A return value of None means
8971 + that the format is unset. If there is no metadata available or the
8972 + BINPKG_FORMAT key is missing from the metadata, then raise
8973 + AttributeError.
8974 +
8975 + @rtype: str or None
8976 + @return: a non-empty BINPKG_FORMAT string, or None
8977 + """
8978 + try:
8979 + return self._metadata["BINPKG_FORMAT"] or None
8980 + except (AttributeError, KeyError):
8981 + raise AttributeError("binpkg_format")
8982 +
8983
8984 def pkgsplit(mypkg, silent=1, eapi=None):
8985 """
8986
8987 diff --git a/man/make.conf.5 b/man/make.conf.5
8988 index 868a2ca50..8afde4a00 100644
8989 --- a/man/make.conf.5
8990 +++ b/man/make.conf.5
8991 @@ -137,6 +137,55 @@ Defaults to "".
8992 BINPKG_COMPRESS_FLAGS="-9"
8993 .fi
8994 .TP
8995 +\fBBINPKG_GPG_SIGNING_BASE_COMMAND\fR = \fI"GPG command and arguments \
8996 +[PORTAGE_CONFIG]"\fR
8997 +The base command will be used for all signing operations.
8998 +Portage will replace \fB[PORTAGE_CONFIG]\fR under different operations.
8999 +Please do not add arguments that can be configured independently.
9000 +.br
9001 +Defaults to "/usr/bin/flock /run/lock/portage-binpkg-gpg.lock /usr/bin/gpg \
9002 +--sign --armor [PORTAGE_CONFIG]".
9003 +.br
9004 +.TP
9005 +\fBBINPKG_GPG_SIGNING_DIGEST=\fR = \fI"GPG supported digest"\fR
9006 +The digest that will be used for signature.
9007 +.br
9008 +Defaults to "SHA512"
9009 +.br
9010 +.TP
9011 +\fBBINPKG_GPG_SIGNING_GPG_HOME\fR = \fI[path]\fR
9012 +The GPG home where the signing private key located.
9013 +.br
9014 +Defaults to "/root/.gnupg"
9015 +.br
9016 +.TP
9017 +\fBBINPKG_GPG_SIGNING_KEY\fR = \fI"GPG key ID"\fR
9018 +GPG key ID used to sign binary packages, must exists in \
9019 +\fBBINPKG_GPG_SIGNING_GPG_HOME\fR.
9020 +.br
9021 +Defaults to ""
9022 +.br
9023 +Example: "0x40DCF18E97150795!"
9024 +.br
9025 +.TP
9026 +\fBBINPKG_GPG_VERIFY_BASE_COMMAND=\fR = \fI"GPG command and arguments"\fR
9027 +The base command will be used for all verify operations.
9028 +Portage will replace \fB[PORTAGE_CONFIG]\fR and \fB[SIGNATURE]\fR under \
9029 +different operations.
9030 +Please do not add arguments that can be configured independently.
9031 +.br
9032 +Defaults to "/usr/bin/gpg --verify --batch --no-tty --no-auto-check-trustdb \
9033 +--status-fd 2 [PORTAGE_CONFIG] [SIGNATURE]"
9034 +.br
9035 +.TP
9036 +\fBBINPKG_GPG_VERIFY_GPG_HOME\fR = \fI[path]\fR
9037 +The GPG home where the trusted keys located. Please make sure the target \
9038 +directory is globally readable, as the user will be dropped to \fBnobody\fR \
9039 +during verification.
9040 +.br
9041 +Defaults to "/etc/portage/gnupg"
9042 +.br
9043 +.TP
9044 .B CBUILD
9045 This variable is passed by the \fIebuild scripts\fR to the \fIconfigure\fR
9046 as \fI\-\-build=${CBUILD}\fR only if it is defined. Do not set this yourself
9047 @@ -286,6 +335,11 @@ strips (or splits) them before installing.
9048
9049 \fBbinpkg\-dostrip\fR must be enabled for \fBinstallsources\fR to work.
9050 .TP
9051 +.B binpkg-ignore-signature
9052 +This will disable GPG signature check for all binary packages. Enable this
9053 +could be dangerous if you get binary packages from remote site or use third
9054 +party packages.
9055 +.TP
9056 .B binpkg\-logs
9057 Keep logs from successful binary package merges. This is relevant only when
9058 \fBPORTAGE_LOGDIR\fR is set.
9059 @@ -318,6 +372,16 @@ It is also possible to remove packages manually, and then run
9060 \(aqemaint \-\-fix binhost' to update the ${PKGDIR}/Packages index.
9061 This feature is enabled by default.
9062 .TP
9063 +.B binpkg-request-signature
9064 +Binary packages are requested to be signed by trusted GPG signature.
9065 +Portage will reject to process any binary package without a valid GPG
9066 +signature. The verify command is defined in
9067 +\fBBINPKG_GPG_VERIFY_COMMAND\fR variable.
9068 +.TP
9069 +.B binpkg-signing
9070 +Binary packages will be signed by given GPG command. The signing command
9071 +is defined in \fBBINPKG_GPG_SIGNING_COMMAND\fR variable.
9072 +.TP
9073 .B buildpkg
9074 Binary packages will be created for all packages that are merged. Also see
9075 \fBquickpkg\fR(1) and \fBemerge\fR(1) \fB\-\-buildpkg\fR and
9076 @@ -451,6 +515,10 @@ for all EAPIs (for obvious reasons).
9077 Force emerges to always try to fetch files from the \fIPORTAGE_BINHOST\fR. See
9078 \fBmake.conf\fR(5) for more information.
9079 .TP
9080 +.B gpg-keepalive
9081 +Run GPG unlock command every 5 mins to avoid the passphrase expired.
9082 +If your GPG is auto unlocked on login, you do not need this.
9083 +.TP
9084 .B icecream
9085 Enable portage support for the icecream package.
9086 .TP
9087 @@ -781,6 +849,18 @@ the \fIebuild scripts\fR. Merging 'mirrorselect' can help. Entries in this
9088 variable that have no protocol and simply start with a '/' path separator may
9089 be used to specify mounted filesystem mirrors.
9090 .TP
9091 +\fBGPG_VERIFY_GROUP_DROP\fR = \fI[group]\fR
9092 +The group name used to drop root privileges during verification.
9093 +.br
9094 +Defaults to "nogroup"
9095 +.br
9096 +.TP
9097 +\fBGPG_VERIFY_USER_DROP\fR = \fI[user]\fR
9098 +The user name used to drop root privileges during verification.
9099 +.br
9100 +Defaults to "nobody"
9101 +.br
9102 +.TP
9103 \fBhttp_proxy ftp_proxy RSYNC_PROXY\fR = \fI[protocol://host:port]\fR
9104 These variables are used by network clients such as \fBwget\fR(1) and
9105 \fBrsync\fR(1). They are only required if you use a
9106
9107 diff --git a/repoman/lib/repoman/tests/.gnupg/openpgp-revocs.d/06B3A311BD775C280D22A9305D90EA06352177F6.rev b/repoman/lib/repoman/tests/.gnupg/openpgp-revocs.d/06B3A311BD775C280D22A9305D90EA06352177F6.rev
9108 new file mode 100644
9109 index 000000000..a6752fd30
9110 --- /dev/null
9111 +++ b/repoman/lib/repoman/tests/.gnupg/openpgp-revocs.d/06B3A311BD775C280D22A9305D90EA06352177F6.rev
9112 @@ -0,0 +1,37 @@
9113 +This is a revocation certificate for the OpenPGP key:
9114 +
9115 +pub rsa4096 2020-07-14 [S]
9116 + 06B3A311BD775C280D22A9305D90EA06352177F6
9117 +uid Gentoo Portage Test Trusted Key (Test Only, Do NOT Trust!!!) (Gentoo Test Key) <test@×××××××.org>
9118 +
9119 +A revocation certificate is a kind of "kill switch" to publicly
9120 +declare that a key shall not anymore be used. It is not possible
9121 +to retract such a revocation certificate once it has been published.
9122 +
9123 +Use it to revoke this key in case of a compromise or loss of
9124 +the secret key. However, if the secret key is still accessible,
9125 +it is better to generate a new revocation certificate and give
9126 +a reason for the revocation. For details see the description of
9127 +of the gpg command "--generate-revocation" in the GnuPG manual.
9128 +
9129 +To avoid an accidental use of this file, a colon has been inserted
9130 +before the 5 dashes below. Remove this colon with a text editor
9131 +before importing and publishing this revocation certificate.
9132 +
9133 +:-----BEGIN PGP PUBLIC KEY BLOCK-----
9134 +Comment: This is a revocation certificate
9135 +
9136 +iQI2BCABCAAgFiEEBrOjEb13XCgNIqkwXZDqBjUhd/YFAl8OFTwCHQAACgkQXZDq
9137 +BjUhd/aXCA/+OgzosMDaDe5DNwkSi2yKdC2X18v8JcaYnXBUR93nXA0LVN7iVWkR
9138 +WEH3NuVspQZ5vK+3AHTKabqZFC/buA5oQOH01Ncd4lQISfOOhFiBn5DIPX31BVT0
9139 +iPmVkcxHAD4031ptP4oat6EFclT13SRchtlnAO04JofeHnzQIw3SozQGzXpAA1g4
9140 +BogQ0HWA88HzuEYYE+e/yzZL4D496X1DTaXksg0Py5c4SS6u5pND6lcUtAGxAwa9
9141 +sJFPs+coeURaRV99CrJfdh4u2OkvINTfrKOS6NFBQq6HVH5mLsRXZlcE4Oo4d+fN
9142 +XoPrTZnRUqpJADUdjHFvO/lr0fArJTS5IQCVBNFeCMlvgmUPeKWJ1r6Uiwe/UHor
9143 +9OP/tK97EqpsaXmHbo0jOUkn5iiUwy784+JBSSu/Q2NxqcBr74aaRdfxvs62dmv7
9144 +droCDQi3ebqTdnlDSaeCIWHyVlSroOhZ+ZETVy193K1X7VXFX3hYKiJ3G8QZwy3e
9145 +AlsVGjIHWfC+K+enIn+uwSUvOWPN3upK8kqMRuXvAOppFCE4sTqNbxUnHHXaqo/r
9146 +s1q6zVsWVILBk97BHlJph2IaqhV7iIgPU97/r4U/BT11VqDFdVSHcXcs4PDNs5vh
9147 +6qttaDiyDqZjwMr+0iDoouHxFpqY8e+3M2gycUgGr2XV6ML0pXE6BqA=
9148 +=nIjC
9149 +-----END PGP PUBLIC KEY BLOCK-----
9150
9151 diff --git a/repoman/lib/repoman/tests/.gnupg/openpgp-revocs.d/8DEDA2CDED49C8809287B89D8812797DDF1DD192.rev b/repoman/lib/repoman/tests/.gnupg/openpgp-revocs.d/8DEDA2CDED49C8809287B89D8812797DDF1DD192.rev
9152 new file mode 100644
9153 index 000000000..456e0aa50
9154 --- /dev/null
9155 +++ b/repoman/lib/repoman/tests/.gnupg/openpgp-revocs.d/8DEDA2CDED49C8809287B89D8812797DDF1DD192.rev
9156 @@ -0,0 +1,37 @@
9157 +This is a revocation certificate for the OpenPGP key:
9158 +
9159 +pub rsa4096 2020-07-14 [S]
9160 + 8DEDA2CDED49C8809287B89D8812797DDF1DD192
9161 +uid Gentoo Portage Test Untrusted Key (Test Only, Do NOT Trust!!!) (Gentoo Test Key) <test@×××××××.org>
9162 +
9163 +A revocation certificate is a kind of "kill switch" to publicly
9164 +declare that a key shall not anymore be used. It is not possible
9165 +to retract such a revocation certificate once it has been published.
9166 +
9167 +Use it to revoke this key in case of a compromise or loss of
9168 +the secret key. However, if the secret key is still accessible,
9169 +it is better to generate a new revocation certificate and give
9170 +a reason for the revocation. For details see the description of
9171 +of the gpg command "--generate-revocation" in the GnuPG manual.
9172 +
9173 +To avoid an accidental use of this file, a colon has been inserted
9174 +before the 5 dashes below. Remove this colon with a text editor
9175 +before importing and publishing this revocation certificate.
9176 +
9177 +:-----BEGIN PGP PUBLIC KEY BLOCK-----
9178 +Comment: This is a revocation certificate
9179 +
9180 +iQI2BCABCAAgFiEEje2ize1JyICSh7idiBJ5fd8d0ZIFAl8OFXUCHQAACgkQiBJ5
9181 +fd8d0ZKdwxAAhmkC0V+OLyOU9PCV6ogD9/3b3nVqNIreoc+gxHTLmEvxiMSItqmq
9182 +DkcW9RJKAduA/HiLZQ8Yzxw+ldC6kuWqYEjNpSM54VDkrgOePi8W1bVDTCoSp7bo
9183 +0JOG4frieqIxA6lhAA2UppH7EPRXoODPLYqooNxWAs3xxVrR6eGAb5l8NXzrymvN
9184 +acFfOZ0s5FgADQskQHWVq6TaJn9DrcZxd+b+plSwPYDXqzTChKQ5jw7uMAPUvDkG
9185 +JUWgoKiKSrK64bslUq8aEDEZQ4uxjyEi6G0vO/wPL/ysGhS7KkPgCZsEfNjWjajb
9186 +jAsdvl1raoHxK/O7llMNr9uRAZtC56pJ//SRDc3kylZrkAo0RNoXQFowT739HWei
9187 +2UkCFDfz488VKKrOI8TzTyUvLFEo14ZAXGg1wdHaGnbYMzxpKjP15alOFo6fKIcS
9188 +Kz1f/Mab4wf4Sg0XAjQ9pnai1/U9ZF3/NSnRtYgJkLCrIEtRLrgSHJsLDPxjCfGV
9189 +jWszAbIk167aA0yKsSmuwkpc5bZqqBaTo904r857fxyt5Les6SOHsV7iNXt7F+am
9190 +03Y6u6m2eROba7M67l115vTyYcw5EZVp5j0nI81PXsC9X2DD1ci5xrNmPyEeupC4
9191 +7y7mcGbUYPJAJHJ0kHG4ZYLnNMl42ZYr1ssEeasDwUsLWgVqvx9RkKI=
9192 +=kVUQ
9193 +-----END PGP PUBLIC KEY BLOCK-----
9194
9195 diff --git a/repoman/lib/repoman/tests/.gnupg/private-keys-v1.d/273B030399E7BEA66A9AD42216DE7CA17BA5D42E.key b/repoman/lib/repoman/tests/.gnupg/private-keys-v1.d/273B030399E7BEA66A9AD42216DE7CA17BA5D42E.key
9196 new file mode 100644
9197 index 000000000..0bd1026ad
9198 Binary files /dev/null and b/repoman/lib/repoman/tests/.gnupg/private-keys-v1.d/273B030399E7BEA66A9AD42216DE7CA17BA5D42E.key differ
9199
9200 diff --git a/repoman/lib/repoman/tests/.gnupg/private-keys-v1.d/C99796FB85B0C3DF03314A11B5850C51167D6282.key b/repoman/lib/repoman/tests/.gnupg/private-keys-v1.d/C99796FB85B0C3DF03314A11B5850C51167D6282.key
9201 new file mode 100644
9202 index 000000000..8e29ef43c
9203 Binary files /dev/null and b/repoman/lib/repoman/tests/.gnupg/private-keys-v1.d/C99796FB85B0C3DF03314A11B5850C51167D6282.key differ
9204
9205 diff --git a/repoman/lib/repoman/tests/.gnupg/pubring.kbx b/repoman/lib/repoman/tests/.gnupg/pubring.kbx
9206 new file mode 100644
9207 index 000000000..f6367f83b
9208 Binary files /dev/null and b/repoman/lib/repoman/tests/.gnupg/pubring.kbx differ
9209
9210 diff --git a/repoman/lib/repoman/tests/.gnupg/trustdb.gpg b/repoman/lib/repoman/tests/.gnupg/trustdb.gpg
9211 new file mode 100644
9212 index 000000000..db5b1023b
9213 Binary files /dev/null and b/repoman/lib/repoman/tests/.gnupg/trustdb.gpg differ
9214
9215 diff --git a/repoman/lib/repoman/tests/runTests.py b/repoman/lib/repoman/tests/runTests.py
9216 index e23ded192..4a081551e 100644
9217 --- a/repoman/lib/repoman/tests/runTests.py
9218 +++ b/repoman/lib/repoman/tests/runTests.py
9219 @@ -10,6 +10,8 @@ import grp
9220 import platform
9221 import pwd
9222 import signal
9223 +import tempfile
9224 +from distutils.dir_util import copy_tree
9225
9226
9227 def debug_signal(signum, frame):
9228 @@ -68,6 +70,14 @@ if insert_bin_path:
9229 path.insert(0, PORTAGE_BIN_PATH)
9230 os.environ["PATH"] = ":".join(path)
9231
9232 +# Copy GPG test keys to temporary directory
9233 +gpg_path = tempfile.mkdtemp(prefix="gpg_")
9234 +
9235 +copy_tree(os.path.join(os.path.dirname(os.path.realpath(__file__)), ".gnupg"), gpg_path)
9236 +
9237 +os.chmod(gpg_path, 0o700)
9238 +os.environ["PORTAGE_GNUPGHOME"] = gpg_path
9239 +
9240 if __name__ == "__main__":
9241 try:
9242 sys.exit(tests.main())
9243
9244 diff --git a/repoman/setup.py b/repoman/setup.py
9245 index 17c9c5e8a..8231c8f75 100755
9246 --- a/repoman/setup.py
9247 +++ b/repoman/setup.py
9248 @@ -13,7 +13,7 @@ try:
9249 from setuptools.command.install_scripts import install_scripts
9250 from setuptools.command.sdist import sdist
9251 from setuptools.dep_util import newer
9252 - from setuptools.dir_util import mkpath, remove_tree
9253 + from setuptools.dir_util import mkpath, remove_tree, copy_tree
9254 from setuptools.util import change_root, subst_vars
9255 except ImportError:
9256 from distutils.core import setup, Command
9257 @@ -26,7 +26,7 @@ except ImportError:
9258 from distutils.command.install_scripts import install_scripts
9259 from distutils.command.sdist import sdist
9260 from distutils.dep_util import newer
9261 - from distutils.dir_util import mkpath, remove_tree
9262 + from distutils.dir_util import mkpath, remove_tree, copy_tree
9263 from distutils.util import change_root, subst_vars
9264
9265 import codecs
9266 @@ -425,6 +425,14 @@ class test(Command):
9267
9268 def run(self):
9269 self.run_command("build_tests")
9270 + # copy GPG test keys
9271 + copy_tree(
9272 + os.path.join(
9273 + self.build_lib, "..", "..", "lib", "repoman", "tests", ".gnupg"
9274 + ),
9275 + os.path.join(self.build_lib, "repoman", "tests", ".gnupg"),
9276 + )
9277 + os.chmod(os.path.join(self.build_lib, "repoman", "tests", ".gnupg"), 0o700)
9278 subprocess.check_call(
9279 [
9280 sys.executable,
9281
9282 diff --git a/setup.py b/setup.py
9283 index b525454e0..9e6b4cdb2 100755
9284 --- a/setup.py
9285 +++ b/setup.py
9286 @@ -14,7 +14,7 @@ try:
9287 from setuptools.command.install_scripts import install_scripts
9288 from setuptools.command.sdist import sdist
9289 from setuptools.dep_util import newer
9290 - from setuptools.dir_util import mkpath, remove_tree
9291 + from setuptools.dir_util import mkpath, remove_tree, copy_tree
9292 from setuptools.util import change_root, subst_vars
9293 except ImportError:
9294 from distutils.core import setup, Command, Extension
9295 @@ -28,7 +28,7 @@ except ImportError:
9296 from distutils.command.install_scripts import install_scripts
9297 from distutils.command.sdist import sdist
9298 from distutils.dep_util import newer
9299 - from distutils.dir_util import mkpath, remove_tree
9300 + from distutils.dir_util import mkpath, remove_tree, copy_tree
9301 from distutils.util import change_root, subst_vars
9302
9303 import codecs
9304 @@ -700,6 +700,16 @@ class test(Command):
9305
9306 def run(self):
9307 self.run_command("build_tests")
9308 +
9309 + # copy GPG test keys
9310 + copy_tree(
9311 + os.path.join(
9312 + self.build_lib, "..", "..", "lib", "portage", "tests", ".gnupg"
9313 + ),
9314 + os.path.join(self.build_lib, "portage", "tests", ".gnupg"),
9315 + )
9316 + os.chmod(os.path.join(self.build_lib, "portage", "tests", ".gnupg"), 0o700)
9317 +
9318 subprocess.check_call(
9319 [
9320 sys.executable,