Gentoo Archives: gentoo-portage-dev

From: Zac Medico <zmedico@g.o>
To: gentoo-portage-dev@l.g.o
Cc: Zac Medico <zmedico@g.o>
Subject: [gentoo-portage-dev] [PATCH 1/2] Add FEATURES=binpkg-multi-instance (bug 150031)
Date: Tue, 17 Feb 2015 08:37:37
Message-Id: 1424162233-25071-1-git-send-email-zmedico@gentoo.org
1 FEATURES=binpkg-multi-instance causes an integer build-id to be
2 associated with each binary package instance. Inclusion of the build-id
3 in the file name of the binary package file makes it possible to store
4 an arbitrary number of binary packages built from the same ebuild.
5
6 Having multiple instances is useful for a number of purposes, such as
7 retaining builds that were built with different USE flags or linked
8 against different versions of libraries. The location of any particular
9 package within PKGDIR can be expressed as follows:
10
11 ${PKGDIR}/${CATEGORY}/${PN}/${PF}-${BUILD_ID}.xpak
12
13 The build-id starts at 1 for the first build of a particular ebuild,
14 and is incremented by 1 for each new build. It is possible to share a
15 writable PKGDIR over NFS, and locking ensures that each package added
16 to PKGDIR will have a unique build-id. It is not necessary to migrate
17 an existing PKGDIR to the new layout, since portage is capable of
18 working with a mixed PKGDIR layout, where packages using the old layout
19 are allowed to remain in place.
20
21 The new PKGDIR layout is backward-compatible with binhost clients
22 running older portage, since the file format is identical, the
23 per-package PATH attribute in the 'Packages' index directs them to
24 download the file from the correct URI, and they automatically use
25 BUILD_TIME metadata to select the latest builds.
26
27 There is currently no automated way to prune old builds from PKGDIR,
28 although it is possible to remove packages manually, and then run
29 'emaint --fix binhost' to update the ${PKGDIR}/Packages index.
30
31 It is not necessary to migrate an existing PKGDIR to the new layout,
32 since portage is capable of working with a mixed PKGDIR layout, where
33 packages using the old layout are allowed to remain in-place.
34
35 There is currently no automated way to prune old builds from PKGDIR,
36 although it is possible to remove packages manually, and then run
37 'emaint --fix binhost' update the ${PKGDIR}/Packages index. Support for
38 FEATURES=binpkg-multi-instance is planned for eclean-pkg.
39
40 X-Gentoo-Bug: 150031
41 X-Gentoo-Bug-URL: https://bugs.gentoo.org/show_bug.cgi?id=150031
42 ---
43 bin/quickpkg | 1 -
44 man/make.conf.5 | 27 +
45 pym/_emerge/Binpkg.py | 33 +-
46 pym/_emerge/BinpkgFetcher.py | 13 +-
47 pym/_emerge/BinpkgVerifier.py | 6 +-
48 pym/_emerge/EbuildBinpkg.py | 9 +-
49 pym/_emerge/EbuildBuild.py | 36 +-
50 pym/_emerge/Package.py | 67 +-
51 pym/_emerge/Scheduler.py | 6 +-
52 pym/_emerge/clear_caches.py | 1 -
53 pym/_emerge/resolver/output.py | 21 +-
54 pym/portage/const.py | 2 +
55 pym/portage/dbapi/__init__.py | 10 +-
56 pym/portage/dbapi/bintree.py | 842 +++++++++++----------
57 pym/portage/dbapi/vartree.py | 8 +-
58 pym/portage/dbapi/virtual.py | 113 ++-
59 pym/portage/emaint/modules/binhost/binhost.py | 47 +-
60 pym/portage/package/ebuild/config.py | 3 +-
61 pym/portage/tests/resolver/ResolverPlayground.py | 26 +-
62 .../resolver/binpkg_multi_instance/__init__.py | 2 +
63 .../resolver/binpkg_multi_instance/__test__.py | 2 +
64 .../binpkg_multi_instance/test_rebuilt_binaries.py | 101 +++
65 pym/portage/versions.py | 48 +-
66 23 files changed, 932 insertions(+), 492 deletions(-)
67 create mode 100644 pym/portage/tests/resolver/binpkg_multi_instance/__init__.py
68 create mode 100644 pym/portage/tests/resolver/binpkg_multi_instance/__test__.py
69 create mode 100644 pym/portage/tests/resolver/binpkg_multi_instance/test_rebuilt_binaries.py
70
71 diff --git a/bin/quickpkg b/bin/quickpkg
72 index 2c69a69..8b71c3e 100755
73 --- a/bin/quickpkg
74 +++ b/bin/quickpkg
75 @@ -63,7 +63,6 @@ def quickpkg_atom(options, infos, arg, eout):
76 pkgs_for_arg = 0
77 for cpv in matches:
78 excluded_config_files = []
79 - bintree.prevent_collision(cpv)
80 dblnk = vardb._dblink(cpv)
81 have_lock = False
82
83 diff --git a/man/make.conf.5 b/man/make.conf.5
84 index 84b7191..6ead61b 100644
85 --- a/man/make.conf.5
86 +++ b/man/make.conf.5
87 @@ -256,6 +256,33 @@ has a \fB\-\-force\fR option that can be used to force regeneration of digests.
88 Keep logs from successful binary package merges. This is relevant only when
89 \fBPORT_LOGDIR\fR is set.
90 .TP
91 +.B binpkg\-multi\-instance
92 +Enable support for multiple binary package instances per ebuild.
93 +Having multiple instances is useful for a number of purposes, such as
94 +retaining builds that were built with different USE flags or linked
95 +against different versions of libraries. The location of any particular
96 +package within PKGDIR can be expressed as follows:
97 +
98 + ${PKGDIR}/${CATEGORY}/${PN}/${PF}\-${BUILD_ID}.xpak
99 +
100 +The build\-id starts at 1 for the first build of a particular ebuild,
101 +and is incremented by 1 for each new build. It is possible to share a
102 +writable PKGDIR over NFS, and locking ensures that each package added
103 +to PKGDIR will have a unique build\-id. It is not necessary to migrate
104 +an existing PKGDIR to the new layout, since portage is capable of
105 +working with a mixed PKGDIR layout, where packages using the old layout
106 +are allowed to remain in place.
107 +
108 +The new PKGDIR layout is backward\-compatible with binhost clients
109 +running older portage, since the file format is identical, the
110 +per\-package PATH attribute in the 'Packages' index directs them to
111 +download the file from the correct URI, and they automatically use
112 +BUILD_TIME metadata to select the latest builds.
113 +
114 +There is currently no automated way to prune old builds from PKGDIR,
115 +although it is possible to remove packages manually, and then run
116 +\(aqemaint \-\-fix binhost' to update the ${PKGDIR}/Packages index.
117 +.TP
118 .B buildpkg
119 Binary packages will be created for all packages that are merged. Also see
120 \fBquickpkg\fR(1) and \fBemerge\fR(1) \fB\-\-buildpkg\fR and
121 diff --git a/pym/_emerge/Binpkg.py b/pym/_emerge/Binpkg.py
122 index ded6dfd..7b7ae17 100644
123 --- a/pym/_emerge/Binpkg.py
124 +++ b/pym/_emerge/Binpkg.py
125 @@ -121,16 +121,11 @@ class Binpkg(CompositeTask):
126 fetcher = BinpkgFetcher(background=self.background,
127 logfile=self.settings.get("PORTAGE_LOG_FILE"), pkg=self.pkg,
128 pretend=self.opts.pretend, scheduler=self.scheduler)
129 - pkg_path = fetcher.pkg_path
130 - self._pkg_path = pkg_path
131 - # This gives bashrc users an opportunity to do various things
132 - # such as remove binary packages after they're installed.
133 - self.settings["PORTAGE_BINPKG_FILE"] = pkg_path
134
135 if self.opts.getbinpkg and self._bintree.isremote(pkg.cpv):
136 -
137 msg = " --- (%s of %s) Fetching Binary (%s::%s)" %\
138 - (pkg_count.curval, pkg_count.maxval, pkg.cpv, pkg_path)
139 + (pkg_count.curval, pkg_count.maxval, pkg.cpv,
140 + fetcher.pkg_path)
141 short_msg = "emerge: (%s of %s) %s Fetch" % \
142 (pkg_count.curval, pkg_count.maxval, pkg.cpv)
143 self.logger.log(msg, short_msg=short_msg)
144 @@ -149,7 +144,7 @@ class Binpkg(CompositeTask):
145 # The fetcher only has a returncode when
146 # --getbinpkg is enabled.
147 if fetcher.returncode is not None:
148 - self._fetched_pkg = True
149 + self._fetched_pkg = fetcher.pkg_path
150 if self._default_exit(fetcher) != os.EX_OK:
151 self._unlock_builddir()
152 self.wait()
153 @@ -163,9 +158,15 @@ class Binpkg(CompositeTask):
154
155 verifier = None
156 if self._verify:
157 + if self._fetched_pkg:
158 + path = self._fetched_pkg
159 + else:
160 + path = self.pkg.root_config.trees["bintree"].getname(
161 + self.pkg.cpv)
162 logfile = self.settings.get("PORTAGE_LOG_FILE")
163 verifier = BinpkgVerifier(background=self.background,
164 - logfile=logfile, pkg=self.pkg, scheduler=self.scheduler)
165 + logfile=logfile, pkg=self.pkg, scheduler=self.scheduler,
166 + _pkg_path=path)
167 self._start_task(verifier, self._verifier_exit)
168 return
169
170 @@ -181,10 +182,20 @@ class Binpkg(CompositeTask):
171 logger = self.logger
172 pkg = self.pkg
173 pkg_count = self.pkg_count
174 - pkg_path = self._pkg_path
175
176 if self._fetched_pkg:
177 - self._bintree.inject(pkg.cpv, filename=pkg_path)
178 + pkg_path = self._bintree.getname(
179 + self._bintree.inject(pkg.cpv,
180 + filename=self._fetched_pkg),
181 + allocate_new=False)
182 + else:
183 + pkg_path = self.pkg.root_config.trees["bintree"].getname(
184 + self.pkg.cpv)
185 +
186 + # This gives bashrc users an opportunity to do various things
187 + # such as remove binary packages after they're installed.
188 + self.settings["PORTAGE_BINPKG_FILE"] = pkg_path
189 + self._pkg_path = pkg_path
190
191 logfile = self.settings.get("PORTAGE_LOG_FILE")
192 if logfile is not None and os.path.isfile(logfile):
193 diff --git a/pym/_emerge/BinpkgFetcher.py b/pym/_emerge/BinpkgFetcher.py
194 index 543881e..a7f2d44 100644
195 --- a/pym/_emerge/BinpkgFetcher.py
196 +++ b/pym/_emerge/BinpkgFetcher.py
197 @@ -24,7 +24,8 @@ class BinpkgFetcher(SpawnProcess):
198 def __init__(self, **kwargs):
199 SpawnProcess.__init__(self, **kwargs)
200 pkg = self.pkg
201 - self.pkg_path = pkg.root_config.trees["bintree"].getname(pkg.cpv)
202 + self.pkg_path = pkg.root_config.trees["bintree"].getname(
203 + pkg.cpv) + ".partial"
204
205 def _start(self):
206
207 @@ -51,10 +52,12 @@ class BinpkgFetcher(SpawnProcess):
208 # urljoin doesn't work correctly with
209 # unrecognized protocols like sftp
210 if bintree._remote_has_index:
211 - rel_uri = bintree._remotepkgs[pkg.cpv].get("PATH")
212 + instance_key = bintree.dbapi._instance_key(pkg.cpv)
213 + rel_uri = bintree._remotepkgs[instance_key].get("PATH")
214 if not rel_uri:
215 rel_uri = pkg.cpv + ".tbz2"
216 - remote_base_uri = bintree._remotepkgs[pkg.cpv]["BASE_URI"]
217 + remote_base_uri = bintree._remotepkgs[
218 + instance_key]["BASE_URI"]
219 uri = remote_base_uri.rstrip("/") + "/" + rel_uri.lstrip("/")
220 else:
221 uri = settings["PORTAGE_BINHOST"].rstrip("/") + \
222 @@ -128,7 +131,9 @@ class BinpkgFetcher(SpawnProcess):
223 # the fetcher didn't already do it automatically.
224 bintree = self.pkg.root_config.trees["bintree"]
225 if bintree._remote_has_index:
226 - remote_mtime = bintree._remotepkgs[self.pkg.cpv].get("MTIME")
227 + remote_mtime = bintree._remotepkgs[
228 + bintree.dbapi._instance_key(
229 + self.pkg.cpv)].get("MTIME")
230 if remote_mtime is not None:
231 try:
232 remote_mtime = long(remote_mtime)
233 diff --git a/pym/_emerge/BinpkgVerifier.py b/pym/_emerge/BinpkgVerifier.py
234 index 2c69792..7a6d15e 100644
235 --- a/pym/_emerge/BinpkgVerifier.py
236 +++ b/pym/_emerge/BinpkgVerifier.py
237 @@ -33,7 +33,6 @@ class BinpkgVerifier(CompositeTask):
238 digests = _apply_hash_filter(digests, hash_filter)
239
240 self._digests = digests
241 - self._pkg_path = bintree.getname(self.pkg.cpv)
242
243 try:
244 size = os.stat(self._pkg_path).st_size
245 @@ -90,8 +89,11 @@ class BinpkgVerifier(CompositeTask):
246 if portage.output.havecolor:
247 portage.output.havecolor = not self.background
248
249 + path = self._pkg_path
250 + if path.endswith(".partial"):
251 + path = path[:-len(".partial")]
252 eout = EOutput()
253 - eout.ebegin("%s %s ;-)" % (os.path.basename(self._pkg_path),
254 + eout.ebegin("%s %s ;-)" % (os.path.basename(path),
255 " ".join(sorted(self._digests))))
256 eout.eend(0)
257
258 diff --git a/pym/_emerge/EbuildBinpkg.py b/pym/_emerge/EbuildBinpkg.py
259 index 34a6aef..6e098eb 100644
260 --- a/pym/_emerge/EbuildBinpkg.py
261 +++ b/pym/_emerge/EbuildBinpkg.py
262 @@ -10,13 +10,12 @@ class EbuildBinpkg(CompositeTask):
263 This assumes that src_install() has successfully completed.
264 """
265 __slots__ = ('pkg', 'settings') + \
266 - ('_binpkg_tmpfile',)
267 + ('_binpkg_tmpfile', '_binpkg_info')
268
269 def _start(self):
270 pkg = self.pkg
271 root_config = pkg.root_config
272 bintree = root_config.trees["bintree"]
273 - bintree.prevent_collision(pkg.cpv)
274 binpkg_tmpfile = os.path.join(bintree.pkgdir,
275 pkg.cpv + ".tbz2." + str(os.getpid()))
276 bintree._ensure_dir(os.path.dirname(binpkg_tmpfile))
277 @@ -43,8 +42,12 @@ class EbuildBinpkg(CompositeTask):
278
279 pkg = self.pkg
280 bintree = pkg.root_config.trees["bintree"]
281 - bintree.inject(pkg.cpv, filename=self._binpkg_tmpfile)
282 + self._binpkg_info = bintree.inject(pkg.cpv,
283 + filename=self._binpkg_tmpfile)
284
285 self._current_task = None
286 self.returncode = os.EX_OK
287 self.wait()
288 +
289 + def get_binpkg_info(self):
290 + return self._binpkg_info
291 diff --git a/pym/_emerge/EbuildBuild.py b/pym/_emerge/EbuildBuild.py
292 index b5b1e87..0e98602 100644
293 --- a/pym/_emerge/EbuildBuild.py
294 +++ b/pym/_emerge/EbuildBuild.py
295 @@ -1,6 +1,10 @@
296 # Copyright 1999-2014 Gentoo Foundation
297 # Distributed under the terms of the GNU General Public License v2
298
299 +from __future__ import unicode_literals
300 +
301 +import io
302 +
303 import _emerge.emergelog
304 from _emerge.EbuildExecuter import EbuildExecuter
305 from _emerge.EbuildPhase import EbuildPhase
306 @@ -15,7 +19,7 @@ from _emerge.TaskSequence import TaskSequence
307
308 from portage.util import writemsg
309 import portage
310 -from portage import os
311 +from portage import _encodings, _unicode_decode, _unicode_encode, os
312 from portage.output import colorize
313 from portage.package.ebuild.digestcheck import digestcheck
314 from portage.package.ebuild.digestgen import digestgen
315 @@ -317,9 +321,13 @@ class EbuildBuild(CompositeTask):
316 phase="rpm", scheduler=self.scheduler,
317 settings=self.settings))
318 else:
319 - binpkg_tasks.add(EbuildBinpkg(background=self.background,
320 + task = EbuildBinpkg(
321 + background=self.background,
322 pkg=self.pkg, scheduler=self.scheduler,
323 - settings=self.settings))
324 + settings=self.settings)
325 + binpkg_tasks.add(task)
326 + task.addExitListener(
327 + self._record_binpkg_info)
328
329 if binpkg_tasks:
330 self._start_task(binpkg_tasks, self._buildpkg_exit)
331 @@ -356,6 +364,28 @@ class EbuildBuild(CompositeTask):
332 self.returncode = packager.returncode
333 self.wait()
334
335 + def _record_binpkg_info(self, task):
336 + if task.returncode != os.EX_OK:
337 + return
338 +
339 + # Save info about the created binary package, so that
340 + # identifying information can be passed to the install
341 + # task, to be recorded in the installed package database.
342 + pkg = task.get_binpkg_info()
343 + infoloc = os.path.join(self.settings["PORTAGE_BUILDDIR"],
344 + "build-info")
345 + info = {
346 + "BINPKGMD5": "%s\n" % pkg._metadata["MD5"],
347 + }
348 + if pkg.build_id is not None:
349 + info["BUILD_ID"] = "%s\n" % pkg.build_id
350 + for k, v in info.items():
351 + with io.open(_unicode_encode(os.path.join(infoloc, k),
352 + encoding=_encodings['fs'], errors='strict'),
353 + mode='w', encoding=_encodings['repo.content'],
354 + errors='strict') as f:
355 + f.write(v)
356 +
357 def _buildpkgonly_success_hook_exit(self, success_hooks):
358 self._default_exit(success_hooks)
359 self.returncode = None
360 diff --git a/pym/_emerge/Package.py b/pym/_emerge/Package.py
361 index e8a13cb..2c1a116 100644
362 --- a/pym/_emerge/Package.py
363 +++ b/pym/_emerge/Package.py
364 @@ -41,12 +41,12 @@ class Package(Task):
365 "_validated_atoms", "_visible")
366
367 metadata_keys = [
368 - "BUILD_TIME", "CHOST", "COUNTER", "DEPEND", "EAPI",
369 - "HDEPEND", "INHERITED", "IUSE", "KEYWORDS",
370 - "LICENSE", "PDEPEND", "PROVIDE", "RDEPEND",
371 - "repository", "PROPERTIES", "RESTRICT", "SLOT", "USE",
372 - "_mtime_", "DEFINED_PHASES", "REQUIRED_USE", "PROVIDES",
373 - "REQUIRES"]
374 + "BUILD_ID", "BUILD_TIME", "CHOST", "COUNTER", "DEFINED_PHASES",
375 + "DEPEND", "EAPI", "HDEPEND", "INHERITED", "IUSE", "KEYWORDS",
376 + "LICENSE", "MD5", "PDEPEND", "PROVIDE", "PROVIDES",
377 + "RDEPEND", "repository", "REQUIRED_USE",
378 + "PROPERTIES", "REQUIRES", "RESTRICT", "SIZE",
379 + "SLOT", "USE", "_mtime_"]
380
381 _dep_keys = ('DEPEND', 'HDEPEND', 'PDEPEND', 'RDEPEND')
382 _buildtime_keys = ('DEPEND', 'HDEPEND')
383 @@ -114,13 +114,14 @@ class Package(Task):
384 return self._metadata["EAPI"]
385
386 @property
387 + def build_id(self):
388 + return self.cpv.build_id
389 +
390 + @property
391 def build_time(self):
392 if not self.built:
393 raise AttributeError('build_time')
394 - try:
395 - return long(self._metadata['BUILD_TIME'])
396 - except (KeyError, ValueError):
397 - return 0
398 + return self.cpv.build_time
399
400 @property
401 def defined_phases(self):
402 @@ -218,6 +219,8 @@ class Package(Task):
403 else:
404 raise TypeError("root_config argument is required")
405
406 + elements = [type_name, root, _unicode(cpv), operation]
407 +
408 # For installed (and binary) packages we don't care for the repo
409 # when it comes to hashing, because there can only be one cpv.
410 # So overwrite the repo_key with type_name.
411 @@ -228,14 +231,22 @@ class Package(Task):
412 raise AssertionError(
413 "Package._gen_hash_key() " + \
414 "called without 'repo_name' argument")
415 - repo_key = repo_name
416 + elements.append(repo_name)
417 + elif type_name == "binary":
418 + # Including a variety of fingerprints in the hash makes
419 + # it possible to simultaneously consider multiple similar
420 + # packages. Note that digests are not included here, since
421 + # they are relatively expensive to compute, and they may
422 + # not necessarily be available.
423 + elements.extend([cpv.build_id, cpv.file_size,
424 + cpv.build_time, cpv.mtime])
425 else:
426 # For installed (and binary) packages we don't care for the repo
427 # when it comes to hashing, because there can only be one cpv.
428 # So overwrite the repo_key with type_name.
429 - repo_key = type_name
430 + elements.append(type_name)
431
432 - return (type_name, root, _unicode(cpv), operation, repo_key)
433 + return tuple(elements)
434
435 def _validate_deps(self):
436 """
437 @@ -509,9 +520,15 @@ class Package(Task):
438 else:
439 cpv_color = "PKG_NOMERGE"
440
441 + build_id_str = ""
442 + if isinstance(self.cpv.build_id, long) and self.cpv.build_id > 0:
443 + build_id_str = "-%s" % self.cpv.build_id
444 +
445 s = "(%s, %s" \
446 - % (portage.output.colorize(cpv_color, self.cpv + _slot_separator + \
447 - self.slot + "/" + self.sub_slot + _repo_separator + self.repo) , self.type_name)
448 + % (portage.output.colorize(cpv_color, self.cpv +
449 + build_id_str + _slot_separator + self.slot + "/" +
450 + self.sub_slot + _repo_separator + self.repo),
451 + self.type_name)
452
453 if self.type_name == "installed":
454 if self.root_config.settings['ROOT'] != "/":
455 @@ -755,29 +772,41 @@ class Package(Task):
456 def __lt__(self, other):
457 if other.cp != self.cp:
458 return self.cp < other.cp
459 - if portage.vercmp(self.version, other.version) < 0:
460 + result = portage.vercmp(self.version, other.version)
461 + if result < 0:
462 return True
463 + if result == 0 and self.built and other.built:
464 + return self.build_time < other.build_time
465 return False
466
467 def __le__(self, other):
468 if other.cp != self.cp:
469 return self.cp <= other.cp
470 - if portage.vercmp(self.version, other.version) <= 0:
471 + result = portage.vercmp(self.version, other.version)
472 + if result <= 0:
473 return True
474 + if result == 0 and self.built and other.built:
475 + return self.build_time <= other.build_time
476 return False
477
478 def __gt__(self, other):
479 if other.cp != self.cp:
480 return self.cp > other.cp
481 - if portage.vercmp(self.version, other.version) > 0:
482 + result = portage.vercmp(self.version, other.version)
483 + if result > 0:
484 return True
485 + if result == 0 and self.built and other.built:
486 + return self.build_time > other.build_time
487 return False
488
489 def __ge__(self, other):
490 if other.cp != self.cp:
491 return self.cp >= other.cp
492 - if portage.vercmp(self.version, other.version) >= 0:
493 + result = portage.vercmp(self.version, other.version)
494 + if result >= 0:
495 return True
496 + if result == 0 and self.built and other.built:
497 + return self.build_time >= other.build_time
498 return False
499
500 def with_use(self, use):
501 diff --git a/pym/_emerge/Scheduler.py b/pym/_emerge/Scheduler.py
502 index d6db311..dae7944 100644
503 --- a/pym/_emerge/Scheduler.py
504 +++ b/pym/_emerge/Scheduler.py
505 @@ -862,8 +862,12 @@ class Scheduler(PollScheduler):
506 continue
507 fetched = fetcher.pkg_path
508
509 + if fetched is False:
510 + filename = bintree.getname(x.cpv)
511 + else:
512 + filename = fetched
513 verifier = BinpkgVerifier(pkg=x,
514 - scheduler=sched_iface)
515 + scheduler=sched_iface, _pkg_path=filename)
516 current_task = verifier
517 verifier.start()
518 if verifier.wait() != os.EX_OK:
519 diff --git a/pym/_emerge/clear_caches.py b/pym/_emerge/clear_caches.py
520 index 513df62..cb0db10 100644
521 --- a/pym/_emerge/clear_caches.py
522 +++ b/pym/_emerge/clear_caches.py
523 @@ -7,7 +7,6 @@ def clear_caches(trees):
524 for d in trees.values():
525 d["porttree"].dbapi.melt()
526 d["porttree"].dbapi._aux_cache.clear()
527 - d["bintree"].dbapi._aux_cache.clear()
528 d["bintree"].dbapi._clear_cache()
529 if d["vartree"].dbapi._linkmap is None:
530 # preserve-libs is entirely disabled
531 diff --git a/pym/_emerge/resolver/output.py b/pym/_emerge/resolver/output.py
532 index 7df0302..400617d 100644
533 --- a/pym/_emerge/resolver/output.py
534 +++ b/pym/_emerge/resolver/output.py
535 @@ -424,6 +424,18 @@ class Display(object):
536 pkg_str += _repo_separator + pkg.repo
537 return pkg_str
538
539 + def _append_build_id(self, pkg_str, pkg, pkg_info):
540 + """Potentially appends repository to package string.
541 +
542 + @param pkg_str: string
543 + @param pkg: _emerge.Package.Package instance
544 + @param pkg_info: dictionary
545 + @rtype string
546 + """
547 + if pkg.type_name == "binary" and pkg.cpv.build_id is not None:
548 + pkg_str += "-%s" % pkg.cpv.build_id
549 + return pkg_str
550 +
551 def _set_non_root_columns(self, pkg, pkg_info):
552 """sets the indent level and formats the output
553
554 @@ -431,7 +443,7 @@ class Display(object):
555 @param pkg_info: dictionary
556 @rtype string
557 """
558 - ver_str = pkg_info.ver
559 + ver_str = self._append_build_id(pkg_info.ver, pkg, pkg_info)
560 if self.conf.verbosity == 3:
561 ver_str = self._append_slot(ver_str, pkg, pkg_info)
562 ver_str = self._append_repository(ver_str, pkg, pkg_info)
563 @@ -470,7 +482,7 @@ class Display(object):
564 @rtype string
565 Modifies self.verboseadd
566 """
567 - ver_str = pkg_info.ver
568 + ver_str = self._append_build_id(pkg_info.ver, pkg, pkg_info)
569 if self.conf.verbosity == 3:
570 ver_str = self._append_slot(ver_str, pkg, pkg_info)
571 ver_str = self._append_repository(ver_str, pkg, pkg_info)
572 @@ -507,7 +519,7 @@ class Display(object):
573 @param pkg_info: dictionary
574 @rtype the updated addl
575 """
576 - pkg_str = pkg.cpv
577 + pkg_str = self._append_build_id(pkg.cpv, pkg, pkg_info)
578 if self.conf.verbosity == 3:
579 pkg_str = self._append_slot(pkg_str, pkg, pkg_info)
580 pkg_str = self._append_repository(pkg_str, pkg, pkg_info)
581 @@ -868,7 +880,8 @@ class Display(object):
582 if self.conf.columns:
583 myprint = self._set_non_root_columns(pkg, pkg_info)
584 else:
585 - pkg_str = pkg.cpv
586 + pkg_str = self._append_build_id(
587 + pkg.cpv, pkg, pkg_info)
588 if self.conf.verbosity == 3:
589 pkg_str = self._append_slot(pkg_str, pkg, pkg_info)
590 pkg_str = self._append_repository(pkg_str, pkg, pkg_info)
591 diff --git a/pym/portage/const.py b/pym/portage/const.py
592 index febdb4a..c7ecda2 100644
593 --- a/pym/portage/const.py
594 +++ b/pym/portage/const.py
595 @@ -122,6 +122,7 @@ EBUILD_PHASES = (
596 SUPPORTED_FEATURES = frozenset([
597 "assume-digests",
598 "binpkg-logs",
599 + "binpkg-multi-instance",
600 "buildpkg",
601 "buildsyspkg",
602 "candy",
603 @@ -268,6 +269,7 @@ LIVE_ECLASSES = frozenset([
604 ])
605
606 SUPPORTED_BINPKG_FORMATS = ("tar", "rpm")
607 +SUPPORTED_XPAK_EXTENSIONS = (".tbz2", ".xpak")
608
609 # Time formats used in various places like metadata.chk.
610 TIMESTAMP_FORMAT = "%a, %d %b %Y %H:%M:%S +0000" # to be used with time.gmtime()
611 diff --git a/pym/portage/dbapi/__init__.py b/pym/portage/dbapi/__init__.py
612 index 34dfaa7..044faec 100644
613 --- a/pym/portage/dbapi/__init__.py
614 +++ b/pym/portage/dbapi/__init__.py
615 @@ -31,7 +31,8 @@ class dbapi(object):
616 _use_mutable = False
617 _known_keys = frozenset(x for x in auxdbkeys
618 if not x.startswith("UNUSED_0"))
619 - _pkg_str_aux_keys = ("EAPI", "KEYWORDS", "SLOT", "repository")
620 + _pkg_str_aux_keys = ("BUILD_TIME", "EAPI", "BUILD_ID",
621 + "KEYWORDS", "SLOT", "repository")
622
623 def __init__(self):
624 pass
625 @@ -57,7 +58,12 @@ class dbapi(object):
626
627 @staticmethod
628 def _cmp_cpv(cpv1, cpv2):
629 - return vercmp(cpv1.version, cpv2.version)
630 + result = vercmp(cpv1.version, cpv2.version)
631 + if (result == 0 and cpv1.build_time is not None and
632 + cpv2.build_time is not None):
633 + result = ((cpv1.build_time > cpv2.build_time) -
634 + (cpv1.build_time < cpv2.build_time))
635 + return result
636
637 @staticmethod
638 def _cpv_sort_ascending(cpv_list):
639 diff --git a/pym/portage/dbapi/bintree.py b/pym/portage/dbapi/bintree.py
640 index 583e208..b98b26e 100644
641 --- a/pym/portage/dbapi/bintree.py
642 +++ b/pym/portage/dbapi/bintree.py
643 @@ -17,14 +17,13 @@ portage.proxy.lazyimport.lazyimport(globals(),
644 'portage.update:update_dbentries',
645 'portage.util:atomic_ofstream,ensure_dirs,normalize_path,' + \
646 'writemsg,writemsg_stdout',
647 - 'portage.util.listdir:listdir',
648 'portage.util.path:first_existing',
649 'portage.util._urlopen:urlopen@_urlopen',
650 'portage.versions:best,catpkgsplit,catsplit,_pkg_str',
651 )
652
653 from portage.cache.mappings import slot_dict_class
654 -from portage.const import CACHE_PATH
655 +from portage.const import CACHE_PATH, SUPPORTED_XPAK_EXTENSIONS
656 from portage.dbapi.virtual import fakedbapi
657 from portage.dep import Atom, use_reduce, paren_enclose
658 from portage.exception import AlarmSignal, InvalidData, InvalidPackageName, \
659 @@ -71,18 +70,26 @@ class bindbapi(fakedbapi):
660 _known_keys = frozenset(list(fakedbapi._known_keys) + \
661 ["CHOST", "repository", "USE"])
662 def __init__(self, mybintree=None, **kwargs):
663 - fakedbapi.__init__(self, **kwargs)
664 + # Always enable multi_instance mode for bindbapi indexing. This
665 + # does not affect the local PKGDIR file layout, since that is
666 + # controlled independently by FEATURES=binpkg-multi-instance.
667 + # The multi_instance mode is useful for the following reasons:
668 + # * binary packages with the same cpv from multiple binhosts
669 + # can be considered simultaneously
670 + # * if binpkg-multi-instance is disabled, it's still possible
671 + # to properly access a PKGDIR which has binpkg-multi-instance
672 + # layout (or mixed layout)
673 + fakedbapi.__init__(self, exclusive_slots=False,
674 + multi_instance=True, **kwargs)
675 self.bintree = mybintree
676 self.move_ent = mybintree.move_ent
677 - self.cpvdict={}
678 - self.cpdict={}
679 # Selectively cache metadata in order to optimize dep matching.
680 self._aux_cache_keys = set(
681 - ["BUILD_TIME", "CHOST", "DEPEND", "EAPI",
682 - "HDEPEND", "IUSE", "KEYWORDS",
683 - "LICENSE", "PDEPEND", "PROPERTIES", "PROVIDE",
684 - "RDEPEND", "repository", "RESTRICT", "SLOT", "USE",
685 - "DEFINED_PHASES", "PROVIDES", "REQUIRES"
686 + ["BUILD_ID", "BUILD_TIME", "CHOST", "DEFINED_PHASES",
687 + "DEPEND", "EAPI", "HDEPEND", "IUSE", "KEYWORDS",
688 + "LICENSE", "MD5", "PDEPEND", "PROPERTIES", "PROVIDE",
689 + "PROVIDES", "RDEPEND", "repository", "REQUIRES", "RESTRICT",
690 + "SIZE", "SLOT", "USE", "_mtime_"
691 ])
692 self._aux_cache_slot_dict = slot_dict_class(self._aux_cache_keys)
693 self._aux_cache = {}
694 @@ -109,33 +116,49 @@ class bindbapi(fakedbapi):
695 return fakedbapi.cpv_exists(self, cpv)
696
697 def cpv_inject(self, cpv, **kwargs):
698 - self._aux_cache.pop(cpv, None)
699 - fakedbapi.cpv_inject(self, cpv, **kwargs)
700 + if not self.bintree.populated:
701 + self.bintree.populate()
702 + fakedbapi.cpv_inject(self, cpv,
703 + metadata=cpv._metadata, **kwargs)
704
705 def cpv_remove(self, cpv):
706 - self._aux_cache.pop(cpv, None)
707 + if not self.bintree.populated:
708 + self.bintree.populate()
709 fakedbapi.cpv_remove(self, cpv)
710
711 def aux_get(self, mycpv, wants, myrepo=None):
712 if self.bintree and not self.bintree.populated:
713 self.bintree.populate()
714 - cache_me = False
715 + # Support plain string for backward compatibility with API
716 + # consumers (including portageq, which passes in a cpv from
717 + # a command-line argument).
718 + instance_key = self._instance_key(mycpv,
719 + support_string=True)
720 if not self._known_keys.intersection(
721 wants).difference(self._aux_cache_keys):
722 - aux_cache = self._aux_cache.get(mycpv)
723 + aux_cache = self.cpvdict[instance_key]
724 if aux_cache is not None:
725 return [aux_cache.get(x, "") for x in wants]
726 - cache_me = True
727 mysplit = mycpv.split("/")
728 mylist = []
729 tbz2name = mysplit[1]+".tbz2"
730 if not self.bintree._remotepkgs or \
731 not self.bintree.isremote(mycpv):
732 - tbz2_path = self.bintree.getname(mycpv)
733 - if not os.path.exists(tbz2_path):
734 + try:
735 + tbz2_path = self.bintree._pkg_paths[instance_key]
736 + except KeyError:
737 + raise KeyError(mycpv)
738 + tbz2_path = os.path.join(self.bintree.pkgdir, tbz2_path)
739 + try:
740 + st = os.lstat(tbz2_path)
741 + except OSError:
742 raise KeyError(mycpv)
743 metadata_bytes = portage.xpak.tbz2(tbz2_path).get_data()
744 def getitem(k):
745 + if k == "_mtime_":
746 + return _unicode(st[stat.ST_MTIME])
747 + elif k == "SIZE":
748 + return _unicode(st.st_size)
749 v = metadata_bytes.get(_unicode_encode(k,
750 encoding=_encodings['repo.content'],
751 errors='backslashreplace'))
752 @@ -144,11 +167,9 @@ class bindbapi(fakedbapi):
753 encoding=_encodings['repo.content'], errors='replace')
754 return v
755 else:
756 - getitem = self.bintree._remotepkgs[mycpv].get
757 + getitem = self.cpvdict[instance_key].get
758 mydata = {}
759 mykeys = wants
760 - if cache_me:
761 - mykeys = self._aux_cache_keys.union(wants)
762 for x in mykeys:
763 myval = getitem(x)
764 # myval is None if the key doesn't exist
765 @@ -159,16 +180,24 @@ class bindbapi(fakedbapi):
766 if not mydata.setdefault('EAPI', '0'):
767 mydata['EAPI'] = '0'
768
769 - if cache_me:
770 - aux_cache = self._aux_cache_slot_dict()
771 - for x in self._aux_cache_keys:
772 - aux_cache[x] = mydata.get(x, '')
773 - self._aux_cache[mycpv] = aux_cache
774 return [mydata.get(x, '') for x in wants]
775
776 def aux_update(self, cpv, values):
777 if not self.bintree.populated:
778 self.bintree.populate()
779 + build_id = None
780 + try:
781 + build_id = cpv.build_id
782 + except AttributeError:
783 + if self.bintree._multi_instance:
784 + # The cpv.build_id attribute is required if we are in
785 + # multi-instance mode, since otherwise we won't know
786 + # which instance to update.
787 + raise
788 + else:
789 + cpv = self._instance_key(cpv, support_string=True)[0]
790 + build_id = cpv.build_id
791 +
792 tbz2path = self.bintree.getname(cpv)
793 if not os.path.exists(tbz2path):
794 raise KeyError(cpv)
795 @@ -187,7 +216,7 @@ class bindbapi(fakedbapi):
796 del mydata[k]
797 mytbz2.recompose_mem(portage.xpak.xpak_mem(mydata))
798 # inject will clear stale caches via cpv_inject.
799 - self.bintree.inject(cpv)
800 + self.bintree.inject(cpv, filename=tbz2path)
801
802 def cp_list(self, *pargs, **kwargs):
803 if not self.bintree.populated:
804 @@ -219,7 +248,7 @@ class bindbapi(fakedbapi):
805 if not self.bintree.isremote(pkg):
806 pass
807 else:
808 - metadata = self.bintree._remotepkgs[pkg]
809 + metadata = self.bintree._remotepkgs[self._instance_key(pkg)]
810 try:
811 size = int(metadata["SIZE"])
812 except KeyError:
813 @@ -232,48 +261,6 @@ class bindbapi(fakedbapi):
814
815 return filesdict
816
817 -def _pkgindex_cpv_map_latest_build(pkgindex):
818 - """
819 - Given a PackageIndex instance, create a dict of cpv -> metadata map.
820 - If multiple packages have identical CPV values, prefer the package
821 - with latest BUILD_TIME value.
822 - @param pkgindex: A PackageIndex instance.
823 - @type pkgindex: PackageIndex
824 - @rtype: dict
825 - @return: a dict containing entry for the give cpv.
826 - """
827 - cpv_map = {}
828 -
829 - for d in pkgindex.packages:
830 - cpv = d["CPV"]
831 -
832 - try:
833 - cpv = _pkg_str(cpv)
834 - except InvalidData:
835 - writemsg(_("!!! Invalid remote binary package: %s\n") % cpv,
836 - noiselevel=-1)
837 - continue
838 -
839 - btime = d.get('BUILD_TIME', '')
840 - try:
841 - btime = int(btime)
842 - except ValueError:
843 - btime = None
844 -
845 - other_d = cpv_map.get(cpv)
846 - if other_d is not None:
847 - other_btime = other_d.get('BUILD_TIME', '')
848 - try:
849 - other_btime = int(other_btime)
850 - except ValueError:
851 - other_btime = None
852 - if other_btime and (not btime or other_btime > btime):
853 - continue
854 -
855 - cpv_map[_pkg_str(cpv)] = d
856 -
857 - return cpv_map
858 -
859 class binarytree(object):
860 "this tree scans for a list of all packages available in PKGDIR"
861 def __init__(self, _unused=DeprecationWarning, pkgdir=None,
862 @@ -300,6 +287,13 @@ class binarytree(object):
863
864 if True:
865 self.pkgdir = normalize_path(pkgdir)
866 + # NOTE: Event if binpkg-multi-instance is disabled, it's
867 + # still possible to access a PKGDIR which uses the
868 + # binpkg-multi-instance layout (or mixed layout).
869 + self._multi_instance = ("binpkg-multi-instance" in
870 + settings.features)
871 + if self._multi_instance:
872 + self._allocate_filename = self._allocate_filename_multi
873 self.dbapi = bindbapi(self, settings=settings)
874 self.update_ents = self.dbapi.update_ents
875 self.move_slot_ent = self.dbapi.move_slot_ent
876 @@ -310,7 +304,6 @@ class binarytree(object):
877 self.invalids = []
878 self.settings = settings
879 self._pkg_paths = {}
880 - self._pkgindex_uri = {}
881 self._populating = False
882 self._all_directory = os.path.isdir(
883 os.path.join(self.pkgdir, "All"))
884 @@ -318,12 +311,14 @@ class binarytree(object):
885 self._pkgindex_hashes = ["MD5","SHA1"]
886 self._pkgindex_file = os.path.join(self.pkgdir, "Packages")
887 self._pkgindex_keys = self.dbapi._aux_cache_keys.copy()
888 - self._pkgindex_keys.update(["CPV", "MTIME", "SIZE"])
889 + self._pkgindex_keys.update(["CPV", "SIZE"])
890 self._pkgindex_aux_keys = \
891 - ["BUILD_TIME", "CHOST", "DEPEND", "DESCRIPTION", "EAPI",
892 - "HDEPEND", "IUSE", "KEYWORDS", "LICENSE", "PDEPEND", "PROPERTIES",
893 - "PROVIDE", "RESTRICT", "RDEPEND", "repository", "SLOT", "USE", "DEFINED_PHASES",
894 - "BASE_URI", "PROVIDES", "REQUIRES"]
895 + ["BASE_URI", "BUILD_ID", "BUILD_TIME", "CHOST",
896 + "DEFINED_PHASES", "DEPEND", "DESCRIPTION", "EAPI",
897 + "HDEPEND", "IUSE", "KEYWORDS", "LICENSE", "PDEPEND",
898 + "PKGINDEX_URI", "PROPERTIES", "PROVIDE", "PROVIDES",
899 + "RDEPEND", "repository", "REQUIRES", "RESTRICT",
900 + "SIZE", "SLOT", "USE"]
901 self._pkgindex_aux_keys = list(self._pkgindex_aux_keys)
902 self._pkgindex_use_evaluated_keys = \
903 ("DEPEND", "HDEPEND", "LICENSE", "RDEPEND",
904 @@ -336,6 +331,7 @@ class binarytree(object):
905 "USE_EXPAND", "USE_EXPAND_HIDDEN", "USE_EXPAND_IMPLICIT",
906 "USE_EXPAND_UNPREFIXED"])
907 self._pkgindex_default_pkg_data = {
908 + "BUILD_ID" : "",
909 "BUILD_TIME" : "",
910 "DEFINED_PHASES" : "",
911 "DEPEND" : "",
912 @@ -363,6 +359,7 @@ class binarytree(object):
913
914 self._pkgindex_translated_keys = (
915 ("DESCRIPTION" , "DESC"),
916 + ("_mtime_" , "MTIME"),
917 ("repository" , "REPO"),
918 )
919
920 @@ -453,103 +450,30 @@ class binarytree(object):
921 mytbz2.recompose_mem(portage.xpak.xpak_mem(mydata))
922
923 self.dbapi.cpv_remove(mycpv)
924 - del self._pkg_paths[mycpv]
925 + del self._pkg_paths[self.dbapi._instance_key(mycpv)]
926 + metadata = self.dbapi._aux_cache_slot_dict()
927 + for k in self.dbapi._aux_cache_keys:
928 + v = mydata.get(_unicode_encode(k))
929 + if v is not None:
930 + v = _unicode_decode(v)
931 + metadata[k] = " ".join(v.split())
932 + mynewcpv = _pkg_str(mynewcpv, metadata=metadata)
933 new_path = self.getname(mynewcpv)
934 - self._pkg_paths[mynewcpv] = os.path.join(
935 + self._pkg_paths[
936 + self.dbapi._instance_key(mynewcpv)] = os.path.join(
937 *new_path.split(os.path.sep)[-2:])
938 if new_path != mytbz2:
939 self._ensure_dir(os.path.dirname(new_path))
940 _movefile(tbz2path, new_path, mysettings=self.settings)
941 - self._remove_symlink(mycpv)
942 - if new_path.split(os.path.sep)[-2] == "All":
943 - self._create_symlink(mynewcpv)
944 self.inject(mynewcpv)
945
946 return moves
947
948 - def _remove_symlink(self, cpv):
949 - """Remove a ${PKGDIR}/${CATEGORY}/${PF}.tbz2 symlink and also remove
950 - the ${PKGDIR}/${CATEGORY} directory if empty. The file will not be
951 - removed if os.path.islink() returns False."""
952 - mycat, mypkg = catsplit(cpv)
953 - mylink = os.path.join(self.pkgdir, mycat, mypkg + ".tbz2")
954 - if os.path.islink(mylink):
955 - """Only remove it if it's really a link so that this method never
956 - removes a real package that was placed here to avoid a collision."""
957 - os.unlink(mylink)
958 - try:
959 - os.rmdir(os.path.join(self.pkgdir, mycat))
960 - except OSError as e:
961 - if e.errno not in (errno.ENOENT,
962 - errno.ENOTEMPTY, errno.EEXIST):
963 - raise
964 - del e
965 -
966 - def _create_symlink(self, cpv):
967 - """Create a ${PKGDIR}/${CATEGORY}/${PF}.tbz2 symlink (and
968 - ${PKGDIR}/${CATEGORY} directory, if necessary). Any file that may
969 - exist in the location of the symlink will first be removed."""
970 - mycat, mypkg = catsplit(cpv)
971 - full_path = os.path.join(self.pkgdir, mycat, mypkg + ".tbz2")
972 - self._ensure_dir(os.path.dirname(full_path))
973 - try:
974 - os.unlink(full_path)
975 - except OSError as e:
976 - if e.errno != errno.ENOENT:
977 - raise
978 - del e
979 - os.symlink(os.path.join("..", "All", mypkg + ".tbz2"), full_path)
980 -
981 def prevent_collision(self, cpv):
982 - """Make sure that the file location ${PKGDIR}/All/${PF}.tbz2 is safe to
983 - use for a given cpv. If a collision will occur with an existing
984 - package from another category, the existing package will be bumped to
985 - ${PKGDIR}/${CATEGORY}/${PF}.tbz2 so that both can coexist."""
986 - if not self._all_directory:
987 - return
988 -
989 - # Copy group permissions for new directories that
990 - # may have been created.
991 - for path in ("All", catsplit(cpv)[0]):
992 - path = os.path.join(self.pkgdir, path)
993 - self._ensure_dir(path)
994 - if not os.access(path, os.W_OK):
995 - raise PermissionDenied("access('%s', W_OK)" % path)
996 -
997 - full_path = self.getname(cpv)
998 - if "All" == full_path.split(os.path.sep)[-2]:
999 - return
1000 - """Move a colliding package if it exists. Code below this point only
1001 - executes in rare cases."""
1002 - mycat, mypkg = catsplit(cpv)
1003 - myfile = mypkg + ".tbz2"
1004 - mypath = os.path.join("All", myfile)
1005 - dest_path = os.path.join(self.pkgdir, mypath)
1006 -
1007 - try:
1008 - st = os.lstat(dest_path)
1009 - except OSError:
1010 - st = None
1011 - else:
1012 - if stat.S_ISLNK(st.st_mode):
1013 - st = None
1014 - try:
1015 - os.unlink(dest_path)
1016 - except OSError:
1017 - if os.path.exists(dest_path):
1018 - raise
1019 -
1020 - if st is not None:
1021 - # For invalid packages, other_cat could be None.
1022 - other_cat = portage.xpak.tbz2(dest_path).getfile(b"CATEGORY")
1023 - if other_cat:
1024 - other_cat = _unicode_decode(other_cat,
1025 - encoding=_encodings['repo.content'], errors='replace')
1026 - other_cat = other_cat.strip()
1027 - other_cpv = other_cat + "/" + mypkg
1028 - self._move_from_all(other_cpv)
1029 - self.inject(other_cpv)
1030 - self._move_to_all(cpv)
1031 + warnings.warn("The "
1032 + "portage.dbapi.bintree.binarytree.prevent_collision "
1033 + "method is deprecated.",
1034 + DeprecationWarning, stacklevel=2)
1035
1036 def _ensure_dir(self, path):
1037 """
1038 @@ -585,37 +509,6 @@ class binarytree(object):
1039 except PortageException:
1040 pass
1041
1042 - def _move_to_all(self, cpv):
1043 - """If the file exists, move it. Whether or not it exists, update state
1044 - for future getname() calls."""
1045 - mycat, mypkg = catsplit(cpv)
1046 - myfile = mypkg + ".tbz2"
1047 - self._pkg_paths[cpv] = os.path.join("All", myfile)
1048 - src_path = os.path.join(self.pkgdir, mycat, myfile)
1049 - try:
1050 - mystat = os.lstat(src_path)
1051 - except OSError as e:
1052 - mystat = None
1053 - if mystat and stat.S_ISREG(mystat.st_mode):
1054 - self._ensure_dir(os.path.join(self.pkgdir, "All"))
1055 - dest_path = os.path.join(self.pkgdir, "All", myfile)
1056 - _movefile(src_path, dest_path, mysettings=self.settings)
1057 - self._create_symlink(cpv)
1058 - self.inject(cpv)
1059 -
1060 - def _move_from_all(self, cpv):
1061 - """Move a package from ${PKGDIR}/All/${PF}.tbz2 to
1062 - ${PKGDIR}/${CATEGORY}/${PF}.tbz2 and update state from getname calls."""
1063 - self._remove_symlink(cpv)
1064 - mycat, mypkg = catsplit(cpv)
1065 - myfile = mypkg + ".tbz2"
1066 - mypath = os.path.join(mycat, myfile)
1067 - dest_path = os.path.join(self.pkgdir, mypath)
1068 - self._ensure_dir(os.path.dirname(dest_path))
1069 - src_path = os.path.join(self.pkgdir, "All", myfile)
1070 - _movefile(src_path, dest_path, mysettings=self.settings)
1071 - self._pkg_paths[cpv] = mypath
1072 -
1073 def populate(self, getbinpkgs=0):
1074 "populates the binarytree"
1075
1076 @@ -643,55 +536,63 @@ class binarytree(object):
1077 # prior to performing package moves since it only wants to
1078 # operate on local packages (getbinpkgs=0).
1079 self._remotepkgs = None
1080 - self.dbapi._clear_cache()
1081 - self.dbapi._aux_cache.clear()
1082 + self.dbapi.clear()
1083 + _instance_key = self.dbapi._instance_key
1084 if True:
1085 pkg_paths = {}
1086 self._pkg_paths = pkg_paths
1087 - dirs = listdir(self.pkgdir, dirsonly=True, EmptyOnError=True)
1088 - if "All" in dirs:
1089 - dirs.remove("All")
1090 - dirs.sort()
1091 - dirs.insert(0, "All")
1092 + dir_files = {}
1093 + for parent, dir_names, file_names in os.walk(self.pkgdir):
1094 + relative_parent = parent[len(self.pkgdir)+1:]
1095 + dir_files[relative_parent] = file_names
1096 +
1097 pkgindex = self._load_pkgindex()
1098 - pf_index = None
1099 if not self._pkgindex_version_supported(pkgindex):
1100 pkgindex = self._new_pkgindex()
1101 header = pkgindex.header
1102 metadata = {}
1103 + basename_index = {}
1104 for d in pkgindex.packages:
1105 - metadata[d["CPV"]] = d
1106 + cpv = _pkg_str(d["CPV"], metadata=d,
1107 + settings=self.settings)
1108 + d["CPV"] = cpv
1109 + metadata[_instance_key(cpv)] = d
1110 + path = d.get("PATH")
1111 + if not path:
1112 + path = cpv + ".tbz2"
1113 + basename = os.path.basename(path)
1114 + basename_index.setdefault(basename, []).append(d)
1115 +
1116 update_pkgindex = False
1117 - for mydir in dirs:
1118 - for myfile in listdir(os.path.join(self.pkgdir, mydir)):
1119 - if not myfile.endswith(".tbz2"):
1120 + for mydir, file_names in dir_files.items():
1121 + try:
1122 + mydir = _unicode_decode(mydir,
1123 + encoding=_encodings["fs"], errors="strict")
1124 + except UnicodeDecodeError:
1125 + continue
1126 + for myfile in file_names:
1127 + try:
1128 + myfile = _unicode_decode(myfile,
1129 + encoding=_encodings["fs"], errors="strict")
1130 + except UnicodeDecodeError:
1131 + continue
1132 + if not myfile.endswith(SUPPORTED_XPAK_EXTENSIONS):
1133 continue
1134 mypath = os.path.join(mydir, myfile)
1135 full_path = os.path.join(self.pkgdir, mypath)
1136 s = os.lstat(full_path)
1137 - if stat.S_ISLNK(s.st_mode):
1138 +
1139 + if not stat.S_ISREG(s.st_mode):
1140 continue
1141
1142 # Validate data from the package index and try to avoid
1143 # reading the xpak if possible.
1144 - if mydir != "All":
1145 - possibilities = None
1146 - d = metadata.get(mydir+"/"+myfile[:-5])
1147 - if d:
1148 - possibilities = [d]
1149 - else:
1150 - if pf_index is None:
1151 - pf_index = {}
1152 - for mycpv in metadata:
1153 - mycat, mypf = catsplit(mycpv)
1154 - pf_index.setdefault(
1155 - mypf, []).append(metadata[mycpv])
1156 - possibilities = pf_index.get(myfile[:-5])
1157 + possibilities = basename_index.get(myfile)
1158 if possibilities:
1159 match = None
1160 for d in possibilities:
1161 try:
1162 - if long(d["MTIME"]) != s[stat.ST_MTIME]:
1163 + if long(d["_mtime_"]) != s[stat.ST_MTIME]:
1164 continue
1165 except (KeyError, ValueError):
1166 continue
1167 @@ -705,15 +606,14 @@ class binarytree(object):
1168 break
1169 if match:
1170 mycpv = match["CPV"]
1171 - if mycpv in pkg_paths:
1172 - # discard duplicates (All/ is preferred)
1173 - continue
1174 - mycpv = _pkg_str(mycpv)
1175 - pkg_paths[mycpv] = mypath
1176 + instance_key = _instance_key(mycpv)
1177 + pkg_paths[instance_key] = mypath
1178 # update the path if the package has been moved
1179 oldpath = d.get("PATH")
1180 if oldpath and oldpath != mypath:
1181 update_pkgindex = True
1182 + # Omit PATH if it is the default path for
1183 + # the current Packages format version.
1184 if mypath != mycpv + ".tbz2":
1185 d["PATH"] = mypath
1186 if not oldpath:
1187 @@ -723,11 +623,6 @@ class binarytree(object):
1188 if oldpath:
1189 update_pkgindex = True
1190 self.dbapi.cpv_inject(mycpv)
1191 - if not self.dbapi._aux_cache_keys.difference(d):
1192 - aux_cache = self.dbapi._aux_cache_slot_dict()
1193 - for k in self.dbapi._aux_cache_keys:
1194 - aux_cache[k] = d[k]
1195 - self.dbapi._aux_cache[mycpv] = aux_cache
1196 continue
1197 if not os.access(full_path, os.R_OK):
1198 writemsg(_("!!! Permission denied to read " \
1199 @@ -735,13 +630,12 @@ class binarytree(object):
1200 noiselevel=-1)
1201 self.invalids.append(myfile[:-5])
1202 continue
1203 - metadata_bytes = portage.xpak.tbz2(full_path).get_data()
1204 - mycat = _unicode_decode(metadata_bytes.get(b"CATEGORY", ""),
1205 - encoding=_encodings['repo.content'], errors='replace')
1206 - mypf = _unicode_decode(metadata_bytes.get(b"PF", ""),
1207 - encoding=_encodings['repo.content'], errors='replace')
1208 - slot = _unicode_decode(metadata_bytes.get(b"SLOT", ""),
1209 - encoding=_encodings['repo.content'], errors='replace')
1210 + pkg_metadata = self._read_metadata(full_path, s,
1211 + keys=chain(self.dbapi._aux_cache_keys,
1212 + ("PF", "CATEGORY")))
1213 + mycat = pkg_metadata.get("CATEGORY", "")
1214 + mypf = pkg_metadata.get("PF", "")
1215 + slot = pkg_metadata.get("SLOT", "")
1216 mypkg = myfile[:-5]
1217 if not mycat or not mypf or not slot:
1218 #old-style or corrupt package
1219 @@ -765,16 +659,51 @@ class binarytree(object):
1220 writemsg("!!! %s\n" % line, noiselevel=-1)
1221 self.invalids.append(mypkg)
1222 continue
1223 - mycat = mycat.strip()
1224 - slot = slot.strip()
1225 - if mycat != mydir and mydir != "All":
1226 +
1227 + multi_instance = False
1228 + invalid_name = False
1229 + build_id = None
1230 + if myfile.endswith(".xpak"):
1231 + multi_instance = True
1232 + build_id = self._parse_build_id(myfile)
1233 + if build_id < 1:
1234 + invalid_name = True
1235 + elif myfile != "%s-%s.xpak" % (
1236 + mypf, build_id):
1237 + invalid_name = True
1238 + else:
1239 + mypkg = mypkg[:-len(str(build_id))-1]
1240 + elif myfile != mypf + ".tbz2":
1241 + invalid_name = True
1242 +
1243 + if invalid_name:
1244 + writemsg(_("\n!!! Binary package name is "
1245 + "invalid: '%s'\n") % full_path,
1246 + noiselevel=-1)
1247 + continue
1248 +
1249 + if pkg_metadata.get("BUILD_ID"):
1250 + try:
1251 + build_id = long(pkg_metadata["BUILD_ID"])
1252 + except ValueError:
1253 + writemsg(_("!!! Binary package has "
1254 + "invalid BUILD_ID: '%s'\n") %
1255 + full_path, noiselevel=-1)
1256 + continue
1257 + else:
1258 + build_id = None
1259 +
1260 + if multi_instance:
1261 + name_split = catpkgsplit("%s/%s" %
1262 + (mycat, mypf))
1263 + if (name_split is None or
1264 + tuple(catsplit(mydir)) != name_split[:2]):
1265 + continue
1266 + elif mycat != mydir and mydir != "All":
1267 continue
1268 if mypkg != mypf.strip():
1269 continue
1270 mycpv = mycat + "/" + mypkg
1271 - if mycpv in pkg_paths:
1272 - # All is first, so it's preferred.
1273 - continue
1274 if not self.dbapi._category_re.match(mycat):
1275 writemsg(_("!!! Binary package has an " \
1276 "unrecognized category: '%s'\n") % full_path,
1277 @@ -784,14 +713,23 @@ class binarytree(object):
1278 (mycpv, self.settings["PORTAGE_CONFIGROOT"]),
1279 noiselevel=-1)
1280 continue
1281 - mycpv = _pkg_str(mycpv)
1282 - pkg_paths[mycpv] = mypath
1283 + if build_id is not None:
1284 + pkg_metadata["BUILD_ID"] = _unicode(build_id)
1285 + pkg_metadata["SIZE"] = _unicode(s.st_size)
1286 + # Discard items used only for validation above.
1287 + pkg_metadata.pop("CATEGORY")
1288 + pkg_metadata.pop("PF")
1289 + mycpv = _pkg_str(mycpv,
1290 + metadata=self.dbapi._aux_cache_slot_dict(
1291 + pkg_metadata))
1292 + pkg_paths[_instance_key(mycpv)] = mypath
1293 self.dbapi.cpv_inject(mycpv)
1294 update_pkgindex = True
1295 - d = metadata.get(mycpv, {})
1296 + d = metadata.get(_instance_key(mycpv),
1297 + pkgindex._pkg_slot_dict())
1298 if d:
1299 try:
1300 - if long(d["MTIME"]) != s[stat.ST_MTIME]:
1301 + if long(d["_mtime_"]) != s[stat.ST_MTIME]:
1302 d.clear()
1303 except (KeyError, ValueError):
1304 d.clear()
1305 @@ -802,36 +740,30 @@ class binarytree(object):
1306 except (KeyError, ValueError):
1307 d.clear()
1308
1309 + for k in self._pkgindex_allowed_pkg_keys:
1310 + v = pkg_metadata.get(k)
1311 + if v is not None:
1312 + d[k] = v
1313 d["CPV"] = mycpv
1314 - d["SLOT"] = slot
1315 - d["MTIME"] = _unicode(s[stat.ST_MTIME])
1316 - d["SIZE"] = _unicode(s.st_size)
1317
1318 - d.update(zip(self._pkgindex_aux_keys,
1319 - self.dbapi.aux_get(mycpv, self._pkgindex_aux_keys)))
1320 try:
1321 self._eval_use_flags(mycpv, d)
1322 except portage.exception.InvalidDependString:
1323 writemsg(_("!!! Invalid binary package: '%s'\n") % \
1324 self.getname(mycpv), noiselevel=-1)
1325 self.dbapi.cpv_remove(mycpv)
1326 - del pkg_paths[mycpv]
1327 + del pkg_paths[_instance_key(mycpv)]
1328
1329 # record location if it's non-default
1330 if mypath != mycpv + ".tbz2":
1331 d["PATH"] = mypath
1332 else:
1333 d.pop("PATH", None)
1334 - metadata[mycpv] = d
1335 - if not self.dbapi._aux_cache_keys.difference(d):
1336 - aux_cache = self.dbapi._aux_cache_slot_dict()
1337 - for k in self.dbapi._aux_cache_keys:
1338 - aux_cache[k] = d[k]
1339 - self.dbapi._aux_cache[mycpv] = aux_cache
1340 + metadata[_instance_key(mycpv)] = d
1341
1342 - for cpv in list(metadata):
1343 - if cpv not in pkg_paths:
1344 - del metadata[cpv]
1345 + for instance_key in list(metadata):
1346 + if instance_key not in pkg_paths:
1347 + del metadata[instance_key]
1348
1349 # Do not bother to write the Packages index if $PKGDIR/All/ exists
1350 # since it will provide no benefit due to the need to read CATEGORY
1351 @@ -1056,45 +988,24 @@ class binarytree(object):
1352 # The current user doesn't have permission to cache the
1353 # file, but that's alright.
1354 if pkgindex:
1355 - # Organize remote package list as a cpv -> metadata map.
1356 - remotepkgs = _pkgindex_cpv_map_latest_build(pkgindex)
1357 remote_base_uri = pkgindex.header.get("URI", base_url)
1358 - for cpv, remote_metadata in remotepkgs.items():
1359 - remote_metadata["BASE_URI"] = remote_base_uri
1360 - self._pkgindex_uri[cpv] = url
1361 - self._remotepkgs.update(remotepkgs)
1362 - self._remote_has_index = True
1363 - for cpv in remotepkgs:
1364 + for d in pkgindex.packages:
1365 + cpv = _pkg_str(d["CPV"], metadata=d,
1366 + settings=self.settings)
1367 + instance_key = _instance_key(cpv)
1368 + # Local package instances override remote instances
1369 + # with the same instance_key.
1370 + if instance_key in metadata:
1371 + continue
1372 +
1373 + d["CPV"] = cpv
1374 + d["BASE_URI"] = remote_base_uri
1375 + d["PKGINDEX_URI"] = url
1376 + self._remotepkgs[instance_key] = d
1377 + metadata[instance_key] = d
1378 self.dbapi.cpv_inject(cpv)
1379 - if True:
1380 - # Remote package instances override local package
1381 - # if they are not identical.
1382 - hash_names = ["SIZE"] + self._pkgindex_hashes
1383 - for cpv, local_metadata in metadata.items():
1384 - remote_metadata = self._remotepkgs.get(cpv)
1385 - if remote_metadata is None:
1386 - continue
1387 - # Use digests to compare identity.
1388 - identical = True
1389 - for hash_name in hash_names:
1390 - local_value = local_metadata.get(hash_name)
1391 - if local_value is None:
1392 - continue
1393 - remote_value = remote_metadata.get(hash_name)
1394 - if remote_value is None:
1395 - continue
1396 - if local_value != remote_value:
1397 - identical = False
1398 - break
1399 - if identical:
1400 - del self._remotepkgs[cpv]
1401 - else:
1402 - # Override the local package in the aux_get cache.
1403 - self.dbapi._aux_cache[cpv] = remote_metadata
1404 - else:
1405 - # Local package instances override remote instances.
1406 - for cpv in metadata:
1407 - self._remotepkgs.pop(cpv, None)
1408 +
1409 + self._remote_has_index = True
1410
1411 self.populated=1
1412
1413 @@ -1106,7 +1017,8 @@ class binarytree(object):
1414 @param filename: File path of the package to inject, or None if it's
1415 already in the location returned by getname()
1416 @type filename: string
1417 - @rtype: None
1418 + @rtype: _pkg_str or None
1419 + @return: A _pkg_str instance on success, or None on failure.
1420 """
1421 mycat, mypkg = catsplit(cpv)
1422 if not self.populated:
1423 @@ -1124,24 +1036,45 @@ class binarytree(object):
1424 writemsg(_("!!! Binary package does not exist: '%s'\n") % full_path,
1425 noiselevel=-1)
1426 return
1427 - mytbz2 = portage.xpak.tbz2(full_path)
1428 - slot = mytbz2.getfile("SLOT")
1429 + metadata = self._read_metadata(full_path, s)
1430 + slot = metadata.get("SLOT")
1431 + try:
1432 + self._eval_use_flags(cpv, metadata)
1433 + except portage.exception.InvalidDependString:
1434 + slot = None
1435 if slot is None:
1436 writemsg(_("!!! Invalid binary package: '%s'\n") % full_path,
1437 noiselevel=-1)
1438 return
1439 - slot = slot.strip()
1440 - self.dbapi.cpv_inject(cpv)
1441 +
1442 + fetched = False
1443 + try:
1444 + build_id = cpv.build_id
1445 + except AttributeError:
1446 + build_id = None
1447 + else:
1448 + instance_key = self.dbapi._instance_key(cpv)
1449 + if instance_key in self.dbapi.cpvdict:
1450 + # This means we've been called by aux_update (or
1451 + # similar). The instance key typically changes (due to
1452 + # file modification), so we need to discard existing
1453 + # instance key references.
1454 + self.dbapi.cpv_remove(cpv)
1455 + self._pkg_paths.pop(instance_key, None)
1456 + if self._remotepkgs is not None:
1457 + fetched = self._remotepkgs.pop(instance_key, None)
1458 +
1459 + cpv = _pkg_str(cpv, metadata=metadata, settings=self.settings)
1460
1461 # Reread the Packages index (in case it's been changed by another
1462 # process) and then updated it, all while holding a lock.
1463 pkgindex_lock = None
1464 - created_symlink = False
1465 try:
1466 pkgindex_lock = lockfile(self._pkgindex_file,
1467 wantnewlockfile=1)
1468 if filename is not None:
1469 - new_filename = self.getname(cpv)
1470 + new_filename = self.getname(cpv,
1471 + allocate_new=(build_id is None))
1472 try:
1473 samefile = os.path.samefile(filename, new_filename)
1474 except OSError:
1475 @@ -1151,54 +1084,31 @@ class binarytree(object):
1476 _movefile(filename, new_filename, mysettings=self.settings)
1477 full_path = new_filename
1478
1479 - self._file_permissions(full_path)
1480 + basename = os.path.basename(full_path)
1481 + pf = catsplit(cpv)[1]
1482 + if (build_id is None and not fetched and
1483 + basename.endswith(".xpak")):
1484 + # Apply the newly assigned BUILD_ID. This is intended
1485 + # to occur only for locally built packages. If the
1486 + # package was fetched, we want to preserve its
1487 + # attributes, so that we can later distinguish that it
1488 + # is identical to its remote counterpart.
1489 + build_id = self._parse_build_id(basename)
1490 + metadata["BUILD_ID"] = _unicode(build_id)
1491 + cpv = _pkg_str(cpv, metadata=metadata,
1492 + settings=self.settings)
1493 + binpkg = portage.xpak.tbz2(full_path)
1494 + binary_data = binpkg.get_data()
1495 + binary_data[b"BUILD_ID"] = _unicode_encode(
1496 + metadata["BUILD_ID"])
1497 + binpkg.recompose_mem(portage.xpak.xpak_mem(binary_data))
1498
1499 - if self._all_directory and \
1500 - self.getname(cpv).split(os.path.sep)[-2] == "All":
1501 - self._create_symlink(cpv)
1502 - created_symlink = True
1503 + self._file_permissions(full_path)
1504 pkgindex = self._load_pkgindex()
1505 -
1506 if not self._pkgindex_version_supported(pkgindex):
1507 pkgindex = self._new_pkgindex()
1508
1509 - # Discard remote metadata to ensure that _pkgindex_entry
1510 - # gets the local metadata. This also updates state for future
1511 - # isremote calls.
1512 - if self._remotepkgs is not None:
1513 - self._remotepkgs.pop(cpv, None)
1514 -
1515 - # Discard cached metadata to ensure that _pkgindex_entry
1516 - # doesn't return stale metadata.
1517 - self.dbapi._aux_cache.pop(cpv, None)
1518 -
1519 - try:
1520 - d = self._pkgindex_entry(cpv)
1521 - except portage.exception.InvalidDependString:
1522 - writemsg(_("!!! Invalid binary package: '%s'\n") % \
1523 - self.getname(cpv), noiselevel=-1)
1524 - self.dbapi.cpv_remove(cpv)
1525 - del self._pkg_paths[cpv]
1526 - return
1527 -
1528 - # If found, remove package(s) with duplicate path.
1529 - path = d.get("PATH", "")
1530 - for i in range(len(pkgindex.packages) - 1, -1, -1):
1531 - d2 = pkgindex.packages[i]
1532 - if path and path == d2.get("PATH"):
1533 - # Handle path collisions in $PKGDIR/All
1534 - # when CPV is not identical.
1535 - del pkgindex.packages[i]
1536 - elif cpv == d2.get("CPV"):
1537 - if path == d2.get("PATH", ""):
1538 - del pkgindex.packages[i]
1539 - elif created_symlink and not d2.get("PATH", ""):
1540 - # Delete entry for the package that was just
1541 - # overwritten by a symlink to this package.
1542 - del pkgindex.packages[i]
1543 -
1544 - pkgindex.packages.append(d)
1545 -
1546 + d = self._inject_file(pkgindex, cpv, full_path)
1547 self._update_pkgindex_header(pkgindex.header)
1548 self._pkgindex_write(pkgindex)
1549
1550 @@ -1206,6 +1116,72 @@ class binarytree(object):
1551 if pkgindex_lock:
1552 unlockfile(pkgindex_lock)
1553
1554 + # This is used to record BINPKGMD5 in the installed package
1555 + # database, for a package that has just been built.
1556 + cpv._metadata["MD5"] = d["MD5"]
1557 +
1558 + return cpv
1559 +
1560 + def _read_metadata(self, filename, st, keys=None):
1561 + if keys is None:
1562 + keys = self.dbapi._aux_cache_keys
1563 + metadata = self.dbapi._aux_cache_slot_dict()
1564 + else:
1565 + metadata = {}
1566 + binary_metadata = portage.xpak.tbz2(filename).get_data()
1567 + for k in keys:
1568 + if k == "_mtime_":
1569 + metadata[k] = _unicode(st[stat.ST_MTIME])
1570 + elif k == "SIZE":
1571 + metadata[k] = _unicode(st.st_size)
1572 + else:
1573 + v = binary_metadata.get(_unicode_encode(k))
1574 + if v is not None:
1575 + v = _unicode_decode(v)
1576 + metadata[k] = " ".join(v.split())
1577 + return metadata
1578 +
1579 + def _inject_file(self, pkgindex, cpv, filename):
1580 + """
1581 + Add a package to internal data structures, and add an
1582 + entry to the given pkgindex.
1583 + @param pkgindex: The PackageIndex instance to which an entry
1584 + will be added.
1585 + @type pkgindex: PackageIndex
1586 + @param cpv: A _pkg_str instance corresponding to the package
1587 + being injected.
1588 + @type cpv: _pkg_str
1589 + @param filename: Absolute file path of the package to inject.
1590 + @type filename: string
1591 + @rtype: dict
1592 + @return: A dict corresponding to the new entry which has been
1593 + added to pkgindex. This may be used to access the checksums
1594 + which have just been generated.
1595 + """
1596 + # Update state for future isremote calls.
1597 + instance_key = self.dbapi._instance_key(cpv)
1598 + if self._remotepkgs is not None:
1599 + self._remotepkgs.pop(instance_key, None)
1600 +
1601 + self.dbapi.cpv_inject(cpv)
1602 + self._pkg_paths[instance_key] = filename[len(self.pkgdir)+1:]
1603 + d = self._pkgindex_entry(cpv)
1604 +
1605 + # If found, remove package(s) with duplicate path.
1606 + path = d.get("PATH", "")
1607 + for i in range(len(pkgindex.packages) - 1, -1, -1):
1608 + d2 = pkgindex.packages[i]
1609 + if path and path == d2.get("PATH"):
1610 + # Handle path collisions in $PKGDIR/All
1611 + # when CPV is not identical.
1612 + del pkgindex.packages[i]
1613 + elif cpv == d2.get("CPV"):
1614 + if path == d2.get("PATH", ""):
1615 + del pkgindex.packages[i]
1616 +
1617 + pkgindex.packages.append(d)
1618 + return d
1619 +
1620 def _pkgindex_write(self, pkgindex):
1621 contents = codecs.getwriter(_encodings['repo.content'])(io.BytesIO())
1622 pkgindex.write(contents)
1623 @@ -1231,7 +1207,7 @@ class binarytree(object):
1624
1625 def _pkgindex_entry(self, cpv):
1626 """
1627 - Performs checksums and evaluates USE flag conditionals.
1628 + Performs checksums, and gets size and mtime via lstat.
1629 Raises InvalidDependString if necessary.
1630 @rtype: dict
1631 @return: a dict containing entry for the give cpv.
1632 @@ -1239,23 +1215,20 @@ class binarytree(object):
1633
1634 pkg_path = self.getname(cpv)
1635
1636 - d = dict(zip(self._pkgindex_aux_keys,
1637 - self.dbapi.aux_get(cpv, self._pkgindex_aux_keys)))
1638 -
1639 + d = dict(cpv._metadata.items())
1640 d.update(perform_multiple_checksums(
1641 pkg_path, hashes=self._pkgindex_hashes))
1642
1643 d["CPV"] = cpv
1644 - st = os.stat(pkg_path)
1645 - d["MTIME"] = _unicode(st[stat.ST_MTIME])
1646 + st = os.lstat(pkg_path)
1647 + d["_mtime_"] = _unicode(st[stat.ST_MTIME])
1648 d["SIZE"] = _unicode(st.st_size)
1649
1650 - rel_path = self._pkg_paths[cpv]
1651 + rel_path = pkg_path[len(self.pkgdir)+1:]
1652 # record location if it's non-default
1653 if rel_path != cpv + ".tbz2":
1654 d["PATH"] = rel_path
1655
1656 - self._eval_use_flags(cpv, d)
1657 return d
1658
1659 def _new_pkgindex(self):
1660 @@ -1309,15 +1282,17 @@ class binarytree(object):
1661 return False
1662
1663 def _eval_use_flags(self, cpv, metadata):
1664 - use = frozenset(metadata["USE"].split())
1665 + use = frozenset(metadata.get("USE", "").split())
1666 for k in self._pkgindex_use_evaluated_keys:
1667 if k.endswith('DEPEND'):
1668 token_class = Atom
1669 else:
1670 token_class = None
1671
1672 + deps = metadata.get(k)
1673 + if deps is None:
1674 + continue
1675 try:
1676 - deps = metadata[k]
1677 deps = use_reduce(deps, uselist=use, token_class=token_class)
1678 deps = paren_enclose(deps)
1679 except portage.exception.InvalidDependString as e:
1680 @@ -1347,46 +1322,129 @@ class binarytree(object):
1681 return ""
1682 return mymatch
1683
1684 - def getname(self, pkgname):
1685 - """Returns a file location for this package. The default location is
1686 - ${PKGDIR}/All/${PF}.tbz2, but will be ${PKGDIR}/${CATEGORY}/${PF}.tbz2
1687 - in the rare event of a collision. The prevent_collision() method can
1688 - be called to ensure that ${PKGDIR}/All/${PF}.tbz2 is available for a
1689 - specific cpv."""
1690 + def getname(self, cpv, allocate_new=None):
1691 + """Returns a file location for this package.
1692 + If cpv has both build_time and build_id attributes, then the
1693 + path to the specific corresponding instance is returned.
1694 + Otherwise, allocate a new path and return that. When allocating
1695 + a new path, behavior depends on the binpkg-multi-instance
1696 + FEATURES setting.
1697 + """
1698 if not self.populated:
1699 self.populate()
1700 - mycpv = pkgname
1701 - mypath = self._pkg_paths.get(mycpv, None)
1702 - if mypath:
1703 - return os.path.join(self.pkgdir, mypath)
1704 - mycat, mypkg = catsplit(mycpv)
1705 - if self._all_directory:
1706 - mypath = os.path.join("All", mypkg + ".tbz2")
1707 - if mypath in self._pkg_paths.values():
1708 - mypath = os.path.join(mycat, mypkg + ".tbz2")
1709 +
1710 + try:
1711 + cpv.cp
1712 + except AttributeError:
1713 + cpv = _pkg_str(cpv)
1714 +
1715 + filename = None
1716 + if allocate_new:
1717 + filename = self._allocate_filename(cpv)
1718 + elif self._is_specific_instance(cpv):
1719 + instance_key = self.dbapi._instance_key(cpv)
1720 + path = self._pkg_paths.get(instance_key)
1721 + if path is not None:
1722 + filename = os.path.join(self.pkgdir, path)
1723 +
1724 + if filename is None and not allocate_new:
1725 + try:
1726 + instance_key = self.dbapi._instance_key(cpv,
1727 + support_string=True)
1728 + except KeyError:
1729 + pass
1730 + else:
1731 + filename = self._pkg_paths.get(instance_key)
1732 + if filename is not None:
1733 + filename = os.path.join(self.pkgdir, filename)
1734 +
1735 + if filename is None:
1736 + if self._multi_instance:
1737 + pf = catsplit(cpv)[1]
1738 + filename = "%s-%s.xpak" % (
1739 + os.path.join(self.pkgdir, cpv.cp, pf), "1")
1740 + else:
1741 + filename = os.path.join(self.pkgdir, cpv + ".tbz2")
1742 +
1743 + return filename
1744 +
1745 + def _is_specific_instance(self, cpv):
1746 + specific = True
1747 + try:
1748 + build_time = cpv.build_time
1749 + build_id = cpv.build_id
1750 + except AttributeError:
1751 + specific = False
1752 else:
1753 - mypath = os.path.join(mycat, mypkg + ".tbz2")
1754 - self._pkg_paths[mycpv] = mypath # cache for future lookups
1755 - return os.path.join(self.pkgdir, mypath)
1756 + if build_time is None or build_id is None:
1757 + specific = False
1758 + return specific
1759 +
1760 + def _max_build_id(self, cpv):
1761 + max_build_id = 0
1762 + for x in self.dbapi.cp_list(cpv.cp):
1763 + if (x == cpv and x.build_id is not None and
1764 + x.build_id > max_build_id):
1765 + max_build_id = x.build_id
1766 + return max_build_id
1767 +
1768 + def _allocate_filename(self, cpv):
1769 + return os.path.join(self.pkgdir, cpv + ".tbz2")
1770 +
1771 + def _allocate_filename_multi(self, cpv):
1772 +
1773 + # First, get the max build_id found when _populate was
1774 + # called.
1775 + max_build_id = self._max_build_id(cpv)
1776 +
1777 + # A new package may have been added concurrently since the
1778 + # last _populate call, so use increment build_id until
1779 + # we locate an unused id.
1780 + pf = catsplit(cpv)[1]
1781 + build_id = max_build_id + 1
1782 +
1783 + while True:
1784 + filename = "%s-%s.xpak" % (
1785 + os.path.join(self.pkgdir, cpv.cp, pf), build_id)
1786 + if os.path.exists(filename):
1787 + build_id += 1
1788 + else:
1789 + return filename
1790 +
1791 + @staticmethod
1792 + def _parse_build_id(filename):
1793 + build_id = -1
1794 + hyphen = filename.rfind("-", 0, -6)
1795 + if hyphen != -1:
1796 + build_id = filename[hyphen+1:-5]
1797 + try:
1798 + build_id = long(build_id)
1799 + except ValueError:
1800 + pass
1801 + return build_id
1802
1803 def isremote(self, pkgname):
1804 """Returns true if the package is kept remotely and it has not been
1805 downloaded (or it is only partially downloaded)."""
1806 - if self._remotepkgs is None or pkgname not in self._remotepkgs:
1807 + if (self._remotepkgs is None or
1808 + self.dbapi._instance_key(pkgname) not in self._remotepkgs):
1809 return False
1810 # Presence in self._remotepkgs implies that it's remote. When a
1811 # package is downloaded, state is updated by self.inject().
1812 return True
1813
1814 - def get_pkgindex_uri(self, pkgname):
1815 + def get_pkgindex_uri(self, cpv):
1816 """Returns the URI to the Packages file for a given package."""
1817 - return self._pkgindex_uri.get(pkgname)
1818 -
1819 -
1820 + uri = None
1821 + metadata = self._remotepkgs.get(self.dbapi._instance_key(cpv))
1822 + if metadata is not None:
1823 + uri = metadata["PKGINDEX_URI"]
1824 + return uri
1825
1826 def gettbz2(self, pkgname):
1827 """Fetches the package from a remote site, if necessary. Attempts to
1828 resume if the file appears to be partially downloaded."""
1829 + instance_key = self.dbapi._instance_key(pkgname)
1830 tbz2_path = self.getname(pkgname)
1831 tbz2name = os.path.basename(tbz2_path)
1832 resume = False
1833 @@ -1402,10 +1460,10 @@ class binarytree(object):
1834 self._ensure_dir(mydest)
1835 # urljoin doesn't work correctly with unrecognized protocols like sftp
1836 if self._remote_has_index:
1837 - rel_url = self._remotepkgs[pkgname].get("PATH")
1838 + rel_url = self._remotepkgs[instance_key].get("PATH")
1839 if not rel_url:
1840 rel_url = pkgname+".tbz2"
1841 - remote_base_uri = self._remotepkgs[pkgname]["BASE_URI"]
1842 + remote_base_uri = self._remotepkgs[instance_key]["BASE_URI"]
1843 url = remote_base_uri.rstrip("/") + "/" + rel_url.lstrip("/")
1844 else:
1845 url = self.settings["PORTAGE_BINHOST"].rstrip("/") + "/" + tbz2name
1846 @@ -1448,15 +1506,19 @@ class binarytree(object):
1847 except AttributeError:
1848 cpv = pkg
1849
1850 + _instance_key = self.dbapi._instance_key
1851 + instance_key = _instance_key(cpv)
1852 digests = {}
1853 - metadata = None
1854 - if self._remotepkgs is None or cpv not in self._remotepkgs:
1855 + metadata = (None if self._remotepkgs is None else
1856 + self._remotepkgs.get(instance_key))
1857 + if metadata is None:
1858 for d in self._load_pkgindex().packages:
1859 - if d["CPV"] == cpv:
1860 + if (d["CPV"] == cpv and
1861 + instance_key == _instance_key(_pkg_str(d["CPV"],
1862 + metadata=d, settings=self.settings))):
1863 metadata = d
1864 break
1865 - else:
1866 - metadata = self._remotepkgs[cpv]
1867 +
1868 if metadata is None:
1869 return digests
1870
1871 diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
1872 index cf31c8e..277c2f1 100644
1873 --- a/pym/portage/dbapi/vartree.py
1874 +++ b/pym/portage/dbapi/vartree.py
1875 @@ -173,7 +173,8 @@ class vardbapi(dbapi):
1876 self.vartree = vartree
1877 self._aux_cache_keys = set(
1878 ["BUILD_TIME", "CHOST", "COUNTER", "DEPEND", "DESCRIPTION",
1879 - "EAPI", "HDEPEND", "HOMEPAGE", "IUSE", "KEYWORDS",
1880 + "EAPI", "HDEPEND", "HOMEPAGE",
1881 + "BUILD_ID", "IUSE", "KEYWORDS",
1882 "LICENSE", "PDEPEND", "PROPERTIES", "PROVIDE", "RDEPEND",
1883 "repository", "RESTRICT" , "SLOT", "USE", "DEFINED_PHASES",
1884 "PROVIDES", "REQUIRES"
1885 @@ -425,7 +426,10 @@ class vardbapi(dbapi):
1886 continue
1887 if len(mysplit) > 1:
1888 if ps[0] == mysplit[1]:
1889 - returnme.append(_pkg_str(mysplit[0]+"/"+x))
1890 + cpv = "%s/%s" % (mysplit[0], x)
1891 + metadata = dict(zip(self._aux_cache_keys,
1892 + self.aux_get(cpv, self._aux_cache_keys)))
1893 + returnme.append(_pkg_str(cpv, metadata=metadata))
1894 self._cpv_sort_ascending(returnme)
1895 if use_cache:
1896 self.cpcache[mycp] = [mystat, returnme[:]]
1897 diff --git a/pym/portage/dbapi/virtual.py b/pym/portage/dbapi/virtual.py
1898 index ba9745c..3b7d10e 100644
1899 --- a/pym/portage/dbapi/virtual.py
1900 +++ b/pym/portage/dbapi/virtual.py
1901 @@ -11,12 +11,17 @@ class fakedbapi(dbapi):
1902 """A fake dbapi that allows consumers to inject/remove packages to/from it
1903 portage.settings is required to maintain the dbAPI.
1904 """
1905 - def __init__(self, settings=None, exclusive_slots=True):
1906 + def __init__(self, settings=None, exclusive_slots=True,
1907 + multi_instance=False):
1908 """
1909 @param exclusive_slots: When True, injecting a package with SLOT
1910 metadata causes an existing package in the same slot to be
1911 automatically removed (default is True).
1912 @type exclusive_slots: Boolean
1913 + @param multi_instance: When True, multiple instances with the
1914 + same cpv may be stored simultaneously, as long as they are
1915 + distinguishable (default is False).
1916 + @type multi_instance: Boolean
1917 """
1918 self._exclusive_slots = exclusive_slots
1919 self.cpvdict = {}
1920 @@ -25,6 +30,56 @@ class fakedbapi(dbapi):
1921 from portage import settings
1922 self.settings = settings
1923 self._match_cache = {}
1924 + self._set_multi_instance(multi_instance)
1925 +
1926 + def _set_multi_instance(self, multi_instance):
1927 + """
1928 + Enable or disable multi_instance mode. This should before any
1929 + packages are injected, so that all packages are indexed with
1930 + the same implementation of self._instance_key.
1931 + """
1932 + if self.cpvdict:
1933 + raise AssertionError("_set_multi_instance called after "
1934 + "packages have already been added")
1935 + self._multi_instance = multi_instance
1936 + if multi_instance:
1937 + self._instance_key = self._instance_key_multi_instance
1938 + else:
1939 + self._instance_key = self._instance_key_cpv
1940 +
1941 + def _instance_key_cpv(self, cpv, support_string=False):
1942 + return cpv
1943 +
1944 + def _instance_key_multi_instance(self, cpv, support_string=False):
1945 + try:
1946 + return (cpv, cpv.build_id, cpv.file_size, cpv.build_time,
1947 + cpv.mtime)
1948 + except AttributeError:
1949 + if not support_string:
1950 + raise
1951 +
1952 + # Fallback for interfaces such as aux_get where API consumers
1953 + # may pass in a plain string.
1954 + latest = None
1955 + for pkg in self.cp_list(cpv_getkey(cpv)):
1956 + if pkg == cpv and (
1957 + latest is None or
1958 + latest.build_time < pkg.build_time):
1959 + latest = pkg
1960 +
1961 + if latest is not None:
1962 + return (latest, latest.build_id, latest.file_size,
1963 + latest.build_time, latest.mtime)
1964 +
1965 + raise KeyError(cpv)
1966 +
1967 + def clear(self):
1968 + """
1969 + Remove all packages.
1970 + """
1971 + self._clear_cache()
1972 + self.cpvdict.clear()
1973 + self.cpdict.clear()
1974
1975 def _clear_cache(self):
1976 if self._categories is not None:
1977 @@ -43,7 +98,8 @@ class fakedbapi(dbapi):
1978 return result[:]
1979
1980 def cpv_exists(self, mycpv, myrepo=None):
1981 - return mycpv in self.cpvdict
1982 + return self._instance_key(mycpv,
1983 + support_string=True) in self.cpvdict
1984
1985 def cp_list(self, mycp, use_cache=1, myrepo=None):
1986 # NOTE: Cache can be safely shared with the match cache, since the
1987 @@ -63,7 +119,10 @@ class fakedbapi(dbapi):
1988 return list(self.cpdict)
1989
1990 def cpv_all(self):
1991 - return list(self.cpvdict)
1992 + if self._multi_instance:
1993 + return [x[0] for x in self.cpvdict]
1994 + else:
1995 + return list(self.cpvdict)
1996
1997 def cpv_inject(self, mycpv, metadata=None):
1998 """Adds a cpv to the list of available packages. See the
1999 @@ -99,13 +158,14 @@ class fakedbapi(dbapi):
2000 except AttributeError:
2001 pass
2002
2003 - self.cpvdict[mycpv] = metadata
2004 + instance_key = self._instance_key(mycpv)
2005 + self.cpvdict[instance_key] = metadata
2006 if not self._exclusive_slots:
2007 myslot = None
2008 if myslot and mycp in self.cpdict:
2009 # If necessary, remove another package in the same SLOT.
2010 for cpv in self.cpdict[mycp]:
2011 - if mycpv != cpv:
2012 + if instance_key != self._instance_key(cpv):
2013 try:
2014 other_slot = cpv.slot
2015 except AttributeError:
2016 @@ -115,40 +175,41 @@ class fakedbapi(dbapi):
2017 self.cpv_remove(cpv)
2018 break
2019
2020 - cp_list = self.cpdict.get(mycp)
2021 - if cp_list is None:
2022 - cp_list = []
2023 - self.cpdict[mycp] = cp_list
2024 - try:
2025 - cp_list.remove(mycpv)
2026 - except ValueError:
2027 - pass
2028 + cp_list = self.cpdict.get(mycp, [])
2029 + cp_list = [x for x in cp_list
2030 + if self._instance_key(x) != instance_key]
2031 cp_list.append(mycpv)
2032 + self.cpdict[mycp] = cp_list
2033
2034 def cpv_remove(self,mycpv):
2035 """Removes a cpv from the list of available packages."""
2036 self._clear_cache()
2037 mycp = cpv_getkey(mycpv)
2038 - if mycpv in self.cpvdict:
2039 - del self.cpvdict[mycpv]
2040 - if mycp not in self.cpdict:
2041 - return
2042 - while mycpv in self.cpdict[mycp]:
2043 - del self.cpdict[mycp][self.cpdict[mycp].index(mycpv)]
2044 - if not len(self.cpdict[mycp]):
2045 - del self.cpdict[mycp]
2046 + instance_key = self._instance_key(mycpv)
2047 + self.cpvdict.pop(instance_key, None)
2048 + cp_list = self.cpdict.get(mycp)
2049 + if cp_list is not None:
2050 + cp_list = [x for x in cp_list
2051 + if self._instance_key(x) != instance_key]
2052 + if cp_list:
2053 + self.cpdict[mycp] = cp_list
2054 + else:
2055 + del self.cpdict[mycp]
2056
2057 def aux_get(self, mycpv, wants, myrepo=None):
2058 - if not self.cpv_exists(mycpv):
2059 + metadata = self.cpvdict.get(
2060 + self._instance_key(mycpv, support_string=True))
2061 + if metadata is None:
2062 raise KeyError(mycpv)
2063 - metadata = self.cpvdict[mycpv]
2064 - if not metadata:
2065 - return ["" for x in wants]
2066 return [metadata.get(x, "") for x in wants]
2067
2068 def aux_update(self, cpv, values):
2069 self._clear_cache()
2070 - self.cpvdict[cpv].update(values)
2071 + metadata = self.cpvdict.get(
2072 + self._instance_key(cpv, support_string=True))
2073 + if metadata is None:
2074 + raise KeyError(cpv)
2075 + metadata.update(values)
2076
2077 class testdbapi(object):
2078 """A dbapi instance with completely fake functions to get by hitting disk
2079 diff --git a/pym/portage/emaint/modules/binhost/binhost.py b/pym/portage/emaint/modules/binhost/binhost.py
2080 index 1138a8c..cf1213e 100644
2081 --- a/pym/portage/emaint/modules/binhost/binhost.py
2082 +++ b/pym/portage/emaint/modules/binhost/binhost.py
2083 @@ -7,6 +7,7 @@ import stat
2084 import portage
2085 from portage import os
2086 from portage.util import writemsg
2087 +from portage.versions import _pkg_str
2088
2089 import sys
2090
2091 @@ -38,7 +39,7 @@ class BinhostHandler(object):
2092 if size is None:
2093 return True
2094
2095 - mtime = data.get("MTIME")
2096 + mtime = data.get("_mtime_")
2097 if mtime is None:
2098 return True
2099
2100 @@ -90,6 +91,7 @@ class BinhostHandler(object):
2101 def fix(self, **kwargs):
2102 onProgress = kwargs.get('onProgress', None)
2103 bintree = self._bintree
2104 + _instance_key = bintree.dbapi._instance_key
2105 cpv_all = self._bintree.dbapi.cpv_all()
2106 cpv_all.sort()
2107 missing = []
2108 @@ -98,16 +100,21 @@ class BinhostHandler(object):
2109 onProgress(maxval, 0)
2110 pkgindex = self._pkgindex
2111 missing = []
2112 + stale = []
2113 metadata = {}
2114 for d in pkgindex.packages:
2115 - metadata[d["CPV"]] = d
2116 -
2117 - for i, cpv in enumerate(cpv_all):
2118 - d = metadata.get(cpv)
2119 + cpv = _pkg_str(d["CPV"], metadata=d,
2120 + settings=bintree.settings)
2121 + d["CPV"] = cpv
2122 + metadata[_instance_key(cpv)] = d
2123 + if not bintree.dbapi.cpv_exists(cpv):
2124 + stale.append(cpv)
2125 +
2126 + for cpv in cpv_all:
2127 + d = metadata.get(_instance_key(cpv))
2128 if not d or self._need_update(cpv, d):
2129 missing.append(cpv)
2130
2131 - stale = set(metadata).difference(cpv_all)
2132 if missing or stale:
2133 from portage import locks
2134 pkgindex_lock = locks.lockfile(
2135 @@ -121,31 +128,39 @@ class BinhostHandler(object):
2136 pkgindex = bintree._load_pkgindex()
2137 self._pkgindex = pkgindex
2138
2139 + # Recount stale/missing packages, with lock held.
2140 + missing = []
2141 + stale = []
2142 metadata = {}
2143 for d in pkgindex.packages:
2144 - metadata[d["CPV"]] = d
2145 -
2146 - # Recount missing packages, with lock held.
2147 - del missing[:]
2148 - for i, cpv in enumerate(cpv_all):
2149 - d = metadata.get(cpv)
2150 + cpv = _pkg_str(d["CPV"], metadata=d,
2151 + settings=bintree.settings)
2152 + d["CPV"] = cpv
2153 + metadata[_instance_key(cpv)] = d
2154 + if not bintree.dbapi.cpv_exists(cpv):
2155 + stale.append(cpv)
2156 +
2157 + for cpv in cpv_all:
2158 + d = metadata.get(_instance_key(cpv))
2159 if not d or self._need_update(cpv, d):
2160 missing.append(cpv)
2161
2162 maxval = len(missing)
2163 for i, cpv in enumerate(missing):
2164 + d = bintree._pkgindex_entry(cpv)
2165 try:
2166 - metadata[cpv] = bintree._pkgindex_entry(cpv)
2167 + bintree._eval_use_flags(cpv, d)
2168 except portage.exception.InvalidDependString:
2169 writemsg("!!! Invalid binary package: '%s'\n" % \
2170 bintree.getname(cpv), noiselevel=-1)
2171 + else:
2172 + metadata[_instance_key(cpv)] = d
2173
2174 if onProgress:
2175 onProgress(maxval, i+1)
2176
2177 - for cpv in set(metadata).difference(
2178 - self._bintree.dbapi.cpv_all()):
2179 - del metadata[cpv]
2180 + for cpv in stale:
2181 + del metadata[_instance_key(cpv)]
2182
2183 # We've updated the pkgindex, so set it to
2184 # repopulate when necessary.
2185 diff --git a/pym/portage/package/ebuild/config.py b/pym/portage/package/ebuild/config.py
2186 index 71fe4df..961b1c8 100644
2187 --- a/pym/portage/package/ebuild/config.py
2188 +++ b/pym/portage/package/ebuild/config.py
2189 @@ -154,7 +154,8 @@ class config(object):
2190 'PORTAGE_PYM_PATH', 'PORTAGE_PYTHONPATH'])
2191
2192 _setcpv_aux_keys = ('DEFINED_PHASES', 'DEPEND', 'EAPI', 'HDEPEND',
2193 - 'INHERITED', 'IUSE', 'REQUIRED_USE', 'KEYWORDS', 'LICENSE', 'PDEPEND',
2194 + 'INHERITED', 'BUILD_ID', 'IUSE', 'REQUIRED_USE',
2195 + 'KEYWORDS', 'LICENSE', 'PDEPEND',
2196 'PROPERTIES', 'PROVIDE', 'RDEPEND', 'SLOT',
2197 'repository', 'RESTRICT', 'LICENSE',)
2198
2199 diff --git a/pym/portage/tests/resolver/ResolverPlayground.py b/pym/portage/tests/resolver/ResolverPlayground.py
2200 index 84ad17c..48b86fc 100644
2201 --- a/pym/portage/tests/resolver/ResolverPlayground.py
2202 +++ b/pym/portage/tests/resolver/ResolverPlayground.py
2203 @@ -39,6 +39,7 @@ class ResolverPlayground(object):
2204
2205 config_files = frozenset(("eapi", "layout.conf", "make.conf", "package.accept_keywords",
2206 "package.keywords", "package.license", "package.mask", "package.properties",
2207 + "package.provided", "packages",
2208 "package.unmask", "package.use", "package.use.aliases", "package.use.stable.mask",
2209 "soname.provided",
2210 "unpack_dependencies", "use.aliases", "use.force", "use.mask", "layout.conf"))
2211 @@ -208,25 +209,37 @@ class ResolverPlayground(object):
2212 raise AssertionError('digest creation failed for %s' % ebuild_path)
2213
2214 def _create_binpkgs(self, binpkgs):
2215 - for cpv, metadata in binpkgs.items():
2216 + # When using BUILD_ID, there can be mutiple instances for the
2217 + # same cpv. Therefore, binpkgs may be an iterable instead of
2218 + # a dict.
2219 + items = getattr(binpkgs, 'items', None)
2220 + items = items() if items is not None else binpkgs
2221 + for cpv, metadata in items:
2222 a = Atom("=" + cpv, allow_repo=True)
2223 repo = a.repo
2224 if repo is None:
2225 repo = "test_repo"
2226
2227 + pn = catsplit(a.cp)[1]
2228 cat, pf = catsplit(a.cpv)
2229 metadata = metadata.copy()
2230 metadata.setdefault("SLOT", "0")
2231 metadata.setdefault("KEYWORDS", "x86")
2232 metadata.setdefault("BUILD_TIME", "0")
2233 + metadata.setdefault("EAPI", "0")
2234 metadata["repository"] = repo
2235 metadata["CATEGORY"] = cat
2236 metadata["PF"] = pf
2237
2238 repo_dir = self.pkgdir
2239 category_dir = os.path.join(repo_dir, cat)
2240 - binpkg_path = os.path.join(category_dir, pf + ".tbz2")
2241 - ensure_dirs(category_dir)
2242 + if "BUILD_ID" in metadata:
2243 + binpkg_path = os.path.join(category_dir, pn,
2244 + "%s-%s.xpak"% (pf, metadata["BUILD_ID"]))
2245 + else:
2246 + binpkg_path = os.path.join(category_dir, pf + ".tbz2")
2247 +
2248 + ensure_dirs(os.path.dirname(binpkg_path))
2249 t = portage.xpak.tbz2(binpkg_path)
2250 t.recompose_mem(portage.xpak.xpak_mem(metadata))
2251
2252 @@ -252,6 +265,7 @@ class ResolverPlayground(object):
2253 unknown_keys = set(metadata).difference(
2254 portage.dbapi.dbapi._known_keys)
2255 unknown_keys.discard("BUILD_TIME")
2256 + unknown_keys.discard("BUILD_ID")
2257 unknown_keys.discard("COUNTER")
2258 unknown_keys.discard("repository")
2259 unknown_keys.discard("USE")
2260 @@ -749,7 +763,11 @@ class ResolverPlaygroundResult(object):
2261 repo_str = ""
2262 if x.repo != "test_repo":
2263 repo_str = _repo_separator + x.repo
2264 - mergelist_str = x.cpv + repo_str
2265 + build_id_str = ""
2266 + if (x.type_name == "binary" and
2267 + x.cpv.build_id is not None):
2268 + build_id_str = "-%s" % x.cpv.build_id
2269 + mergelist_str = x.cpv + build_id_str + repo_str
2270 if x.built:
2271 if x.operation == "merge":
2272 desc = x.type_name
2273 diff --git a/pym/portage/tests/resolver/binpkg_multi_instance/__init__.py b/pym/portage/tests/resolver/binpkg_multi_instance/__init__.py
2274 new file mode 100644
2275 index 0000000..4725d33
2276 --- /dev/null
2277 +++ b/pym/portage/tests/resolver/binpkg_multi_instance/__init__.py
2278 @@ -0,0 +1,2 @@
2279 +# Copyright 2015 Gentoo Foundation
2280 +# Distributed under the terms of the GNU General Public License v2
2281 diff --git a/pym/portage/tests/resolver/binpkg_multi_instance/__test__.py b/pym/portage/tests/resolver/binpkg_multi_instance/__test__.py
2282 new file mode 100644
2283 index 0000000..4725d33
2284 --- /dev/null
2285 +++ b/pym/portage/tests/resolver/binpkg_multi_instance/__test__.py
2286 @@ -0,0 +1,2 @@
2287 +# Copyright 2015 Gentoo Foundation
2288 +# Distributed under the terms of the GNU General Public License v2
2289 diff --git a/pym/portage/tests/resolver/binpkg_multi_instance/test_rebuilt_binaries.py b/pym/portage/tests/resolver/binpkg_multi_instance/test_rebuilt_binaries.py
2290 new file mode 100644
2291 index 0000000..5729df4
2292 --- /dev/null
2293 +++ b/pym/portage/tests/resolver/binpkg_multi_instance/test_rebuilt_binaries.py
2294 @@ -0,0 +1,101 @@
2295 +# Copyright 2015 Gentoo Foundation
2296 +# Distributed under the terms of the GNU General Public License v2
2297 +
2298 +from portage.tests import TestCase
2299 +from portage.tests.resolver.ResolverPlayground import (ResolverPlayground,
2300 + ResolverPlaygroundTestCase)
2301 +
2302 +class RebuiltBinariesCase(TestCase):
2303 +
2304 + def testRebuiltBinaries(self):
2305 +
2306 + user_config = {
2307 + "make.conf":
2308 + (
2309 + "FEATURES=\"binpkg-multi-instance\"",
2310 + ),
2311 + }
2312 +
2313 + binpkgs = (
2314 + ("app-misc/A-1", {
2315 + "EAPI": "5",
2316 + "BUILD_ID": "1",
2317 + "BUILD_TIME": "1",
2318 + }),
2319 + ("app-misc/A-1", {
2320 + "EAPI": "5",
2321 + "BUILD_ID": "2",
2322 + "BUILD_TIME": "2",
2323 + }),
2324 + ("app-misc/A-1", {
2325 + "EAPI": "5",
2326 + "BUILD_ID": "3",
2327 + "BUILD_TIME": "3",
2328 + }),
2329 + ("dev-libs/B-1", {
2330 + "EAPI": "5",
2331 + "BUILD_ID": "1",
2332 + "BUILD_TIME": "1",
2333 + }),
2334 + ("dev-libs/B-1", {
2335 + "EAPI": "5",
2336 + "BUILD_ID": "2",
2337 + "BUILD_TIME": "2",
2338 + }),
2339 + ("dev-libs/B-1", {
2340 + "EAPI": "5",
2341 + "BUILD_ID": "3",
2342 + "BUILD_TIME": "3",
2343 + }),
2344 + )
2345 +
2346 + installed = {
2347 + "app-misc/A-1" : {
2348 + "EAPI": "5",
2349 + "BUILD_ID": "1",
2350 + "BUILD_TIME": "1",
2351 + },
2352 + "dev-libs/B-1" : {
2353 + "EAPI": "5",
2354 + "BUILD_ID": "2",
2355 + "BUILD_TIME": "2",
2356 + },
2357 + }
2358 +
2359 + world = (
2360 + "app-misc/A",
2361 + "dev-libs/B",
2362 + )
2363 +
2364 + test_cases = (
2365 +
2366 + ResolverPlaygroundTestCase(
2367 + ["@world"],
2368 + options = {
2369 + "--deep": True,
2370 + "--rebuilt-binaries": True,
2371 + "--update": True,
2372 + "--usepkgonly": True,
2373 + },
2374 + success = True,
2375 + ignore_mergelist_order=True,
2376 + mergelist = [
2377 + "[binary]dev-libs/B-1-3",
2378 + "[binary]app-misc/A-1-3"
2379 + ]
2380 + ),
2381 +
2382 + )
2383 +
2384 + playground = ResolverPlayground(debug=False,
2385 + binpkgs=binpkgs, installed=installed,
2386 + user_config=user_config, world=world)
2387 + try:
2388 + for test_case in test_cases:
2389 + playground.run_TestCase(test_case)
2390 + self.assertEqual(test_case.test_success, True,
2391 + test_case.fail_msg)
2392 + finally:
2393 + # Disable debug so that cleanup works.
2394 + #playground.debug = False
2395 + playground.cleanup()
2396 diff --git a/pym/portage/versions.py b/pym/portage/versions.py
2397 index 2c9fe5b..c2c6675 100644
2398 --- a/pym/portage/versions.py
2399 +++ b/pym/portage/versions.py
2400 @@ -18,6 +18,7 @@ if sys.hexversion < 0x3000000:
2401 _unicode = unicode
2402 else:
2403 _unicode = str
2404 + long = int
2405
2406 import portage
2407 portage.proxy.lazyimport.lazyimport(globals(),
2408 @@ -361,11 +362,13 @@ class _pkg_str(_unicode):
2409 """
2410
2411 def __new__(cls, cpv, metadata=None, settings=None, eapi=None,
2412 - repo=None, slot=None):
2413 + repo=None, slot=None, build_time=None, build_id=None,
2414 + file_size=None, mtime=None):
2415 return _unicode.__new__(cls, cpv)
2416
2417 def __init__(self, cpv, metadata=None, settings=None, eapi=None,
2418 - repo=None, slot=None):
2419 + repo=None, slot=None, build_time=None, build_id=None,
2420 + file_size=None, mtime=None):
2421 if not isinstance(cpv, _unicode):
2422 # Avoid TypeError from _unicode.__init__ with PyPy.
2423 cpv = _unicode_decode(cpv)
2424 @@ -375,10 +378,51 @@ class _pkg_str(_unicode):
2425 slot = metadata.get('SLOT', slot)
2426 repo = metadata.get('repository', repo)
2427 eapi = metadata.get('EAPI', eapi)
2428 + build_time = metadata.get('BUILD_TIME', build_time)
2429 + file_size = metadata.get('SIZE', file_size)
2430 + build_id = metadata.get('BUILD_ID', build_id)
2431 + mtime = metadata.get('_mtime_', mtime)
2432 if settings is not None:
2433 self.__dict__['_settings'] = settings
2434 if eapi is not None:
2435 self.__dict__['eapi'] = eapi
2436 + if build_time is not None:
2437 + try:
2438 + build_time = long(build_time)
2439 + except ValueError:
2440 + if build_time:
2441 + build_time = -1
2442 + else:
2443 + build_time = 0
2444 + if build_id is not None:
2445 + try:
2446 + build_id = long(build_id)
2447 + except ValueError:
2448 + if build_id:
2449 + build_id = -1
2450 + else:
2451 + build_id = None
2452 + if file_size is not None:
2453 + try:
2454 + file_size = long(file_size)
2455 + except ValueError:
2456 + if file_size:
2457 + file_size = -1
2458 + else:
2459 + file_size = None
2460 + if mtime is not None:
2461 + try:
2462 + mtime = long(mtime)
2463 + except ValueError:
2464 + if mtime:
2465 + mtime = -1
2466 + else:
2467 + mtime = None
2468 +
2469 + self.__dict__['build_time'] = build_time
2470 + self.__dict__['file_size'] = file_size
2471 + self.__dict__['build_id'] = build_id
2472 + self.__dict__['mtime'] = mtime
2473 self.__dict__['cpv_split'] = catpkgsplit(cpv, eapi=eapi)
2474 if self.cpv_split is None:
2475 raise InvalidData(cpv)
2476 --
2477 2.0.5

Replies