1 |
FEATURES=binpkg-multi-instance causes an integer build-id to be |
2 |
associated with each binary package instance. Inclusion of the build-id |
3 |
in the file name of the binary package file makes it possible to store |
4 |
an arbitrary number of binary packages built from the same ebuild. |
5 |
|
6 |
Having multiple instances is useful for a number of purposes, such as |
7 |
retaining builds that were built with different USE flags or linked |
8 |
against different versions of libraries. The location of any particular |
9 |
package within PKGDIR can be expressed as follows: |
10 |
|
11 |
${PKGDIR}/${CATEGORY}/${PN}/${PF}-${BUILD_ID}.xpak |
12 |
|
13 |
The build-id starts at 1 for the first build of a particular ebuild, |
14 |
and is incremented by 1 for each new build. It is possible to share a |
15 |
writable PKGDIR over NFS, and locking ensures that each package added |
16 |
to PKGDIR will have a unique build-id. It is not necessary to migrate |
17 |
an existing PKGDIR to the new layout, since portage is capable of |
18 |
working with a mixed PKGDIR layout, where packages using the old layout |
19 |
are allowed to remain in place. |
20 |
|
21 |
The new PKGDIR layout is backward-compatible with binhost clients |
22 |
running older portage, since the file format is identical, the |
23 |
per-package PATH attribute in the 'Packages' index directs them to |
24 |
download the file from the correct URI, and they automatically use |
25 |
BUILD_TIME metadata to select the latest builds. |
26 |
|
27 |
There is currently no automated way to prune old builds from PKGDIR, |
28 |
although it is possible to remove packages manually, and then run |
29 |
'emaint --fix binhost' to update the ${PKGDIR}/Packages index. Support |
30 |
for FEATURES=binpkg-multi-instance is planned for eclean-pkg. |
31 |
|
32 |
X-Gentoo-Bug: 150031 |
33 |
X-Gentoo-Bug-URL: https://bugs.gentoo.org/show_bug.cgi?id=150031 |
34 |
--- |
35 |
PATCH 3/7 v2 fixes BinpkgPrefetcher to pass the _pkg_path to |
36 |
BinpkgVerifier. Thanks to David James <davidjames@××××××.com>. |
37 |
|
38 |
bin/quickpkg | 1 - |
39 |
man/make.conf.5 | 27 + |
40 |
pym/_emerge/Binpkg.py | 33 +- |
41 |
pym/_emerge/BinpkgFetcher.py | 13 +- |
42 |
pym/_emerge/BinpkgPrefetcher.py | 2 +- |
43 |
pym/_emerge/BinpkgVerifier.py | 6 +- |
44 |
pym/_emerge/EbuildBinpkg.py | 9 +- |
45 |
pym/_emerge/EbuildBuild.py | 36 +- |
46 |
pym/_emerge/Package.py | 16 +- |
47 |
pym/_emerge/Scheduler.py | 6 +- |
48 |
pym/_emerge/clear_caches.py | 1 - |
49 |
pym/portage/const.py | 2 + |
50 |
pym/portage/dbapi/bintree.py | 684 +++++++++++++++++--------- |
51 |
pym/portage/emaint/modules/binhost/binhost.py | 47 +- |
52 |
14 files changed, 602 insertions(+), 281 deletions(-) |
53 |
|
54 |
diff --git a/bin/quickpkg b/bin/quickpkg |
55 |
index 2c69a69..8b71c3e 100755 |
56 |
--- a/bin/quickpkg |
57 |
+++ b/bin/quickpkg |
58 |
@@ -63,7 +63,6 @@ def quickpkg_atom(options, infos, arg, eout): |
59 |
pkgs_for_arg = 0 |
60 |
for cpv in matches: |
61 |
excluded_config_files = [] |
62 |
- bintree.prevent_collision(cpv) |
63 |
dblnk = vardb._dblink(cpv) |
64 |
have_lock = False |
65 |
|
66 |
diff --git a/man/make.conf.5 b/man/make.conf.5 |
67 |
index cd1ae21..1b71b97 100644 |
68 |
--- a/man/make.conf.5 |
69 |
+++ b/man/make.conf.5 |
70 |
@@ -256,6 +256,33 @@ has a \fB\-\-force\fR option that can be used to force regeneration of digests. |
71 |
Keep logs from successful binary package merges. This is relevant only when |
72 |
\fBPORT_LOGDIR\fR is set. |
73 |
.TP |
74 |
+.B binpkg\-multi\-instance |
75 |
+Enable support for multiple binary package instances per ebuild. |
76 |
+Having multiple instances is useful for a number of purposes, such as |
77 |
+retaining builds that were built with different USE flags or linked |
78 |
+against different versions of libraries. The location of any particular |
79 |
+package within PKGDIR can be expressed as follows: |
80 |
+ |
81 |
+ ${PKGDIR}/${CATEGORY}/${PN}/${PF}\-${BUILD_ID}.xpak |
82 |
+ |
83 |
+The build\-id starts at 1 for the first build of a particular ebuild, |
84 |
+and is incremented by 1 for each new build. It is possible to share a |
85 |
+writable PKGDIR over NFS, and locking ensures that each package added |
86 |
+to PKGDIR will have a unique build\-id. It is not necessary to migrate |
87 |
+an existing PKGDIR to the new layout, since portage is capable of |
88 |
+working with a mixed PKGDIR layout, where packages using the old layout |
89 |
+are allowed to remain in place. |
90 |
+ |
91 |
+The new PKGDIR layout is backward\-compatible with binhost clients |
92 |
+running older portage, since the file format is identical, the |
93 |
+per\-package PATH attribute in the 'Packages' index directs them to |
94 |
+download the file from the correct URI, and they automatically use |
95 |
+BUILD_TIME metadata to select the latest builds. |
96 |
+ |
97 |
+There is currently no automated way to prune old builds from PKGDIR, |
98 |
+although it is possible to remove packages manually, and then run |
99 |
+\(aqemaint \-\-fix binhost' to update the ${PKGDIR}/Packages index. |
100 |
+.TP |
101 |
.B buildpkg |
102 |
Binary packages will be created for all packages that are merged. Also see |
103 |
\fBquickpkg\fR(1) and \fBemerge\fR(1) \fB\-\-buildpkg\fR and |
104 |
diff --git a/pym/_emerge/Binpkg.py b/pym/_emerge/Binpkg.py |
105 |
index ded6dfd..7b7ae17 100644 |
106 |
--- a/pym/_emerge/Binpkg.py |
107 |
+++ b/pym/_emerge/Binpkg.py |
108 |
@@ -121,16 +121,11 @@ class Binpkg(CompositeTask): |
109 |
fetcher = BinpkgFetcher(background=self.background, |
110 |
logfile=self.settings.get("PORTAGE_LOG_FILE"), pkg=self.pkg, |
111 |
pretend=self.opts.pretend, scheduler=self.scheduler) |
112 |
- pkg_path = fetcher.pkg_path |
113 |
- self._pkg_path = pkg_path |
114 |
- # This gives bashrc users an opportunity to do various things |
115 |
- # such as remove binary packages after they're installed. |
116 |
- self.settings["PORTAGE_BINPKG_FILE"] = pkg_path |
117 |
|
118 |
if self.opts.getbinpkg and self._bintree.isremote(pkg.cpv): |
119 |
- |
120 |
msg = " --- (%s of %s) Fetching Binary (%s::%s)" %\ |
121 |
- (pkg_count.curval, pkg_count.maxval, pkg.cpv, pkg_path) |
122 |
+ (pkg_count.curval, pkg_count.maxval, pkg.cpv, |
123 |
+ fetcher.pkg_path) |
124 |
short_msg = "emerge: (%s of %s) %s Fetch" % \ |
125 |
(pkg_count.curval, pkg_count.maxval, pkg.cpv) |
126 |
self.logger.log(msg, short_msg=short_msg) |
127 |
@@ -149,7 +144,7 @@ class Binpkg(CompositeTask): |
128 |
# The fetcher only has a returncode when |
129 |
# --getbinpkg is enabled. |
130 |
if fetcher.returncode is not None: |
131 |
- self._fetched_pkg = True |
132 |
+ self._fetched_pkg = fetcher.pkg_path |
133 |
if self._default_exit(fetcher) != os.EX_OK: |
134 |
self._unlock_builddir() |
135 |
self.wait() |
136 |
@@ -163,9 +158,15 @@ class Binpkg(CompositeTask): |
137 |
|
138 |
verifier = None |
139 |
if self._verify: |
140 |
+ if self._fetched_pkg: |
141 |
+ path = self._fetched_pkg |
142 |
+ else: |
143 |
+ path = self.pkg.root_config.trees["bintree"].getname( |
144 |
+ self.pkg.cpv) |
145 |
logfile = self.settings.get("PORTAGE_LOG_FILE") |
146 |
verifier = BinpkgVerifier(background=self.background, |
147 |
- logfile=logfile, pkg=self.pkg, scheduler=self.scheduler) |
148 |
+ logfile=logfile, pkg=self.pkg, scheduler=self.scheduler, |
149 |
+ _pkg_path=path) |
150 |
self._start_task(verifier, self._verifier_exit) |
151 |
return |
152 |
|
153 |
@@ -181,10 +182,20 @@ class Binpkg(CompositeTask): |
154 |
logger = self.logger |
155 |
pkg = self.pkg |
156 |
pkg_count = self.pkg_count |
157 |
- pkg_path = self._pkg_path |
158 |
|
159 |
if self._fetched_pkg: |
160 |
- self._bintree.inject(pkg.cpv, filename=pkg_path) |
161 |
+ pkg_path = self._bintree.getname( |
162 |
+ self._bintree.inject(pkg.cpv, |
163 |
+ filename=self._fetched_pkg), |
164 |
+ allocate_new=False) |
165 |
+ else: |
166 |
+ pkg_path = self.pkg.root_config.trees["bintree"].getname( |
167 |
+ self.pkg.cpv) |
168 |
+ |
169 |
+ # This gives bashrc users an opportunity to do various things |
170 |
+ # such as remove binary packages after they're installed. |
171 |
+ self.settings["PORTAGE_BINPKG_FILE"] = pkg_path |
172 |
+ self._pkg_path = pkg_path |
173 |
|
174 |
logfile = self.settings.get("PORTAGE_LOG_FILE") |
175 |
if logfile is not None and os.path.isfile(logfile): |
176 |
diff --git a/pym/_emerge/BinpkgFetcher.py b/pym/_emerge/BinpkgFetcher.py |
177 |
index 543881e..a7f2d44 100644 |
178 |
--- a/pym/_emerge/BinpkgFetcher.py |
179 |
+++ b/pym/_emerge/BinpkgFetcher.py |
180 |
@@ -24,7 +24,8 @@ class BinpkgFetcher(SpawnProcess): |
181 |
def __init__(self, **kwargs): |
182 |
SpawnProcess.__init__(self, **kwargs) |
183 |
pkg = self.pkg |
184 |
- self.pkg_path = pkg.root_config.trees["bintree"].getname(pkg.cpv) |
185 |
+ self.pkg_path = pkg.root_config.trees["bintree"].getname( |
186 |
+ pkg.cpv) + ".partial" |
187 |
|
188 |
def _start(self): |
189 |
|
190 |
@@ -51,10 +52,12 @@ class BinpkgFetcher(SpawnProcess): |
191 |
# urljoin doesn't work correctly with |
192 |
# unrecognized protocols like sftp |
193 |
if bintree._remote_has_index: |
194 |
- rel_uri = bintree._remotepkgs[pkg.cpv].get("PATH") |
195 |
+ instance_key = bintree.dbapi._instance_key(pkg.cpv) |
196 |
+ rel_uri = bintree._remotepkgs[instance_key].get("PATH") |
197 |
if not rel_uri: |
198 |
rel_uri = pkg.cpv + ".tbz2" |
199 |
- remote_base_uri = bintree._remotepkgs[pkg.cpv]["BASE_URI"] |
200 |
+ remote_base_uri = bintree._remotepkgs[ |
201 |
+ instance_key]["BASE_URI"] |
202 |
uri = remote_base_uri.rstrip("/") + "/" + rel_uri.lstrip("/") |
203 |
else: |
204 |
uri = settings["PORTAGE_BINHOST"].rstrip("/") + \ |
205 |
@@ -128,7 +131,9 @@ class BinpkgFetcher(SpawnProcess): |
206 |
# the fetcher didn't already do it automatically. |
207 |
bintree = self.pkg.root_config.trees["bintree"] |
208 |
if bintree._remote_has_index: |
209 |
- remote_mtime = bintree._remotepkgs[self.pkg.cpv].get("MTIME") |
210 |
+ remote_mtime = bintree._remotepkgs[ |
211 |
+ bintree.dbapi._instance_key( |
212 |
+ self.pkg.cpv)].get("MTIME") |
213 |
if remote_mtime is not None: |
214 |
try: |
215 |
remote_mtime = long(remote_mtime) |
216 |
diff --git a/pym/_emerge/BinpkgPrefetcher.py b/pym/_emerge/BinpkgPrefetcher.py |
217 |
index ffa4900..7ca8970 100644 |
218 |
--- a/pym/_emerge/BinpkgPrefetcher.py |
219 |
+++ b/pym/_emerge/BinpkgPrefetcher.py |
220 |
@@ -27,7 +27,7 @@ class BinpkgPrefetcher(CompositeTask): |
221 |
|
222 |
verifier = BinpkgVerifier(background=self.background, |
223 |
logfile=self.scheduler.fetch.log_file, pkg=self.pkg, |
224 |
- scheduler=self.scheduler) |
225 |
+ scheduler=self.scheduler, _pkg_path=self.pkg_path) |
226 |
self._start_task(verifier, self._verifier_exit) |
227 |
|
228 |
def _verifier_exit(self, verifier): |
229 |
diff --git a/pym/_emerge/BinpkgVerifier.py b/pym/_emerge/BinpkgVerifier.py |
230 |
index 2c69792..7a6d15e 100644 |
231 |
--- a/pym/_emerge/BinpkgVerifier.py |
232 |
+++ b/pym/_emerge/BinpkgVerifier.py |
233 |
@@ -33,7 +33,6 @@ class BinpkgVerifier(CompositeTask): |
234 |
digests = _apply_hash_filter(digests, hash_filter) |
235 |
|
236 |
self._digests = digests |
237 |
- self._pkg_path = bintree.getname(self.pkg.cpv) |
238 |
|
239 |
try: |
240 |
size = os.stat(self._pkg_path).st_size |
241 |
@@ -90,8 +89,11 @@ class BinpkgVerifier(CompositeTask): |
242 |
if portage.output.havecolor: |
243 |
portage.output.havecolor = not self.background |
244 |
|
245 |
+ path = self._pkg_path |
246 |
+ if path.endswith(".partial"): |
247 |
+ path = path[:-len(".partial")] |
248 |
eout = EOutput() |
249 |
- eout.ebegin("%s %s ;-)" % (os.path.basename(self._pkg_path), |
250 |
+ eout.ebegin("%s %s ;-)" % (os.path.basename(path), |
251 |
" ".join(sorted(self._digests)))) |
252 |
eout.eend(0) |
253 |
|
254 |
diff --git a/pym/_emerge/EbuildBinpkg.py b/pym/_emerge/EbuildBinpkg.py |
255 |
index 34a6aef..6e098eb 100644 |
256 |
--- a/pym/_emerge/EbuildBinpkg.py |
257 |
+++ b/pym/_emerge/EbuildBinpkg.py |
258 |
@@ -10,13 +10,12 @@ class EbuildBinpkg(CompositeTask): |
259 |
This assumes that src_install() has successfully completed. |
260 |
""" |
261 |
__slots__ = ('pkg', 'settings') + \ |
262 |
- ('_binpkg_tmpfile',) |
263 |
+ ('_binpkg_tmpfile', '_binpkg_info') |
264 |
|
265 |
def _start(self): |
266 |
pkg = self.pkg |
267 |
root_config = pkg.root_config |
268 |
bintree = root_config.trees["bintree"] |
269 |
- bintree.prevent_collision(pkg.cpv) |
270 |
binpkg_tmpfile = os.path.join(bintree.pkgdir, |
271 |
pkg.cpv + ".tbz2." + str(os.getpid())) |
272 |
bintree._ensure_dir(os.path.dirname(binpkg_tmpfile)) |
273 |
@@ -43,8 +42,12 @@ class EbuildBinpkg(CompositeTask): |
274 |
|
275 |
pkg = self.pkg |
276 |
bintree = pkg.root_config.trees["bintree"] |
277 |
- bintree.inject(pkg.cpv, filename=self._binpkg_tmpfile) |
278 |
+ self._binpkg_info = bintree.inject(pkg.cpv, |
279 |
+ filename=self._binpkg_tmpfile) |
280 |
|
281 |
self._current_task = None |
282 |
self.returncode = os.EX_OK |
283 |
self.wait() |
284 |
+ |
285 |
+ def get_binpkg_info(self): |
286 |
+ return self._binpkg_info |
287 |
diff --git a/pym/_emerge/EbuildBuild.py b/pym/_emerge/EbuildBuild.py |
288 |
index b5b1e87..0e98602 100644 |
289 |
--- a/pym/_emerge/EbuildBuild.py |
290 |
+++ b/pym/_emerge/EbuildBuild.py |
291 |
@@ -1,6 +1,10 @@ |
292 |
# Copyright 1999-2014 Gentoo Foundation |
293 |
# Distributed under the terms of the GNU General Public License v2 |
294 |
|
295 |
+from __future__ import unicode_literals |
296 |
+ |
297 |
+import io |
298 |
+ |
299 |
import _emerge.emergelog |
300 |
from _emerge.EbuildExecuter import EbuildExecuter |
301 |
from _emerge.EbuildPhase import EbuildPhase |
302 |
@@ -15,7 +19,7 @@ from _emerge.TaskSequence import TaskSequence |
303 |
|
304 |
from portage.util import writemsg |
305 |
import portage |
306 |
-from portage import os |
307 |
+from portage import _encodings, _unicode_decode, _unicode_encode, os |
308 |
from portage.output import colorize |
309 |
from portage.package.ebuild.digestcheck import digestcheck |
310 |
from portage.package.ebuild.digestgen import digestgen |
311 |
@@ -317,9 +321,13 @@ class EbuildBuild(CompositeTask): |
312 |
phase="rpm", scheduler=self.scheduler, |
313 |
settings=self.settings)) |
314 |
else: |
315 |
- binpkg_tasks.add(EbuildBinpkg(background=self.background, |
316 |
+ task = EbuildBinpkg( |
317 |
+ background=self.background, |
318 |
pkg=self.pkg, scheduler=self.scheduler, |
319 |
- settings=self.settings)) |
320 |
+ settings=self.settings) |
321 |
+ binpkg_tasks.add(task) |
322 |
+ task.addExitListener( |
323 |
+ self._record_binpkg_info) |
324 |
|
325 |
if binpkg_tasks: |
326 |
self._start_task(binpkg_tasks, self._buildpkg_exit) |
327 |
@@ -356,6 +364,28 @@ class EbuildBuild(CompositeTask): |
328 |
self.returncode = packager.returncode |
329 |
self.wait() |
330 |
|
331 |
+ def _record_binpkg_info(self, task): |
332 |
+ if task.returncode != os.EX_OK: |
333 |
+ return |
334 |
+ |
335 |
+ # Save info about the created binary package, so that |
336 |
+ # identifying information can be passed to the install |
337 |
+ # task, to be recorded in the installed package database. |
338 |
+ pkg = task.get_binpkg_info() |
339 |
+ infoloc = os.path.join(self.settings["PORTAGE_BUILDDIR"], |
340 |
+ "build-info") |
341 |
+ info = { |
342 |
+ "BINPKGMD5": "%s\n" % pkg._metadata["MD5"], |
343 |
+ } |
344 |
+ if pkg.build_id is not None: |
345 |
+ info["BUILD_ID"] = "%s\n" % pkg.build_id |
346 |
+ for k, v in info.items(): |
347 |
+ with io.open(_unicode_encode(os.path.join(infoloc, k), |
348 |
+ encoding=_encodings['fs'], errors='strict'), |
349 |
+ mode='w', encoding=_encodings['repo.content'], |
350 |
+ errors='strict') as f: |
351 |
+ f.write(v) |
352 |
+ |
353 |
def _buildpkgonly_success_hook_exit(self, success_hooks): |
354 |
self._default_exit(success_hooks) |
355 |
self.returncode = None |
356 |
diff --git a/pym/_emerge/Package.py b/pym/_emerge/Package.py |
357 |
index 975335d..2c1a116 100644 |
358 |
--- a/pym/_emerge/Package.py |
359 |
+++ b/pym/_emerge/Package.py |
360 |
@@ -219,6 +219,8 @@ class Package(Task): |
361 |
else: |
362 |
raise TypeError("root_config argument is required") |
363 |
|
364 |
+ elements = [type_name, root, _unicode(cpv), operation] |
365 |
+ |
366 |
# For installed (and binary) packages we don't care for the repo |
367 |
# when it comes to hashing, because there can only be one cpv. |
368 |
# So overwrite the repo_key with type_name. |
369 |
@@ -229,14 +231,22 @@ class Package(Task): |
370 |
raise AssertionError( |
371 |
"Package._gen_hash_key() " + \ |
372 |
"called without 'repo_name' argument") |
373 |
- repo_key = repo_name |
374 |
+ elements.append(repo_name) |
375 |
+ elif type_name == "binary": |
376 |
+ # Including a variety of fingerprints in the hash makes |
377 |
+ # it possible to simultaneously consider multiple similar |
378 |
+ # packages. Note that digests are not included here, since |
379 |
+ # they are relatively expensive to compute, and they may |
380 |
+ # not necessarily be available. |
381 |
+ elements.extend([cpv.build_id, cpv.file_size, |
382 |
+ cpv.build_time, cpv.mtime]) |
383 |
else: |
384 |
# For installed (and binary) packages we don't care for the repo |
385 |
# when it comes to hashing, because there can only be one cpv. |
386 |
# So overwrite the repo_key with type_name. |
387 |
- repo_key = type_name |
388 |
+ elements.append(type_name) |
389 |
|
390 |
- return (type_name, root, _unicode(cpv), operation, repo_key) |
391 |
+ return tuple(elements) |
392 |
|
393 |
def _validate_deps(self): |
394 |
""" |
395 |
diff --git a/pym/_emerge/Scheduler.py b/pym/_emerge/Scheduler.py |
396 |
index 6e3bf1a..6b39e3b 100644 |
397 |
--- a/pym/_emerge/Scheduler.py |
398 |
+++ b/pym/_emerge/Scheduler.py |
399 |
@@ -862,8 +862,12 @@ class Scheduler(PollScheduler): |
400 |
continue |
401 |
fetched = fetcher.pkg_path |
402 |
|
403 |
+ if fetched is False: |
404 |
+ filename = bintree.getname(x.cpv) |
405 |
+ else: |
406 |
+ filename = fetched |
407 |
verifier = BinpkgVerifier(pkg=x, |
408 |
- scheduler=sched_iface) |
409 |
+ scheduler=sched_iface, _pkg_path=filename) |
410 |
current_task = verifier |
411 |
verifier.start() |
412 |
if verifier.wait() != os.EX_OK: |
413 |
diff --git a/pym/_emerge/clear_caches.py b/pym/_emerge/clear_caches.py |
414 |
index 513df62..cb0db10 100644 |
415 |
--- a/pym/_emerge/clear_caches.py |
416 |
+++ b/pym/_emerge/clear_caches.py |
417 |
@@ -7,7 +7,6 @@ def clear_caches(trees): |
418 |
for d in trees.values(): |
419 |
d["porttree"].dbapi.melt() |
420 |
d["porttree"].dbapi._aux_cache.clear() |
421 |
- d["bintree"].dbapi._aux_cache.clear() |
422 |
d["bintree"].dbapi._clear_cache() |
423 |
if d["vartree"].dbapi._linkmap is None: |
424 |
# preserve-libs is entirely disabled |
425 |
diff --git a/pym/portage/const.py b/pym/portage/const.py |
426 |
index febdb4a..c7ecda2 100644 |
427 |
--- a/pym/portage/const.py |
428 |
+++ b/pym/portage/const.py |
429 |
@@ -122,6 +122,7 @@ EBUILD_PHASES = ( |
430 |
SUPPORTED_FEATURES = frozenset([ |
431 |
"assume-digests", |
432 |
"binpkg-logs", |
433 |
+ "binpkg-multi-instance", |
434 |
"buildpkg", |
435 |
"buildsyspkg", |
436 |
"candy", |
437 |
@@ -268,6 +269,7 @@ LIVE_ECLASSES = frozenset([ |
438 |
]) |
439 |
|
440 |
SUPPORTED_BINPKG_FORMATS = ("tar", "rpm") |
441 |
+SUPPORTED_XPAK_EXTENSIONS = (".tbz2", ".xpak") |
442 |
|
443 |
# Time formats used in various places like metadata.chk. |
444 |
TIMESTAMP_FORMAT = "%a, %d %b %Y %H:%M:%S +0000" # to be used with time.gmtime() |
445 |
diff --git a/pym/portage/dbapi/bintree.py b/pym/portage/dbapi/bintree.py |
446 |
index cd30b67..460b9f7 100644 |
447 |
--- a/pym/portage/dbapi/bintree.py |
448 |
+++ b/pym/portage/dbapi/bintree.py |
449 |
@@ -17,14 +17,13 @@ portage.proxy.lazyimport.lazyimport(globals(), |
450 |
'portage.update:update_dbentries', |
451 |
'portage.util:atomic_ofstream,ensure_dirs,normalize_path,' + \ |
452 |
'writemsg,writemsg_stdout', |
453 |
- 'portage.util.listdir:listdir', |
454 |
'portage.util.path:first_existing', |
455 |
'portage.util._urlopen:urlopen@_urlopen', |
456 |
'portage.versions:best,catpkgsplit,catsplit,_pkg_str', |
457 |
) |
458 |
|
459 |
from portage.cache.mappings import slot_dict_class |
460 |
-from portage.const import CACHE_PATH |
461 |
+from portage.const import CACHE_PATH, SUPPORTED_XPAK_EXTENSIONS |
462 |
from portage.dbapi.virtual import fakedbapi |
463 |
from portage.dep import Atom, use_reduce, paren_enclose |
464 |
from portage.exception import AlarmSignal, InvalidData, InvalidPackageName, \ |
465 |
@@ -71,18 +70,26 @@ class bindbapi(fakedbapi): |
466 |
_known_keys = frozenset(list(fakedbapi._known_keys) + \ |
467 |
["CHOST", "repository", "USE"]) |
468 |
def __init__(self, mybintree=None, **kwargs): |
469 |
- fakedbapi.__init__(self, **kwargs) |
470 |
+ # Always enable multi_instance mode for bindbapi indexing. This |
471 |
+ # does not affect the local PKGDIR file layout, since that is |
472 |
+ # controlled independently by FEATURES=binpkg-multi-instance. |
473 |
+ # The multi_instance mode is useful for the following reasons: |
474 |
+ # * binary packages with the same cpv from multiple binhosts |
475 |
+ # can be considered simultaneously |
476 |
+ # * if binpkg-multi-instance is disabled, it's still possible |
477 |
+ # to properly access a PKGDIR which has binpkg-multi-instance |
478 |
+ # layout (or mixed layout) |
479 |
+ fakedbapi.__init__(self, exclusive_slots=False, |
480 |
+ multi_instance=True, **kwargs) |
481 |
self.bintree = mybintree |
482 |
self.move_ent = mybintree.move_ent |
483 |
- self.cpvdict={} |
484 |
- self.cpdict={} |
485 |
# Selectively cache metadata in order to optimize dep matching. |
486 |
self._aux_cache_keys = set( |
487 |
- ["BUILD_TIME", "CHOST", "DEPEND", "EAPI", |
488 |
- "HDEPEND", "IUSE", "KEYWORDS", |
489 |
- "LICENSE", "PDEPEND", "PROPERTIES", "PROVIDE", |
490 |
- "RDEPEND", "repository", "RESTRICT", "SLOT", "USE", |
491 |
- "DEFINED_PHASES", "PROVIDES", "REQUIRES" |
492 |
+ ["BUILD_ID", "BUILD_TIME", "CHOST", "DEFINED_PHASES", |
493 |
+ "DEPEND", "EAPI", "HDEPEND", "IUSE", "KEYWORDS", |
494 |
+ "LICENSE", "MD5", "PDEPEND", "PROPERTIES", "PROVIDE", |
495 |
+ "PROVIDES", "RDEPEND", "repository", "REQUIRES", "RESTRICT", |
496 |
+ "SIZE", "SLOT", "USE", "_mtime_" |
497 |
]) |
498 |
self._aux_cache_slot_dict = slot_dict_class(self._aux_cache_keys) |
499 |
self._aux_cache = {} |
500 |
@@ -109,33 +116,49 @@ class bindbapi(fakedbapi): |
501 |
return fakedbapi.cpv_exists(self, cpv) |
502 |
|
503 |
def cpv_inject(self, cpv, **kwargs): |
504 |
- self._aux_cache.pop(cpv, None) |
505 |
- fakedbapi.cpv_inject(self, cpv, **kwargs) |
506 |
+ if not self.bintree.populated: |
507 |
+ self.bintree.populate() |
508 |
+ fakedbapi.cpv_inject(self, cpv, |
509 |
+ metadata=cpv._metadata, **kwargs) |
510 |
|
511 |
def cpv_remove(self, cpv): |
512 |
- self._aux_cache.pop(cpv, None) |
513 |
+ if not self.bintree.populated: |
514 |
+ self.bintree.populate() |
515 |
fakedbapi.cpv_remove(self, cpv) |
516 |
|
517 |
def aux_get(self, mycpv, wants, myrepo=None): |
518 |
if self.bintree and not self.bintree.populated: |
519 |
self.bintree.populate() |
520 |
- cache_me = False |
521 |
+ # Support plain string for backward compatibility with API |
522 |
+ # consumers (including portageq, which passes in a cpv from |
523 |
+ # a command-line argument). |
524 |
+ instance_key = self._instance_key(mycpv, |
525 |
+ support_string=True) |
526 |
if not self._known_keys.intersection( |
527 |
wants).difference(self._aux_cache_keys): |
528 |
- aux_cache = self._aux_cache.get(mycpv) |
529 |
+ aux_cache = self.cpvdict[instance_key] |
530 |
if aux_cache is not None: |
531 |
return [aux_cache.get(x, "") for x in wants] |
532 |
- cache_me = True |
533 |
mysplit = mycpv.split("/") |
534 |
mylist = [] |
535 |
tbz2name = mysplit[1]+".tbz2" |
536 |
if not self.bintree._remotepkgs or \ |
537 |
not self.bintree.isremote(mycpv): |
538 |
- tbz2_path = self.bintree.getname(mycpv) |
539 |
- if not os.path.exists(tbz2_path): |
540 |
+ try: |
541 |
+ tbz2_path = self.bintree._pkg_paths[instance_key] |
542 |
+ except KeyError: |
543 |
+ raise KeyError(mycpv) |
544 |
+ tbz2_path = os.path.join(self.bintree.pkgdir, tbz2_path) |
545 |
+ try: |
546 |
+ st = os.lstat(tbz2_path) |
547 |
+ except OSError: |
548 |
raise KeyError(mycpv) |
549 |
metadata_bytes = portage.xpak.tbz2(tbz2_path).get_data() |
550 |
def getitem(k): |
551 |
+ if k == "_mtime_": |
552 |
+ return _unicode(st[stat.ST_MTIME]) |
553 |
+ elif k == "SIZE": |
554 |
+ return _unicode(st.st_size) |
555 |
v = metadata_bytes.get(_unicode_encode(k, |
556 |
encoding=_encodings['repo.content'], |
557 |
errors='backslashreplace')) |
558 |
@@ -144,11 +167,9 @@ class bindbapi(fakedbapi): |
559 |
encoding=_encodings['repo.content'], errors='replace') |
560 |
return v |
561 |
else: |
562 |
- getitem = self.bintree._remotepkgs[mycpv].get |
563 |
+ getitem = self.cpvdict[instance_key].get |
564 |
mydata = {} |
565 |
mykeys = wants |
566 |
- if cache_me: |
567 |
- mykeys = self._aux_cache_keys.union(wants) |
568 |
for x in mykeys: |
569 |
myval = getitem(x) |
570 |
# myval is None if the key doesn't exist |
571 |
@@ -159,16 +180,24 @@ class bindbapi(fakedbapi): |
572 |
if not mydata.setdefault('EAPI', '0'): |
573 |
mydata['EAPI'] = '0' |
574 |
|
575 |
- if cache_me: |
576 |
- aux_cache = self._aux_cache_slot_dict() |
577 |
- for x in self._aux_cache_keys: |
578 |
- aux_cache[x] = mydata.get(x, '') |
579 |
- self._aux_cache[mycpv] = aux_cache |
580 |
return [mydata.get(x, '') for x in wants] |
581 |
|
582 |
def aux_update(self, cpv, values): |
583 |
if not self.bintree.populated: |
584 |
self.bintree.populate() |
585 |
+ build_id = None |
586 |
+ try: |
587 |
+ build_id = cpv.build_id |
588 |
+ except AttributeError: |
589 |
+ if self.bintree._multi_instance: |
590 |
+ # The cpv.build_id attribute is required if we are in |
591 |
+ # multi-instance mode, since otherwise we won't know |
592 |
+ # which instance to update. |
593 |
+ raise |
594 |
+ else: |
595 |
+ cpv = self._instance_key(cpv, support_string=True)[0] |
596 |
+ build_id = cpv.build_id |
597 |
+ |
598 |
tbz2path = self.bintree.getname(cpv) |
599 |
if not os.path.exists(tbz2path): |
600 |
raise KeyError(cpv) |
601 |
@@ -187,7 +216,7 @@ class bindbapi(fakedbapi): |
602 |
del mydata[k] |
603 |
mytbz2.recompose_mem(portage.xpak.xpak_mem(mydata)) |
604 |
# inject will clear stale caches via cpv_inject. |
605 |
- self.bintree.inject(cpv) |
606 |
+ self.bintree.inject(cpv, filename=tbz2path) |
607 |
|
608 |
def cp_list(self, *pargs, **kwargs): |
609 |
if not self.bintree.populated: |
610 |
@@ -219,7 +248,7 @@ class bindbapi(fakedbapi): |
611 |
if not self.bintree.isremote(pkg): |
612 |
pass |
613 |
else: |
614 |
- metadata = self.bintree._remotepkgs[pkg] |
615 |
+ metadata = self.bintree._remotepkgs[self._instance_key(pkg)] |
616 |
try: |
617 |
size = int(metadata["SIZE"]) |
618 |
except KeyError: |
619 |
@@ -300,6 +329,13 @@ class binarytree(object): |
620 |
|
621 |
if True: |
622 |
self.pkgdir = normalize_path(pkgdir) |
623 |
+ # NOTE: Event if binpkg-multi-instance is disabled, it's |
624 |
+ # still possible to access a PKGDIR which uses the |
625 |
+ # binpkg-multi-instance layout (or mixed layout). |
626 |
+ self._multi_instance = ("binpkg-multi-instance" in |
627 |
+ settings.features) |
628 |
+ if self._multi_instance: |
629 |
+ self._allocate_filename = self._allocate_filename_multi |
630 |
self.dbapi = bindbapi(self, settings=settings) |
631 |
self.update_ents = self.dbapi.update_ents |
632 |
self.move_slot_ent = self.dbapi.move_slot_ent |
633 |
@@ -310,7 +346,6 @@ class binarytree(object): |
634 |
self.invalids = [] |
635 |
self.settings = settings |
636 |
self._pkg_paths = {} |
637 |
- self._pkgindex_uri = {} |
638 |
self._populating = False |
639 |
self._all_directory = os.path.isdir( |
640 |
os.path.join(self.pkgdir, "All")) |
641 |
@@ -318,12 +353,14 @@ class binarytree(object): |
642 |
self._pkgindex_hashes = ["MD5","SHA1"] |
643 |
self._pkgindex_file = os.path.join(self.pkgdir, "Packages") |
644 |
self._pkgindex_keys = self.dbapi._aux_cache_keys.copy() |
645 |
- self._pkgindex_keys.update(["CPV", "MTIME", "SIZE"]) |
646 |
+ self._pkgindex_keys.update(["CPV", "SIZE"]) |
647 |
self._pkgindex_aux_keys = \ |
648 |
- ["BUILD_TIME", "CHOST", "DEPEND", "DESCRIPTION", "EAPI", |
649 |
- "HDEPEND", "IUSE", "KEYWORDS", "LICENSE", "PDEPEND", "PROPERTIES", |
650 |
- "PROVIDE", "RESTRICT", "RDEPEND", "repository", "SLOT", "USE", "DEFINED_PHASES", |
651 |
- "BASE_URI", "PROVIDES", "REQUIRES"] |
652 |
+ ["BASE_URI", "BUILD_ID", "BUILD_TIME", "CHOST", |
653 |
+ "DEFINED_PHASES", "DEPEND", "DESCRIPTION", "EAPI", |
654 |
+ "HDEPEND", "IUSE", "KEYWORDS", "LICENSE", "PDEPEND", |
655 |
+ "PKGINDEX_URI", "PROPERTIES", "PROVIDE", "PROVIDES", |
656 |
+ "RDEPEND", "repository", "REQUIRES", "RESTRICT", |
657 |
+ "SIZE", "SLOT", "USE"] |
658 |
self._pkgindex_aux_keys = list(self._pkgindex_aux_keys) |
659 |
self._pkgindex_use_evaluated_keys = \ |
660 |
("DEPEND", "HDEPEND", "LICENSE", "RDEPEND", |
661 |
@@ -336,6 +373,7 @@ class binarytree(object): |
662 |
"USE_EXPAND", "USE_EXPAND_HIDDEN", "USE_EXPAND_IMPLICIT", |
663 |
"USE_EXPAND_UNPREFIXED"]) |
664 |
self._pkgindex_default_pkg_data = { |
665 |
+ "BUILD_ID" : "", |
666 |
"BUILD_TIME" : "", |
667 |
"DEFINED_PHASES" : "", |
668 |
"DEPEND" : "", |
669 |
@@ -365,6 +403,7 @@ class binarytree(object): |
670 |
|
671 |
self._pkgindex_translated_keys = ( |
672 |
("DESCRIPTION" , "DESC"), |
673 |
+ ("_mtime_" , "MTIME"), |
674 |
("repository" , "REPO"), |
675 |
) |
676 |
|
677 |
@@ -455,16 +494,21 @@ class binarytree(object): |
678 |
mytbz2.recompose_mem(portage.xpak.xpak_mem(mydata)) |
679 |
|
680 |
self.dbapi.cpv_remove(mycpv) |
681 |
- del self._pkg_paths[mycpv] |
682 |
+ del self._pkg_paths[self.dbapi._instance_key(mycpv)] |
683 |
+ metadata = self.dbapi._aux_cache_slot_dict() |
684 |
+ for k in self.dbapi._aux_cache_keys: |
685 |
+ v = mydata.get(_unicode_encode(k)) |
686 |
+ if v is not None: |
687 |
+ v = _unicode_decode(v) |
688 |
+ metadata[k] = " ".join(v.split()) |
689 |
+ mynewcpv = _pkg_str(mynewcpv, metadata=metadata) |
690 |
new_path = self.getname(mynewcpv) |
691 |
- self._pkg_paths[mynewcpv] = os.path.join( |
692 |
+ self._pkg_paths[ |
693 |
+ self.dbapi._instance_key(mynewcpv)] = os.path.join( |
694 |
*new_path.split(os.path.sep)[-2:]) |
695 |
if new_path != mytbz2: |
696 |
self._ensure_dir(os.path.dirname(new_path)) |
697 |
_movefile(tbz2path, new_path, mysettings=self.settings) |
698 |
- self._remove_symlink(mycpv) |
699 |
- if new_path.split(os.path.sep)[-2] == "All": |
700 |
- self._create_symlink(mynewcpv) |
701 |
self.inject(mynewcpv) |
702 |
|
703 |
return moves |
704 |
@@ -645,55 +689,63 @@ class binarytree(object): |
705 |
# prior to performing package moves since it only wants to |
706 |
# operate on local packages (getbinpkgs=0). |
707 |
self._remotepkgs = None |
708 |
- self.dbapi._clear_cache() |
709 |
- self.dbapi._aux_cache.clear() |
710 |
+ self.dbapi.clear() |
711 |
+ _instance_key = self.dbapi._instance_key |
712 |
if True: |
713 |
pkg_paths = {} |
714 |
self._pkg_paths = pkg_paths |
715 |
- dirs = listdir(self.pkgdir, dirsonly=True, EmptyOnError=True) |
716 |
- if "All" in dirs: |
717 |
- dirs.remove("All") |
718 |
- dirs.sort() |
719 |
- dirs.insert(0, "All") |
720 |
+ dir_files = {} |
721 |
+ for parent, dir_names, file_names in os.walk(self.pkgdir): |
722 |
+ relative_parent = parent[len(self.pkgdir)+1:] |
723 |
+ dir_files[relative_parent] = file_names |
724 |
+ |
725 |
pkgindex = self._load_pkgindex() |
726 |
- pf_index = None |
727 |
if not self._pkgindex_version_supported(pkgindex): |
728 |
pkgindex = self._new_pkgindex() |
729 |
header = pkgindex.header |
730 |
metadata = {} |
731 |
+ basename_index = {} |
732 |
for d in pkgindex.packages: |
733 |
- metadata[d["CPV"]] = d |
734 |
+ cpv = _pkg_str(d["CPV"], metadata=d, |
735 |
+ settings=self.settings) |
736 |
+ d["CPV"] = cpv |
737 |
+ metadata[_instance_key(cpv)] = d |
738 |
+ path = d.get("PATH") |
739 |
+ if not path: |
740 |
+ path = cpv + ".tbz2" |
741 |
+ basename = os.path.basename(path) |
742 |
+ basename_index.setdefault(basename, []).append(d) |
743 |
+ |
744 |
update_pkgindex = False |
745 |
- for mydir in dirs: |
746 |
- for myfile in listdir(os.path.join(self.pkgdir, mydir)): |
747 |
- if not myfile.endswith(".tbz2"): |
748 |
+ for mydir, file_names in dir_files.items(): |
749 |
+ try: |
750 |
+ mydir = _unicode_decode(mydir, |
751 |
+ encoding=_encodings["fs"], errors="strict") |
752 |
+ except UnicodeDecodeError: |
753 |
+ continue |
754 |
+ for myfile in file_names: |
755 |
+ try: |
756 |
+ myfile = _unicode_decode(myfile, |
757 |
+ encoding=_encodings["fs"], errors="strict") |
758 |
+ except UnicodeDecodeError: |
759 |
+ continue |
760 |
+ if not myfile.endswith(SUPPORTED_XPAK_EXTENSIONS): |
761 |
continue |
762 |
mypath = os.path.join(mydir, myfile) |
763 |
full_path = os.path.join(self.pkgdir, mypath) |
764 |
s = os.lstat(full_path) |
765 |
- if stat.S_ISLNK(s.st_mode): |
766 |
+ |
767 |
+ if not stat.S_ISREG(s.st_mode): |
768 |
continue |
769 |
|
770 |
# Validate data from the package index and try to avoid |
771 |
# reading the xpak if possible. |
772 |
- if mydir != "All": |
773 |
- possibilities = None |
774 |
- d = metadata.get(mydir+"/"+myfile[:-5]) |
775 |
- if d: |
776 |
- possibilities = [d] |
777 |
- else: |
778 |
- if pf_index is None: |
779 |
- pf_index = {} |
780 |
- for mycpv in metadata: |
781 |
- mycat, mypf = catsplit(mycpv) |
782 |
- pf_index.setdefault( |
783 |
- mypf, []).append(metadata[mycpv]) |
784 |
- possibilities = pf_index.get(myfile[:-5]) |
785 |
+ possibilities = basename_index.get(myfile) |
786 |
if possibilities: |
787 |
match = None |
788 |
for d in possibilities: |
789 |
try: |
790 |
- if long(d["MTIME"]) != s[stat.ST_MTIME]: |
791 |
+ if long(d["_mtime_"]) != s[stat.ST_MTIME]: |
792 |
continue |
793 |
except (KeyError, ValueError): |
794 |
continue |
795 |
@@ -707,15 +759,14 @@ class binarytree(object): |
796 |
break |
797 |
if match: |
798 |
mycpv = match["CPV"] |
799 |
- if mycpv in pkg_paths: |
800 |
- # discard duplicates (All/ is preferred) |
801 |
- continue |
802 |
- mycpv = _pkg_str(mycpv) |
803 |
- pkg_paths[mycpv] = mypath |
804 |
+ instance_key = _instance_key(mycpv) |
805 |
+ pkg_paths[instance_key] = mypath |
806 |
# update the path if the package has been moved |
807 |
oldpath = d.get("PATH") |
808 |
if oldpath and oldpath != mypath: |
809 |
update_pkgindex = True |
810 |
+ # Omit PATH if it is the default path for |
811 |
+ # the current Packages format version. |
812 |
if mypath != mycpv + ".tbz2": |
813 |
d["PATH"] = mypath |
814 |
if not oldpath: |
815 |
@@ -725,11 +776,6 @@ class binarytree(object): |
816 |
if oldpath: |
817 |
update_pkgindex = True |
818 |
self.dbapi.cpv_inject(mycpv) |
819 |
- if not self.dbapi._aux_cache_keys.difference(d): |
820 |
- aux_cache = self.dbapi._aux_cache_slot_dict() |
821 |
- for k in self.dbapi._aux_cache_keys: |
822 |
- aux_cache[k] = d[k] |
823 |
- self.dbapi._aux_cache[mycpv] = aux_cache |
824 |
continue |
825 |
if not os.access(full_path, os.R_OK): |
826 |
writemsg(_("!!! Permission denied to read " \ |
827 |
@@ -737,13 +783,12 @@ class binarytree(object): |
828 |
noiselevel=-1) |
829 |
self.invalids.append(myfile[:-5]) |
830 |
continue |
831 |
- metadata_bytes = portage.xpak.tbz2(full_path).get_data() |
832 |
- mycat = _unicode_decode(metadata_bytes.get(b"CATEGORY", ""), |
833 |
- encoding=_encodings['repo.content'], errors='replace') |
834 |
- mypf = _unicode_decode(metadata_bytes.get(b"PF", ""), |
835 |
- encoding=_encodings['repo.content'], errors='replace') |
836 |
- slot = _unicode_decode(metadata_bytes.get(b"SLOT", ""), |
837 |
- encoding=_encodings['repo.content'], errors='replace') |
838 |
+ pkg_metadata = self._read_metadata(full_path, s, |
839 |
+ keys=chain(self.dbapi._aux_cache_keys, |
840 |
+ ("PF", "CATEGORY"))) |
841 |
+ mycat = pkg_metadata.get("CATEGORY", "") |
842 |
+ mypf = pkg_metadata.get("PF", "") |
843 |
+ slot = pkg_metadata.get("SLOT", "") |
844 |
mypkg = myfile[:-5] |
845 |
if not mycat or not mypf or not slot: |
846 |
#old-style or corrupt package |
847 |
@@ -767,16 +812,51 @@ class binarytree(object): |
848 |
writemsg("!!! %s\n" % line, noiselevel=-1) |
849 |
self.invalids.append(mypkg) |
850 |
continue |
851 |
- mycat = mycat.strip() |
852 |
- slot = slot.strip() |
853 |
- if mycat != mydir and mydir != "All": |
854 |
+ |
855 |
+ multi_instance = False |
856 |
+ invalid_name = False |
857 |
+ build_id = None |
858 |
+ if myfile.endswith(".xpak"): |
859 |
+ multi_instance = True |
860 |
+ build_id = self._parse_build_id(myfile) |
861 |
+ if build_id < 1: |
862 |
+ invalid_name = True |
863 |
+ elif myfile != "%s-%s.xpak" % ( |
864 |
+ mypf, build_id): |
865 |
+ invalid_name = True |
866 |
+ else: |
867 |
+ mypkg = mypkg[:-len(str(build_id))-1] |
868 |
+ elif myfile != mypf + ".tbz2": |
869 |
+ invalid_name = True |
870 |
+ |
871 |
+ if invalid_name: |
872 |
+ writemsg(_("\n!!! Binary package name is " |
873 |
+ "invalid: '%s'\n") % full_path, |
874 |
+ noiselevel=-1) |
875 |
+ continue |
876 |
+ |
877 |
+ if pkg_metadata.get("BUILD_ID"): |
878 |
+ try: |
879 |
+ build_id = long(pkg_metadata["BUILD_ID"]) |
880 |
+ except ValueError: |
881 |
+ writemsg(_("!!! Binary package has " |
882 |
+ "invalid BUILD_ID: '%s'\n") % |
883 |
+ full_path, noiselevel=-1) |
884 |
+ continue |
885 |
+ else: |
886 |
+ build_id = None |
887 |
+ |
888 |
+ if multi_instance: |
889 |
+ name_split = catpkgsplit("%s/%s" % |
890 |
+ (mycat, mypf)) |
891 |
+ if (name_split is None or |
892 |
+ tuple(catsplit(mydir)) != name_split[:2]): |
893 |
+ continue |
894 |
+ elif mycat != mydir and mydir != "All": |
895 |
continue |
896 |
if mypkg != mypf.strip(): |
897 |
continue |
898 |
mycpv = mycat + "/" + mypkg |
899 |
- if mycpv in pkg_paths: |
900 |
- # All is first, so it's preferred. |
901 |
- continue |
902 |
if not self.dbapi._category_re.match(mycat): |
903 |
writemsg(_("!!! Binary package has an " \ |
904 |
"unrecognized category: '%s'\n") % full_path, |
905 |
@@ -786,14 +866,23 @@ class binarytree(object): |
906 |
(mycpv, self.settings["PORTAGE_CONFIGROOT"]), |
907 |
noiselevel=-1) |
908 |
continue |
909 |
- mycpv = _pkg_str(mycpv) |
910 |
- pkg_paths[mycpv] = mypath |
911 |
+ if build_id is not None: |
912 |
+ pkg_metadata["BUILD_ID"] = _unicode(build_id) |
913 |
+ pkg_metadata["SIZE"] = _unicode(s.st_size) |
914 |
+ # Discard items used only for validation above. |
915 |
+ pkg_metadata.pop("CATEGORY") |
916 |
+ pkg_metadata.pop("PF") |
917 |
+ mycpv = _pkg_str(mycpv, |
918 |
+ metadata=self.dbapi._aux_cache_slot_dict( |
919 |
+ pkg_metadata)) |
920 |
+ pkg_paths[_instance_key(mycpv)] = mypath |
921 |
self.dbapi.cpv_inject(mycpv) |
922 |
update_pkgindex = True |
923 |
- d = metadata.get(mycpv, {}) |
924 |
+ d = metadata.get(_instance_key(mycpv), |
925 |
+ pkgindex._pkg_slot_dict()) |
926 |
if d: |
927 |
try: |
928 |
- if long(d["MTIME"]) != s[stat.ST_MTIME]: |
929 |
+ if long(d["_mtime_"]) != s[stat.ST_MTIME]: |
930 |
d.clear() |
931 |
except (KeyError, ValueError): |
932 |
d.clear() |
933 |
@@ -804,36 +893,30 @@ class binarytree(object): |
934 |
except (KeyError, ValueError): |
935 |
d.clear() |
936 |
|
937 |
+ for k in self._pkgindex_allowed_pkg_keys: |
938 |
+ v = pkg_metadata.get(k) |
939 |
+ if v is not None: |
940 |
+ d[k] = v |
941 |
d["CPV"] = mycpv |
942 |
- d["SLOT"] = slot |
943 |
- d["MTIME"] = _unicode(s[stat.ST_MTIME]) |
944 |
- d["SIZE"] = _unicode(s.st_size) |
945 |
|
946 |
- d.update(zip(self._pkgindex_aux_keys, |
947 |
- self.dbapi.aux_get(mycpv, self._pkgindex_aux_keys))) |
948 |
try: |
949 |
self._eval_use_flags(mycpv, d) |
950 |
except portage.exception.InvalidDependString: |
951 |
writemsg(_("!!! Invalid binary package: '%s'\n") % \ |
952 |
self.getname(mycpv), noiselevel=-1) |
953 |
self.dbapi.cpv_remove(mycpv) |
954 |
- del pkg_paths[mycpv] |
955 |
+ del pkg_paths[_instance_key(mycpv)] |
956 |
|
957 |
# record location if it's non-default |
958 |
if mypath != mycpv + ".tbz2": |
959 |
d["PATH"] = mypath |
960 |
else: |
961 |
d.pop("PATH", None) |
962 |
- metadata[mycpv] = d |
963 |
- if not self.dbapi._aux_cache_keys.difference(d): |
964 |
- aux_cache = self.dbapi._aux_cache_slot_dict() |
965 |
- for k in self.dbapi._aux_cache_keys: |
966 |
- aux_cache[k] = d[k] |
967 |
- self.dbapi._aux_cache[mycpv] = aux_cache |
968 |
+ metadata[_instance_key(mycpv)] = d |
969 |
|
970 |
- for cpv in list(metadata): |
971 |
- if cpv not in pkg_paths: |
972 |
- del metadata[cpv] |
973 |
+ for instance_key in list(metadata): |
974 |
+ if instance_key not in pkg_paths: |
975 |
+ del metadata[instance_key] |
976 |
|
977 |
# Do not bother to write the Packages index if $PKGDIR/All/ exists |
978 |
# since it will provide no benefit due to the need to read CATEGORY |
979 |
@@ -1058,45 +1141,24 @@ class binarytree(object): |
980 |
# The current user doesn't have permission to cache the |
981 |
# file, but that's alright. |
982 |
if pkgindex: |
983 |
- # Organize remote package list as a cpv -> metadata map. |
984 |
- remotepkgs = _pkgindex_cpv_map_latest_build(pkgindex) |
985 |
remote_base_uri = pkgindex.header.get("URI", base_url) |
986 |
- for cpv, remote_metadata in remotepkgs.items(): |
987 |
- remote_metadata["BASE_URI"] = remote_base_uri |
988 |
- self._pkgindex_uri[cpv] = url |
989 |
- self._remotepkgs.update(remotepkgs) |
990 |
- self._remote_has_index = True |
991 |
- for cpv in remotepkgs: |
992 |
+ for d in pkgindex.packages: |
993 |
+ cpv = _pkg_str(d["CPV"], metadata=d, |
994 |
+ settings=self.settings) |
995 |
+ instance_key = _instance_key(cpv) |
996 |
+ # Local package instances override remote instances |
997 |
+ # with the same instance_key. |
998 |
+ if instance_key in metadata: |
999 |
+ continue |
1000 |
+ |
1001 |
+ d["CPV"] = cpv |
1002 |
+ d["BASE_URI"] = remote_base_uri |
1003 |
+ d["PKGINDEX_URI"] = url |
1004 |
+ self._remotepkgs[instance_key] = d |
1005 |
+ metadata[instance_key] = d |
1006 |
self.dbapi.cpv_inject(cpv) |
1007 |
- if True: |
1008 |
- # Remote package instances override local package |
1009 |
- # if they are not identical. |
1010 |
- hash_names = ["SIZE"] + self._pkgindex_hashes |
1011 |
- for cpv, local_metadata in metadata.items(): |
1012 |
- remote_metadata = self._remotepkgs.get(cpv) |
1013 |
- if remote_metadata is None: |
1014 |
- continue |
1015 |
- # Use digests to compare identity. |
1016 |
- identical = True |
1017 |
- for hash_name in hash_names: |
1018 |
- local_value = local_metadata.get(hash_name) |
1019 |
- if local_value is None: |
1020 |
- continue |
1021 |
- remote_value = remote_metadata.get(hash_name) |
1022 |
- if remote_value is None: |
1023 |
- continue |
1024 |
- if local_value != remote_value: |
1025 |
- identical = False |
1026 |
- break |
1027 |
- if identical: |
1028 |
- del self._remotepkgs[cpv] |
1029 |
- else: |
1030 |
- # Override the local package in the aux_get cache. |
1031 |
- self.dbapi._aux_cache[cpv] = remote_metadata |
1032 |
- else: |
1033 |
- # Local package instances override remote instances. |
1034 |
- for cpv in metadata: |
1035 |
- self._remotepkgs.pop(cpv, None) |
1036 |
+ |
1037 |
+ self._remote_has_index = True |
1038 |
|
1039 |
self.populated=1 |
1040 |
|
1041 |
@@ -1108,7 +1170,8 @@ class binarytree(object): |
1042 |
@param filename: File path of the package to inject, or None if it's |
1043 |
already in the location returned by getname() |
1044 |
@type filename: string |
1045 |
- @rtype: None |
1046 |
+ @rtype: _pkg_str or None |
1047 |
+ @return: A _pkg_str instance on success, or None on failure. |
1048 |
""" |
1049 |
mycat, mypkg = catsplit(cpv) |
1050 |
if not self.populated: |
1051 |
@@ -1126,24 +1189,45 @@ class binarytree(object): |
1052 |
writemsg(_("!!! Binary package does not exist: '%s'\n") % full_path, |
1053 |
noiselevel=-1) |
1054 |
return |
1055 |
- mytbz2 = portage.xpak.tbz2(full_path) |
1056 |
- slot = mytbz2.getfile("SLOT") |
1057 |
+ metadata = self._read_metadata(full_path, s) |
1058 |
+ slot = metadata.get("SLOT") |
1059 |
+ try: |
1060 |
+ self._eval_use_flags(cpv, metadata) |
1061 |
+ except portage.exception.InvalidDependString: |
1062 |
+ slot = None |
1063 |
if slot is None: |
1064 |
writemsg(_("!!! Invalid binary package: '%s'\n") % full_path, |
1065 |
noiselevel=-1) |
1066 |
return |
1067 |
- slot = slot.strip() |
1068 |
- self.dbapi.cpv_inject(cpv) |
1069 |
+ |
1070 |
+ fetched = False |
1071 |
+ try: |
1072 |
+ build_id = cpv.build_id |
1073 |
+ except AttributeError: |
1074 |
+ build_id = None |
1075 |
+ else: |
1076 |
+ instance_key = self.dbapi._instance_key(cpv) |
1077 |
+ if instance_key in self.dbapi.cpvdict: |
1078 |
+ # This means we've been called by aux_update (or |
1079 |
+ # similar). The instance key typically changes (due to |
1080 |
+ # file modification), so we need to discard existing |
1081 |
+ # instance key references. |
1082 |
+ self.dbapi.cpv_remove(cpv) |
1083 |
+ self._pkg_paths.pop(instance_key, None) |
1084 |
+ if self._remotepkgs is not None: |
1085 |
+ fetched = self._remotepkgs.pop(instance_key, None) |
1086 |
+ |
1087 |
+ cpv = _pkg_str(cpv, metadata=metadata, settings=self.settings) |
1088 |
|
1089 |
# Reread the Packages index (in case it's been changed by another |
1090 |
# process) and then updated it, all while holding a lock. |
1091 |
pkgindex_lock = None |
1092 |
- created_symlink = False |
1093 |
try: |
1094 |
pkgindex_lock = lockfile(self._pkgindex_file, |
1095 |
wantnewlockfile=1) |
1096 |
if filename is not None: |
1097 |
- new_filename = self.getname(cpv) |
1098 |
+ new_filename = self.getname(cpv, |
1099 |
+ allocate_new=(build_id is None)) |
1100 |
try: |
1101 |
samefile = os.path.samefile(filename, new_filename) |
1102 |
except OSError: |
1103 |
@@ -1153,54 +1237,31 @@ class binarytree(object): |
1104 |
_movefile(filename, new_filename, mysettings=self.settings) |
1105 |
full_path = new_filename |
1106 |
|
1107 |
- self._file_permissions(full_path) |
1108 |
+ basename = os.path.basename(full_path) |
1109 |
+ pf = catsplit(cpv)[1] |
1110 |
+ if (build_id is None and not fetched and |
1111 |
+ basename.endswith(".xpak")): |
1112 |
+ # Apply the newly assigned BUILD_ID. This is intended |
1113 |
+ # to occur only for locally built packages. If the |
1114 |
+ # package was fetched, we want to preserve its |
1115 |
+ # attributes, so that we can later distinguish that it |
1116 |
+ # is identical to its remote counterpart. |
1117 |
+ build_id = self._parse_build_id(basename) |
1118 |
+ metadata["BUILD_ID"] = _unicode(build_id) |
1119 |
+ cpv = _pkg_str(cpv, metadata=metadata, |
1120 |
+ settings=self.settings) |
1121 |
+ binpkg = portage.xpak.tbz2(full_path) |
1122 |
+ binary_data = binpkg.get_data() |
1123 |
+ binary_data[b"BUILD_ID"] = _unicode_encode( |
1124 |
+ metadata["BUILD_ID"]) |
1125 |
+ binpkg.recompose_mem(portage.xpak.xpak_mem(binary_data)) |
1126 |
|
1127 |
- if self._all_directory and \ |
1128 |
- self.getname(cpv).split(os.path.sep)[-2] == "All": |
1129 |
- self._create_symlink(cpv) |
1130 |
- created_symlink = True |
1131 |
+ self._file_permissions(full_path) |
1132 |
pkgindex = self._load_pkgindex() |
1133 |
- |
1134 |
if not self._pkgindex_version_supported(pkgindex): |
1135 |
pkgindex = self._new_pkgindex() |
1136 |
|
1137 |
- # Discard remote metadata to ensure that _pkgindex_entry |
1138 |
- # gets the local metadata. This also updates state for future |
1139 |
- # isremote calls. |
1140 |
- if self._remotepkgs is not None: |
1141 |
- self._remotepkgs.pop(cpv, None) |
1142 |
- |
1143 |
- # Discard cached metadata to ensure that _pkgindex_entry |
1144 |
- # doesn't return stale metadata. |
1145 |
- self.dbapi._aux_cache.pop(cpv, None) |
1146 |
- |
1147 |
- try: |
1148 |
- d = self._pkgindex_entry(cpv) |
1149 |
- except portage.exception.InvalidDependString: |
1150 |
- writemsg(_("!!! Invalid binary package: '%s'\n") % \ |
1151 |
- self.getname(cpv), noiselevel=-1) |
1152 |
- self.dbapi.cpv_remove(cpv) |
1153 |
- del self._pkg_paths[cpv] |
1154 |
- return |
1155 |
- |
1156 |
- # If found, remove package(s) with duplicate path. |
1157 |
- path = d.get("PATH", "") |
1158 |
- for i in range(len(pkgindex.packages) - 1, -1, -1): |
1159 |
- d2 = pkgindex.packages[i] |
1160 |
- if path and path == d2.get("PATH"): |
1161 |
- # Handle path collisions in $PKGDIR/All |
1162 |
- # when CPV is not identical. |
1163 |
- del pkgindex.packages[i] |
1164 |
- elif cpv == d2.get("CPV"): |
1165 |
- if path == d2.get("PATH", ""): |
1166 |
- del pkgindex.packages[i] |
1167 |
- elif created_symlink and not d2.get("PATH", ""): |
1168 |
- # Delete entry for the package that was just |
1169 |
- # overwritten by a symlink to this package. |
1170 |
- del pkgindex.packages[i] |
1171 |
- |
1172 |
- pkgindex.packages.append(d) |
1173 |
- |
1174 |
+ d = self._inject_file(pkgindex, cpv, full_path) |
1175 |
self._update_pkgindex_header(pkgindex.header) |
1176 |
self._pkgindex_write(pkgindex) |
1177 |
|
1178 |
@@ -1208,6 +1269,73 @@ class binarytree(object): |
1179 |
if pkgindex_lock: |
1180 |
unlockfile(pkgindex_lock) |
1181 |
|
1182 |
+ # This is used to record BINPKGMD5 in the installed package |
1183 |
+ # database, for a package that has just been built. |
1184 |
+ cpv._metadata["MD5"] = d["MD5"] |
1185 |
+ |
1186 |
+ return cpv |
1187 |
+ |
1188 |
+ def _read_metadata(self, filename, st, keys=None): |
1189 |
+ if keys is None: |
1190 |
+ keys = self.dbapi._aux_cache_keys |
1191 |
+ metadata = self.dbapi._aux_cache_slot_dict() |
1192 |
+ else: |
1193 |
+ metadata = {} |
1194 |
+ binary_metadata = portage.xpak.tbz2(filename).get_data() |
1195 |
+ for k in keys: |
1196 |
+ if k == "_mtime_": |
1197 |
+ metadata[k] = _unicode(st[stat.ST_MTIME]) |
1198 |
+ elif k == "SIZE": |
1199 |
+ metadata[k] = _unicode(st.st_size) |
1200 |
+ else: |
1201 |
+ v = binary_metadata.get(_unicode_encode(k)) |
1202 |
+ if v is not None: |
1203 |
+ v = _unicode_decode(v) |
1204 |
+ metadata[k] = " ".join(v.split()) |
1205 |
+ metadata.setdefault("EAPI", "0") |
1206 |
+ return metadata |
1207 |
+ |
1208 |
+ def _inject_file(self, pkgindex, cpv, filename): |
1209 |
+ """ |
1210 |
+ Add a package to internal data structures, and add an |
1211 |
+ entry to the given pkgindex. |
1212 |
+ @param pkgindex: The PackageIndex instance to which an entry |
1213 |
+ will be added. |
1214 |
+ @type pkgindex: PackageIndex |
1215 |
+ @param cpv: A _pkg_str instance corresponding to the package |
1216 |
+ being injected. |
1217 |
+ @type cpv: _pkg_str |
1218 |
+ @param filename: Absolute file path of the package to inject. |
1219 |
+ @type filename: string |
1220 |
+ @rtype: dict |
1221 |
+ @return: A dict corresponding to the new entry which has been |
1222 |
+ added to pkgindex. This may be used to access the checksums |
1223 |
+ which have just been generated. |
1224 |
+ """ |
1225 |
+ # Update state for future isremote calls. |
1226 |
+ instance_key = self.dbapi._instance_key(cpv) |
1227 |
+ if self._remotepkgs is not None: |
1228 |
+ self._remotepkgs.pop(instance_key, None) |
1229 |
+ |
1230 |
+ self.dbapi.cpv_inject(cpv) |
1231 |
+ self._pkg_paths[instance_key] = filename[len(self.pkgdir)+1:] |
1232 |
+ d = self._pkgindex_entry(cpv) |
1233 |
+ |
1234 |
+ # If found, remove package(s) with duplicate path. |
1235 |
+ path = d.get("PATH", "") |
1236 |
+ for i in range(len(pkgindex.packages) - 1, -1, -1): |
1237 |
+ d2 = pkgindex.packages[i] |
1238 |
+ if path and path == d2.get("PATH"): |
1239 |
+ # Handle path collisions in $PKGDIR/All |
1240 |
+ # when CPV is not identical. |
1241 |
+ del pkgindex.packages[i] |
1242 |
+ elif cpv == d2.get("CPV"): |
1243 |
+ if path == d2.get("PATH", ""): |
1244 |
+ del pkgindex.packages[i] |
1245 |
+ |
1246 |
+ pkgindex.packages.append(d) |
1247 |
+ return d |
1248 |
+ |
1249 |
def _pkgindex_write(self, pkgindex): |
1250 |
contents = codecs.getwriter(_encodings['repo.content'])(io.BytesIO()) |
1251 |
pkgindex.write(contents) |
1252 |
@@ -1233,7 +1361,7 @@ class binarytree(object): |
1253 |
|
1254 |
def _pkgindex_entry(self, cpv): |
1255 |
""" |
1256 |
- Performs checksums and evaluates USE flag conditionals. |
1257 |
+ Performs checksums, and gets size and mtime via lstat. |
1258 |
Raises InvalidDependString if necessary. |
1259 |
@rtype: dict |
1260 |
@return: a dict containing entry for the give cpv. |
1261 |
@@ -1241,23 +1369,20 @@ class binarytree(object): |
1262 |
|
1263 |
pkg_path = self.getname(cpv) |
1264 |
|
1265 |
- d = dict(zip(self._pkgindex_aux_keys, |
1266 |
- self.dbapi.aux_get(cpv, self._pkgindex_aux_keys))) |
1267 |
- |
1268 |
+ d = dict(cpv._metadata.items()) |
1269 |
d.update(perform_multiple_checksums( |
1270 |
pkg_path, hashes=self._pkgindex_hashes)) |
1271 |
|
1272 |
d["CPV"] = cpv |
1273 |
- st = os.stat(pkg_path) |
1274 |
- d["MTIME"] = _unicode(st[stat.ST_MTIME]) |
1275 |
+ st = os.lstat(pkg_path) |
1276 |
+ d["_mtime_"] = _unicode(st[stat.ST_MTIME]) |
1277 |
d["SIZE"] = _unicode(st.st_size) |
1278 |
|
1279 |
- rel_path = self._pkg_paths[cpv] |
1280 |
+ rel_path = pkg_path[len(self.pkgdir)+1:] |
1281 |
# record location if it's non-default |
1282 |
if rel_path != cpv + ".tbz2": |
1283 |
d["PATH"] = rel_path |
1284 |
|
1285 |
- self._eval_use_flags(cpv, d) |
1286 |
return d |
1287 |
|
1288 |
def _new_pkgindex(self): |
1289 |
@@ -1311,15 +1436,17 @@ class binarytree(object): |
1290 |
return False |
1291 |
|
1292 |
def _eval_use_flags(self, cpv, metadata): |
1293 |
- use = frozenset(metadata["USE"].split()) |
1294 |
+ use = frozenset(metadata.get("USE", "").split()) |
1295 |
for k in self._pkgindex_use_evaluated_keys: |
1296 |
if k.endswith('DEPEND'): |
1297 |
token_class = Atom |
1298 |
else: |
1299 |
token_class = None |
1300 |
|
1301 |
+ deps = metadata.get(k) |
1302 |
+ if deps is None: |
1303 |
+ continue |
1304 |
try: |
1305 |
- deps = metadata[k] |
1306 |
deps = use_reduce(deps, uselist=use, token_class=token_class) |
1307 |
deps = paren_enclose(deps) |
1308 |
except portage.exception.InvalidDependString as e: |
1309 |
@@ -1349,46 +1476,129 @@ class binarytree(object): |
1310 |
return "" |
1311 |
return mymatch |
1312 |
|
1313 |
- def getname(self, pkgname): |
1314 |
- """Returns a file location for this package. The default location is |
1315 |
- ${PKGDIR}/All/${PF}.tbz2, but will be ${PKGDIR}/${CATEGORY}/${PF}.tbz2 |
1316 |
- in the rare event of a collision. The prevent_collision() method can |
1317 |
- be called to ensure that ${PKGDIR}/All/${PF}.tbz2 is available for a |
1318 |
- specific cpv.""" |
1319 |
+ def getname(self, cpv, allocate_new=None): |
1320 |
+ """Returns a file location for this package. |
1321 |
+ If cpv has both build_time and build_id attributes, then the |
1322 |
+ path to the specific corresponding instance is returned. |
1323 |
+ Otherwise, allocate a new path and return that. When allocating |
1324 |
+ a new path, behavior depends on the binpkg-multi-instance |
1325 |
+ FEATURES setting. |
1326 |
+ """ |
1327 |
if not self.populated: |
1328 |
self.populate() |
1329 |
- mycpv = pkgname |
1330 |
- mypath = self._pkg_paths.get(mycpv, None) |
1331 |
- if mypath: |
1332 |
- return os.path.join(self.pkgdir, mypath) |
1333 |
- mycat, mypkg = catsplit(mycpv) |
1334 |
- if self._all_directory: |
1335 |
- mypath = os.path.join("All", mypkg + ".tbz2") |
1336 |
- if mypath in self._pkg_paths.values(): |
1337 |
- mypath = os.path.join(mycat, mypkg + ".tbz2") |
1338 |
+ |
1339 |
+ try: |
1340 |
+ cpv.cp |
1341 |
+ except AttributeError: |
1342 |
+ cpv = _pkg_str(cpv) |
1343 |
+ |
1344 |
+ filename = None |
1345 |
+ if allocate_new: |
1346 |
+ filename = self._allocate_filename(cpv) |
1347 |
+ elif self._is_specific_instance(cpv): |
1348 |
+ instance_key = self.dbapi._instance_key(cpv) |
1349 |
+ path = self._pkg_paths.get(instance_key) |
1350 |
+ if path is not None: |
1351 |
+ filename = os.path.join(self.pkgdir, path) |
1352 |
+ |
1353 |
+ if filename is None and not allocate_new: |
1354 |
+ try: |
1355 |
+ instance_key = self.dbapi._instance_key(cpv, |
1356 |
+ support_string=True) |
1357 |
+ except KeyError: |
1358 |
+ pass |
1359 |
+ else: |
1360 |
+ filename = self._pkg_paths.get(instance_key) |
1361 |
+ if filename is not None: |
1362 |
+ filename = os.path.join(self.pkgdir, filename) |
1363 |
+ |
1364 |
+ if filename is None: |
1365 |
+ if self._multi_instance: |
1366 |
+ pf = catsplit(cpv)[1] |
1367 |
+ filename = "%s-%s.xpak" % ( |
1368 |
+ os.path.join(self.pkgdir, cpv.cp, pf), "1") |
1369 |
+ else: |
1370 |
+ filename = os.path.join(self.pkgdir, cpv + ".tbz2") |
1371 |
+ |
1372 |
+ return filename |
1373 |
+ |
1374 |
+ def _is_specific_instance(self, cpv): |
1375 |
+ specific = True |
1376 |
+ try: |
1377 |
+ build_time = cpv.build_time |
1378 |
+ build_id = cpv.build_id |
1379 |
+ except AttributeError: |
1380 |
+ specific = False |
1381 |
else: |
1382 |
- mypath = os.path.join(mycat, mypkg + ".tbz2") |
1383 |
- self._pkg_paths[mycpv] = mypath # cache for future lookups |
1384 |
- return os.path.join(self.pkgdir, mypath) |
1385 |
+ if build_time is None or build_id is None: |
1386 |
+ specific = False |
1387 |
+ return specific |
1388 |
+ |
1389 |
+ def _max_build_id(self, cpv): |
1390 |
+ max_build_id = 0 |
1391 |
+ for x in self.dbapi.cp_list(cpv.cp): |
1392 |
+ if (x == cpv and x.build_id is not None and |
1393 |
+ x.build_id > max_build_id): |
1394 |
+ max_build_id = x.build_id |
1395 |
+ return max_build_id |
1396 |
+ |
1397 |
+ def _allocate_filename(self, cpv): |
1398 |
+ return os.path.join(self.pkgdir, cpv + ".tbz2") |
1399 |
+ |
1400 |
+ def _allocate_filename_multi(self, cpv): |
1401 |
+ |
1402 |
+ # First, get the max build_id found when _populate was |
1403 |
+ # called. |
1404 |
+ max_build_id = self._max_build_id(cpv) |
1405 |
+ |
1406 |
+ # A new package may have been added concurrently since the |
1407 |
+ # last _populate call, so use increment build_id until |
1408 |
+ # we locate an unused id. |
1409 |
+ pf = catsplit(cpv)[1] |
1410 |
+ build_id = max_build_id + 1 |
1411 |
+ |
1412 |
+ while True: |
1413 |
+ filename = "%s-%s.xpak" % ( |
1414 |
+ os.path.join(self.pkgdir, cpv.cp, pf), build_id) |
1415 |
+ if os.path.exists(filename): |
1416 |
+ build_id += 1 |
1417 |
+ else: |
1418 |
+ return filename |
1419 |
+ |
1420 |
+ @staticmethod |
1421 |
+ def _parse_build_id(filename): |
1422 |
+ build_id = -1 |
1423 |
+ hyphen = filename.rfind("-", 0, -6) |
1424 |
+ if hyphen != -1: |
1425 |
+ build_id = filename[hyphen+1:-5] |
1426 |
+ try: |
1427 |
+ build_id = long(build_id) |
1428 |
+ except ValueError: |
1429 |
+ pass |
1430 |
+ return build_id |
1431 |
|
1432 |
def isremote(self, pkgname): |
1433 |
"""Returns true if the package is kept remotely and it has not been |
1434 |
downloaded (or it is only partially downloaded).""" |
1435 |
- if self._remotepkgs is None or pkgname not in self._remotepkgs: |
1436 |
+ if (self._remotepkgs is None or |
1437 |
+ self.dbapi._instance_key(pkgname) not in self._remotepkgs): |
1438 |
return False |
1439 |
# Presence in self._remotepkgs implies that it's remote. When a |
1440 |
# package is downloaded, state is updated by self.inject(). |
1441 |
return True |
1442 |
|
1443 |
- def get_pkgindex_uri(self, pkgname): |
1444 |
+ def get_pkgindex_uri(self, cpv): |
1445 |
"""Returns the URI to the Packages file for a given package.""" |
1446 |
- return self._pkgindex_uri.get(pkgname) |
1447 |
- |
1448 |
- |
1449 |
+ uri = None |
1450 |
+ metadata = self._remotepkgs.get(self.dbapi._instance_key(cpv)) |
1451 |
+ if metadata is not None: |
1452 |
+ uri = metadata["PKGINDEX_URI"] |
1453 |
+ return uri |
1454 |
|
1455 |
def gettbz2(self, pkgname): |
1456 |
"""Fetches the package from a remote site, if necessary. Attempts to |
1457 |
resume if the file appears to be partially downloaded.""" |
1458 |
+ instance_key = self.dbapi._instance_key(pkgname) |
1459 |
tbz2_path = self.getname(pkgname) |
1460 |
tbz2name = os.path.basename(tbz2_path) |
1461 |
resume = False |
1462 |
@@ -1404,10 +1614,10 @@ class binarytree(object): |
1463 |
self._ensure_dir(mydest) |
1464 |
# urljoin doesn't work correctly with unrecognized protocols like sftp |
1465 |
if self._remote_has_index: |
1466 |
- rel_url = self._remotepkgs[pkgname].get("PATH") |
1467 |
+ rel_url = self._remotepkgs[instance_key].get("PATH") |
1468 |
if not rel_url: |
1469 |
rel_url = pkgname+".tbz2" |
1470 |
- remote_base_uri = self._remotepkgs[pkgname]["BASE_URI"] |
1471 |
+ remote_base_uri = self._remotepkgs[instance_key]["BASE_URI"] |
1472 |
url = remote_base_uri.rstrip("/") + "/" + rel_url.lstrip("/") |
1473 |
else: |
1474 |
url = self.settings["PORTAGE_BINHOST"].rstrip("/") + "/" + tbz2name |
1475 |
@@ -1450,15 +1660,19 @@ class binarytree(object): |
1476 |
except AttributeError: |
1477 |
cpv = pkg |
1478 |
|
1479 |
+ _instance_key = self.dbapi._instance_key |
1480 |
+ instance_key = _instance_key(cpv) |
1481 |
digests = {} |
1482 |
- metadata = None |
1483 |
- if self._remotepkgs is None or cpv not in self._remotepkgs: |
1484 |
+ metadata = (None if self._remotepkgs is None else |
1485 |
+ self._remotepkgs.get(instance_key)) |
1486 |
+ if metadata is None: |
1487 |
for d in self._load_pkgindex().packages: |
1488 |
- if d["CPV"] == cpv: |
1489 |
+ if (d["CPV"] == cpv and |
1490 |
+ instance_key == _instance_key(_pkg_str(d["CPV"], |
1491 |
+ metadata=d, settings=self.settings))): |
1492 |
metadata = d |
1493 |
break |
1494 |
- else: |
1495 |
- metadata = self._remotepkgs[cpv] |
1496 |
+ |
1497 |
if metadata is None: |
1498 |
return digests |
1499 |
|
1500 |
diff --git a/pym/portage/emaint/modules/binhost/binhost.py b/pym/portage/emaint/modules/binhost/binhost.py |
1501 |
index 1138a8c..cf1213e 100644 |
1502 |
--- a/pym/portage/emaint/modules/binhost/binhost.py |
1503 |
+++ b/pym/portage/emaint/modules/binhost/binhost.py |
1504 |
@@ -7,6 +7,7 @@ import stat |
1505 |
import portage |
1506 |
from portage import os |
1507 |
from portage.util import writemsg |
1508 |
+from portage.versions import _pkg_str |
1509 |
|
1510 |
import sys |
1511 |
|
1512 |
@@ -38,7 +39,7 @@ class BinhostHandler(object): |
1513 |
if size is None: |
1514 |
return True |
1515 |
|
1516 |
- mtime = data.get("MTIME") |
1517 |
+ mtime = data.get("_mtime_") |
1518 |
if mtime is None: |
1519 |
return True |
1520 |
|
1521 |
@@ -90,6 +91,7 @@ class BinhostHandler(object): |
1522 |
def fix(self, **kwargs): |
1523 |
onProgress = kwargs.get('onProgress', None) |
1524 |
bintree = self._bintree |
1525 |
+ _instance_key = bintree.dbapi._instance_key |
1526 |
cpv_all = self._bintree.dbapi.cpv_all() |
1527 |
cpv_all.sort() |
1528 |
missing = [] |
1529 |
@@ -98,16 +100,21 @@ class BinhostHandler(object): |
1530 |
onProgress(maxval, 0) |
1531 |
pkgindex = self._pkgindex |
1532 |
missing = [] |
1533 |
+ stale = [] |
1534 |
metadata = {} |
1535 |
for d in pkgindex.packages: |
1536 |
- metadata[d["CPV"]] = d |
1537 |
- |
1538 |
- for i, cpv in enumerate(cpv_all): |
1539 |
- d = metadata.get(cpv) |
1540 |
+ cpv = _pkg_str(d["CPV"], metadata=d, |
1541 |
+ settings=bintree.settings) |
1542 |
+ d["CPV"] = cpv |
1543 |
+ metadata[_instance_key(cpv)] = d |
1544 |
+ if not bintree.dbapi.cpv_exists(cpv): |
1545 |
+ stale.append(cpv) |
1546 |
+ |
1547 |
+ for cpv in cpv_all: |
1548 |
+ d = metadata.get(_instance_key(cpv)) |
1549 |
if not d or self._need_update(cpv, d): |
1550 |
missing.append(cpv) |
1551 |
|
1552 |
- stale = set(metadata).difference(cpv_all) |
1553 |
if missing or stale: |
1554 |
from portage import locks |
1555 |
pkgindex_lock = locks.lockfile( |
1556 |
@@ -121,31 +128,39 @@ class BinhostHandler(object): |
1557 |
pkgindex = bintree._load_pkgindex() |
1558 |
self._pkgindex = pkgindex |
1559 |
|
1560 |
+ # Recount stale/missing packages, with lock held. |
1561 |
+ missing = [] |
1562 |
+ stale = [] |
1563 |
metadata = {} |
1564 |
for d in pkgindex.packages: |
1565 |
- metadata[d["CPV"]] = d |
1566 |
- |
1567 |
- # Recount missing packages, with lock held. |
1568 |
- del missing[:] |
1569 |
- for i, cpv in enumerate(cpv_all): |
1570 |
- d = metadata.get(cpv) |
1571 |
+ cpv = _pkg_str(d["CPV"], metadata=d, |
1572 |
+ settings=bintree.settings) |
1573 |
+ d["CPV"] = cpv |
1574 |
+ metadata[_instance_key(cpv)] = d |
1575 |
+ if not bintree.dbapi.cpv_exists(cpv): |
1576 |
+ stale.append(cpv) |
1577 |
+ |
1578 |
+ for cpv in cpv_all: |
1579 |
+ d = metadata.get(_instance_key(cpv)) |
1580 |
if not d or self._need_update(cpv, d): |
1581 |
missing.append(cpv) |
1582 |
|
1583 |
maxval = len(missing) |
1584 |
for i, cpv in enumerate(missing): |
1585 |
+ d = bintree._pkgindex_entry(cpv) |
1586 |
try: |
1587 |
- metadata[cpv] = bintree._pkgindex_entry(cpv) |
1588 |
+ bintree._eval_use_flags(cpv, d) |
1589 |
except portage.exception.InvalidDependString: |
1590 |
writemsg("!!! Invalid binary package: '%s'\n" % \ |
1591 |
bintree.getname(cpv), noiselevel=-1) |
1592 |
+ else: |
1593 |
+ metadata[_instance_key(cpv)] = d |
1594 |
|
1595 |
if onProgress: |
1596 |
onProgress(maxval, i+1) |
1597 |
|
1598 |
- for cpv in set(metadata).difference( |
1599 |
- self._bintree.dbapi.cpv_all()): |
1600 |
- del metadata[cpv] |
1601 |
+ for cpv in stale: |
1602 |
+ del metadata[_instance_key(cpv)] |
1603 |
|
1604 |
# We've updated the pkgindex, so set it to |
1605 |
# repopulate when necessary. |
1606 |
-- |
1607 |
2.0.5 |