1 |
Author: zmedico |
2 |
Date: 2008-08-01 04:13:23 +0000 (Fri, 01 Aug 2008) |
3 |
New Revision: 11307 |
4 |
|
5 |
Modified: |
6 |
main/trunk/bin/emaint |
7 |
main/trunk/pym/portage/cache/mappings.py |
8 |
main/trunk/pym/portage/dbapi/bintree.py |
9 |
Log: |
10 |
Optimize `emaint` --fix binhost so that it the Packages file isn't re-read |
11 |
and re-written for each package that's updated. Instead, hold a lock for |
12 |
the whole time the command is running and just update it once. |
13 |
|
14 |
|
15 |
Modified: main/trunk/bin/emaint |
16 |
=================================================================== |
17 |
--- main/trunk/bin/emaint 2008-08-01 02:22:06 UTC (rev 11306) |
18 |
+++ main/trunk/bin/emaint 2008-08-01 04:13:23 UTC (rev 11307) |
19 |
@@ -112,8 +112,7 @@ |
20 |
self._bintree = portage.db[myroot]["bintree"] |
21 |
self._bintree.populate() |
22 |
self._pkgindex_file = os.path.join(self._bintree.pkgdir, "Packages") |
23 |
- from portage import getbinpkg |
24 |
- self._pkgindex = getbinpkg.PackageIndex() |
25 |
+ self._pkgindex = self._bintree._new_pkgindex() |
26 |
f = open(self._pkgindex_file, 'r') |
27 |
try: |
28 |
self._pkgindex.read(f) |
29 |
@@ -149,7 +148,7 @@ |
30 |
cpv_all = self._bintree.dbapi.cpv_all() |
31 |
cpv_all.sort() |
32 |
missing = [] |
33 |
- maxval = len(cpv_all) |
34 |
+ maxval = 0 |
35 |
if onProgress: |
36 |
onProgress(maxval, 0) |
37 |
pkgindex = self._pkgindex |
38 |
@@ -157,35 +156,61 @@ |
39 |
metadata = {} |
40 |
for d in pkgindex.packages: |
41 |
metadata[d["CPV"]] = d |
42 |
+ |
43 |
for i, cpv in enumerate(cpv_all): |
44 |
d = metadata.get(cpv) |
45 |
if not d or "MD5" not in d: |
46 |
- bintree.inject(cpv) |
47 |
- if onProgress: |
48 |
- onProgress(maxval, i+1) |
49 |
+ missing.append(cpv) |
50 |
+ |
51 |
stale = set(metadata).difference(cpv_all) |
52 |
- if stale: |
53 |
+ if missing or stale: |
54 |
from portage import locks |
55 |
pkgindex_lock = locks.lockfile( |
56 |
self._pkgindex_file, wantnewlockfile=1) |
57 |
try: |
58 |
- from portage import getbinpkg |
59 |
- pkgindex = getbinpkg.PackageIndex() |
60 |
+ # Repopulate with lock held. |
61 |
+ bintree._populate() |
62 |
+ cpv_all = self._bintree.dbapi.cpv_all() |
63 |
+ cpv_all.sort() |
64 |
+ |
65 |
+ pkgindex = bintree._new_pkgindex() |
66 |
self._pkgindex = pkgindex |
67 |
f = open(self._pkgindex_file, 'r') |
68 |
try: |
69 |
self._pkgindex.read(f) |
70 |
finally: |
71 |
f.close() |
72 |
- from portage.dbapi.bintree import binarytree |
73 |
- self._bintree = binarytree(bintree.root, bintree.pkgdir, |
74 |
- settings=bintree.settings) |
75 |
- del bintree |
76 |
- portage.db[self._bintree.root]["bintree"] = self._bintree |
77 |
- self._bintree._populate() |
78 |
+ |
79 |
+ metadata = {} |
80 |
+ for d in pkgindex.packages: |
81 |
+ metadata[d["CPV"]] = d |
82 |
+ |
83 |
+ # Recount missing packages, with lock held. |
84 |
+ del missing[:] |
85 |
+ for i, cpv in enumerate(cpv_all): |
86 |
+ d = metadata.get(cpv) |
87 |
+ if not d or "MD5" not in d: |
88 |
+ missing.append(cpv) |
89 |
+ |
90 |
+ maxval = len(missing) |
91 |
+ for i, cpv in enumerate(missing): |
92 |
+ try: |
93 |
+ metadata[cpv] = bintree._pkgindex_entry(cpv) |
94 |
+ except portage.exception.InvalidDependString: |
95 |
+ writemsg("!!! Invalid binary package: '%s'\n" % \ |
96 |
+ bintree.getname(cpv), noiselevel=-1) |
97 |
+ |
98 |
+ if onProgress: |
99 |
+ onProgress(maxval, i+1) |
100 |
+ |
101 |
for cpv in set(metadata).difference( |
102 |
self._bintree.dbapi.cpv_all()): |
103 |
del metadata[cpv] |
104 |
+ |
105 |
+ # We've updated the pkgindex, so set it to |
106 |
+ # repopulate when necessary. |
107 |
+ bintree.populated = False |
108 |
+ |
109 |
del pkgindex.packages[:] |
110 |
pkgindex.packages.extend(metadata.itervalues()) |
111 |
from portage.util import atomic_ofstream |
112 |
@@ -196,6 +221,11 @@ |
113 |
f.close() |
114 |
finally: |
115 |
locks.unlockfile(pkgindex_lock) |
116 |
+ |
117 |
+ if onProgress: |
118 |
+ if maxval == 0: |
119 |
+ maxval = 1 |
120 |
+ onProgress(maxval, maxval) |
121 |
return None |
122 |
|
123 |
class MoveHandler(object): |
124 |
|
125 |
Modified: main/trunk/pym/portage/cache/mappings.py |
126 |
=================================================================== |
127 |
--- main/trunk/pym/portage/cache/mappings.py 2008-08-01 02:22:06 UTC (rev 11306) |
128 |
+++ main/trunk/pym/portage/cache/mappings.py 2008-08-01 04:13:23 UTC (rev 11307) |
129 |
@@ -262,6 +262,9 @@ |
130 |
except AttributeError: |
131 |
pass |
132 |
|
133 |
+ def __str__(self): |
134 |
+ return str(dict(self.iteritems())) |
135 |
+ |
136 |
v = SlotDict |
137 |
_slot_dict_classes[v.allowed_keys] = v |
138 |
return v |
139 |
|
140 |
Modified: main/trunk/pym/portage/dbapi/bintree.py |
141 |
=================================================================== |
142 |
--- main/trunk/pym/portage/dbapi/bintree.py 2008-08-01 02:22:06 UTC (rev 11306) |
143 |
+++ main/trunk/pym/portage/dbapi/bintree.py 2008-08-01 04:13:23 UTC (rev 11307) |
144 |
@@ -764,9 +764,6 @@ |
145 |
noiselevel=-1) |
146 |
return |
147 |
slot = slot.strip() |
148 |
- from portage.checksum import perform_multiple_checksums |
149 |
- digests = perform_multiple_checksums( |
150 |
- full_path, hashes=self._pkgindex_hashes) |
151 |
self.dbapi.cpv_inject(cpv) |
152 |
self.dbapi._aux_cache.pop(cpv, None) |
153 |
|
154 |
@@ -797,26 +794,16 @@ |
155 |
del f |
156 |
if not self._pkgindex_version_supported(pkgindex): |
157 |
pkgindex = self._new_pkgindex() |
158 |
- d = digests |
159 |
- d["CPV"] = cpv |
160 |
- d["SLOT"] = slot |
161 |
- d["MTIME"] = str(long(s.st_mtime)) |
162 |
- d["SIZE"] = str(s.st_size) |
163 |
- rel_path = self._pkg_paths[cpv] |
164 |
- # record location if it's non-default |
165 |
- if rel_path != cpv + ".tbz2": |
166 |
- d["PATH"] = rel_path |
167 |
- from itertools import izip |
168 |
- d.update(izip(self._pkgindex_aux_keys, |
169 |
- self.dbapi.aux_get(cpv, self._pkgindex_aux_keys))) |
170 |
+ |
171 |
try: |
172 |
- self._eval_use_flags(cpv, d) |
173 |
+ d = self._pkgindex_entry(cpv) |
174 |
except portage.exception.InvalidDependString: |
175 |
writemsg("!!! Invalid binary package: '%s'\n" % \ |
176 |
self.getname(cpv), noiselevel=-1) |
177 |
self.dbapi.cpv_remove(cpv) |
178 |
del self._pkg_paths[cpv] |
179 |
return |
180 |
+ |
181 |
# If found, remove package(s) with duplicate path. |
182 |
for i in xrange(len(pkgindex.packages) - 1, -1, -1): |
183 |
d2 = pkgindex.packages[i] |
184 |
@@ -825,6 +812,7 @@ |
185 |
if d2.get("PATH", "") == d.get("PATH", ""): |
186 |
del pkgindex.packages[i] |
187 |
pkgindex.packages.append(d) |
188 |
+ |
189 |
self._update_pkgindex_header(pkgindex.header) |
190 |
from portage.util import atomic_ofstream |
191 |
f = atomic_ofstream(os.path.join(self.pkgdir, "Packages")) |
192 |
@@ -836,6 +824,36 @@ |
193 |
if pkgindex_lock: |
194 |
unlockfile(pkgindex_lock) |
195 |
|
196 |
+ def _pkgindex_entry(self, cpv): |
197 |
+ """ |
198 |
+ Performs checksums and evaluates USE flag conditionals. |
199 |
+ Raises InvalidDependString if necessary. |
200 |
+ @rtype: dict |
201 |
+ @returns: a dict containing entry for the give cpv. |
202 |
+ """ |
203 |
+ |
204 |
+ pkg_path = self.getname(cpv) |
205 |
+ from portage.checksum import perform_multiple_checksums |
206 |
+ |
207 |
+ d = dict(izip(self._pkgindex_aux_keys, |
208 |
+ self.dbapi.aux_get(cpv, self._pkgindex_aux_keys))) |
209 |
+ |
210 |
+ d.update(perform_multiple_checksums( |
211 |
+ pkg_path, hashes=self._pkgindex_hashes)) |
212 |
+ |
213 |
+ d["CPV"] = cpv |
214 |
+ st = os.stat(pkg_path) |
215 |
+ d["MTIME"] = str(long(st.st_mtime)) |
216 |
+ d["SIZE"] = str(st.st_size) |
217 |
+ |
218 |
+ rel_path = self._pkg_paths[cpv] |
219 |
+ # record location if it's non-default |
220 |
+ if rel_path != cpv + ".tbz2": |
221 |
+ d["PATH"] = rel_path |
222 |
+ |
223 |
+ self._eval_use_flags(cpv, d) |
224 |
+ return d |
225 |
+ |
226 |
def _new_pkgindex(self): |
227 |
return portage.getbinpkg.PackageIndex( |
228 |
allowed_pkg_keys=self._pkgindex_allowed_pkg_keys, |