| 1 |
commit: 0e120da008c9d0d41c9372c81145c6e153028a6d |
| 2 |
Author: Zac Medico <zmedico <AT> gentoo <DOT> org> |
| 3 |
AuthorDate: Sat Oct 29 20:36:23 2011 +0000 |
| 4 |
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org> |
| 5 |
CommitDate: Sat Oct 29 20:36:23 2011 +0000 |
| 6 |
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=0e120da0 |
| 7 |
|
| 8 |
egencache: avoid redundant md5-dict writes |
| 9 |
|
| 10 |
The pms cache already does this automatically, since __setitem__ calls |
| 11 |
are used to detect stat collisions in order to solve bug #139134. |
| 12 |
|
| 13 |
--- |
| 14 |
bin/egencache | 58 ++++++++++++++++++++++++++++++++++++++++++++------------ |
| 15 |
1 files changed, 45 insertions(+), 13 deletions(-) |
| 16 |
|
| 17 |
diff --git a/bin/egencache b/bin/egencache |
| 18 |
index 22ce8ec..33839aa 100755 |
| 19 |
--- a/bin/egencache |
| 20 |
+++ b/bin/egencache |
| 21 |
@@ -199,7 +199,12 @@ def parse_args(args): |
| 22 |
class GenCache(object): |
| 23 |
def __init__(self, portdb, cp_iter=None, max_jobs=None, max_load=None, |
| 24 |
rsync=False): |
| 25 |
+ # The caller must set portdb.porttrees in order to constrain |
| 26 |
+ # findname, cp_list, and cpv_list to the desired tree. |
| 27 |
+ tree = portdb.porttrees[0] |
| 28 |
self._portdb = portdb |
| 29 |
+ self._eclass_db = portdb._repo_info[tree].eclass_db |
| 30 |
+ self._auxdbkeys = portage.auxdbkeys |
| 31 |
# We can globally cleanse stale cache only if we |
| 32 |
# iterate over every single cp. |
| 33 |
self._global_cleanse = cp_iter is None |
| 34 |
@@ -214,22 +219,25 @@ class GenCache(object): |
| 35 |
consumer=self._metadata_callback, |
| 36 |
max_jobs=max_jobs, max_load=max_load) |
| 37 |
self.returncode = os.EX_OK |
| 38 |
- conf = portdb.repositories.get_repo_for_location(portdb.porttrees[0]) |
| 39 |
+ conf = portdb.repositories.get_repo_for_location(tree) |
| 40 |
self._trg_caches = tuple(conf.iter_pregenerated_caches( |
| 41 |
- portage.auxdbkeys[:], force=True, readonly=False)) |
| 42 |
+ self._auxdbkeys, force=True, readonly=False)) |
| 43 |
if not self._trg_caches: |
| 44 |
raise Exception("cache formats '%s' aren't supported" % |
| 45 |
(" ".join(conf.cache_formats),)) |
| 46 |
- if rsync: |
| 47 |
- from portage.cache.metadata import database as pms_database |
| 48 |
- for trg_cache in self._trg_caches: |
| 49 |
- if isinstance(trg_cache, pms_database): |
| 50 |
- trg_cache.raise_stat_collision = True |
| 51 |
- # Make _metadata_callback write this cache first, in case |
| 52 |
- # it raises a StatCollision and triggers mtime |
| 53 |
- # modification. |
| 54 |
- self._trg_caches = tuple([trg_cache] + |
| 55 |
- [x for x in self._trg_caches if x is not trg_cache]) |
| 56 |
+ |
| 57 |
+ self._avoid_redundant_write = set() |
| 58 |
+ from portage.cache.metadata import database as pms_database |
| 59 |
+ for trg_cache in self._trg_caches: |
| 60 |
+ if not isinstance(trg_cache, pms_database): |
| 61 |
+ self._avoid_redundant_write.add(id(trg_cache)) |
| 62 |
+ elif rsync: |
| 63 |
+ trg_cache.raise_stat_collision = True |
| 64 |
+ # Make _metadata_callback write this cache first, in case |
| 65 |
+ # it raises a StatCollision and triggers mtime |
| 66 |
+ # modification. |
| 67 |
+ self._trg_caches = tuple([trg_cache] + |
| 68 |
+ [x for x in self._trg_caches if x is not trg_cache]) |
| 69 |
|
| 70 |
self._existing_nodes = set() |
| 71 |
|
| 72 |
@@ -244,6 +252,27 @@ class GenCache(object): |
| 73 |
cpv, repo_path, metadata, ebuild_hash) |
| 74 |
|
| 75 |
def _write_cache(self, trg_cache, cpv, repo_path, metadata, ebuild_hash): |
| 76 |
+ |
| 77 |
+ if id(trg_cache) in self._avoid_redundant_write: |
| 78 |
+ # This cache does not avoid redundant writes automatically, |
| 79 |
+ # so check for an identical existing entry before writing. |
| 80 |
+ # This prevents unecessary disk writes and can also prevent |
| 81 |
+ # unecessary rsync transfers. |
| 82 |
+ try: |
| 83 |
+ dest = trg_cache[cpv] |
| 84 |
+ except (KeyError, CacheError): |
| 85 |
+ pass |
| 86 |
+ else: |
| 87 |
+ if trg_cache.validate_entry(dest, |
| 88 |
+ ebuild_hash, self._eclass_db): |
| 89 |
+ identical = True |
| 90 |
+ for k in self._auxdbkeys: |
| 91 |
+ if dest.get(k, '') != metadata.get(k, ''): |
| 92 |
+ identical = False |
| 93 |
+ break |
| 94 |
+ if identical: |
| 95 |
+ return |
| 96 |
+ |
| 97 |
try: |
| 98 |
chf = trg_cache.validation_chf |
| 99 |
metadata['_%s_' % chf] = getattr(ebuild_hash, chf) |
| 100 |
@@ -256,7 +285,10 @@ class GenCache(object): |
| 101 |
# exception from _setitem() if they detect this type of stat |
| 102 |
# collision. These exceptions are handled by bumping the |
| 103 |
# mtime on the ebuild (and the corresponding cache entry). |
| 104 |
- # See bug #139134. |
| 105 |
+ # This type of cache must not be included in the above |
| 106 |
+ # _avoid_redundant_write set, since __setitem__ must be |
| 107 |
+ # called in order to detect the StatCollision (redundant |
| 108 |
+ # writes will be avoided internally). See bug #139134. |
| 109 |
max_mtime = sc.mtime |
| 110 |
for ec, ec_hash in metadata['_eclasses_'].items(): |
| 111 |
if max_mtime < ec_hash.mtime: |
Archives