[gentoo-portage-dev] [PATCH v2] flat_hash: enable md5 validation for /var/cache/edb/dep (bug 568934)

[Zac Medico]

Since operations like `git reset --hard` (useful to implement shallow
pull) will reset timestamps of all files in the tree, the status quo
of using timestamps for validation of cache in /var/cache/edb/dep
is sub-optimal.

For forward-compatibility, add a flat_hash.mtime_md5_database cache
module which is capable of validating cache entries containing either
mtimes or md5 digests. Update the config class to use this cache
module by default for /var/cache/edb/dep.

X-Gentoo-Bug: 568934
X-Gentoo-Bug-url: https://bugs.gentoo.org/show_bug.cgi?id=568934
---
[PATCH v2] only updates the commit message to mention the motivation for using
md5 instead of mtime.

 pym/portage/cache/flat_hash.py       |  5 +++++
 pym/portage/cache/template.py        | 35 ++++++++++++++++++++++++++++++-----
 pym/portage/package/ebuild/config.py |  6 +++---
 3 files changed, 38 insertions(+), 8 deletions(-)

diff --git a/pym/portage/cache/flat_hash.py b/pym/portage/cache/flat_hash.py
index 5304296..cca0f10 100644
--- a/pym/portage/cache/flat_hash.py
+++ b/pym/portage/cache/flat_hash.py
@@ -160,3 +160,8 @@ class md5_database(database):
 
 	validation_chf = 'md5'
 	store_eclass_paths = False
+
+
+class mtime_md5_database(database):
+	validation_chf = 'mtime'
+	chf_types = ('mtime', 'md5')
diff --git a/pym/portage/cache/template.py b/pym/portage/cache/template.py
index bc81b86..a942b36 100644
--- a/pym/portage/cache/template.py
+++ b/pym/portage/cache/template.py
@@ -47,8 +47,21 @@ class database(object):
 			self.updates = 0
 		d=self._getitem(cpv)
 		if self.serialize_eclasses and "_eclasses_" in d:
-			d["_eclasses_"] = reconstruct_eclasses(cpv, d["_eclasses_"],
-				self.validation_chf, paths=self.store_eclass_paths)
+			try:
+				chf_types = self.chf_types
+			except AttributeError:
+				chf_types = (self.validation_chf,)
+
+			for chf_type in chf_types:
+				try:
+					d["_eclasses_"] = reconstruct_eclasses(cpv, d["_eclasses_"],
+						chf_type, paths=self.store_eclass_paths)
+				except cache_errors.CacheCorruption:
+					if chf_type is chf_types[-1]:
+						raise
+				else:
+					break
+
 		elif "_eclasses_" not in d:
 			d["_eclasses_"] = {}
 		# Never return INHERITED, since portdbapi.aux_get() will
@@ -204,15 +217,27 @@ class database(object):
 			return x
 
 	def validate_entry(self, entry, ebuild_hash, eclass_db):
-		hash_key = '_%s_' % self.validation_chf
+		try:
+			chf_types = self.chf_types
+		except AttributeError:
+			chf_types = (self.validation_chf,)
+
+		for chf_type in chf_types:
+			if self._validate_entry(chf_type, entry, ebuild_hash, eclass_db):
+				return True
+
+		return False
+
+	def _validate_entry(self, chf_type, entry, ebuild_hash, eclass_db):
+		hash_key = '_%s_' % chf_type
 		try:
 			entry_hash = entry[hash_key]
 		except KeyError:
 			return False
 		else:
-			if entry_hash != getattr(ebuild_hash, self.validation_chf):
+			if entry_hash != getattr(ebuild_hash, chf_type):
 				return False
-		update = eclass_db.validate_and_rewrite_cache(entry['_eclasses_'], self.validation_chf,
+		update = eclass_db.validate_and_rewrite_cache(entry['_eclasses_'], chf_type,
 			self.store_eclass_paths)
 		if update is None:
 			return False
diff --git a/pym/portage/package/ebuild/config.py b/pym/portage/package/ebuild/config.py
index d45c2a0..0bae55b 100644
--- a/pym/portage/package/ebuild/config.py
+++ b/pym/portage/package/ebuild/config.py
@@ -160,8 +160,8 @@ class config(object):
 		'repository', 'RESTRICT', 'LICENSE',)
 
 	_module_aliases = {
-		"cache.metadata_overlay.database" : "portage.cache.flat_hash.database",
-		"portage.cache.metadata_overlay.database" : "portage.cache.flat_hash.database",
+		"cache.metadata_overlay.database" : "portage.cache.flat_hash.mtime_md5_database",
+		"portage.cache.metadata_overlay.database" : "portage.cache.flat_hash.mtime_md5_database",
 	}
 
 	_case_insensitive_vars = special_env_vars.case_insensitive_vars
@@ -444,7 +444,7 @@ class config(object):
 				(user_auxdbmodule, modules_file))
 
 			self.modules["default"] = {
-				"portdbapi.auxdbmodule":  "portage.cache.flat_hash.database",
+				"portdbapi.auxdbmodule":  "portage.cache.flat_hash.mtime_md5_database",
 			}
 
 			self.configlist=[]
-- 
2.4.10