[gentoo-portage-dev] [PATCH v2] Make manifest-required-hashes configurable

[Michał Górny]

The set of required hashes specify which hashes must be present for
a distfile not to be refetched. It makes little sense to hardcode this
value, and it is mostly useful for transition periods, so make it
configurable via layout.conf and default to all hashes
in manifest-hashes.
---
 man/portage.5                           | 13 +++++++---
 pym/portage/_emirrordist/FetchTask.py   |  2 +-
 pym/portage/const.py                    |  2 +-
 pym/portage/manifest.py                 | 26 +++++++++++++-------
 pym/portage/package/ebuild/digestgen.py |  4 ++--
 pym/portage/repository/config.py        | 42 ++++++++++++++++++++++++++-------
 pym/portage/tests/ebuild/test_config.py |  1 +
 repoman/pym/repoman/repos.py            | 14 +++++++----
 8 files changed, 74 insertions(+), 30 deletions(-)

diff --git a/man/portage.5 b/man/portage.5
index 7605d7cfa..e724e1f08 100644
--- a/man/portage.5
+++ b/man/portage.5
@@ -1260,9 +1260,16 @@ for every file), "true" (if an entry exists for a file, enforce it), or "false"
 (don't check Manifest files at all).
 .TP
 .BR manifest\-hashes
-List of hashes to generate/check in Manifest files.  Valid hashes depend on the
-current version of portage; see the portage.checksum module for the current
-list.
+List of hashes to generate in new/updated entries Manifest files.  Valid hashes
+depend on the current version of portage; see the portage.checksum module for
+the current list.  Portage will not rewrite old entries if they satisfy
+manifest\-required\-hashes.
+.TP
+.BR manifest\-required\-hashes
+List of hashes that must be used in all Manifest entries.  If the hashes listed
+here are not present in the Manifest, Portage will refetch all distfiles
+and update the respective entries to include them.  Must be a subset
+of manifest\-hashes.  If not specified, defaults to all manifest\-hashes.
 .TP
 .BR update\-changelog " = [true|" false "]"
 The default setting for repoman's --echangelog option.
diff --git a/pym/portage/_emirrordist/FetchTask.py b/pym/portage/_emirrordist/FetchTask.py
index 203b8c213..47908cb6b 100644
--- a/pym/portage/_emirrordist/FetchTask.py
+++ b/pym/portage/_emirrordist/FetchTask.py
@@ -20,7 +20,7 @@ from portage.util._async.PipeLogger import PipeLogger
 from portage.util._async.PopenProcess import PopenProcess
 from _emerge.CompositeTask import CompositeTask
 
-default_hash_name = portage.const.MANIFEST2_REQUIRED_HASH
+default_hash_name = portage.const.MANIFEST2_HASH_DEFAULT
 
 # Use --no-check-certificate since Manifest digests should provide
 # enough security, and certificates can be self-signed or whatnot.
diff --git a/pym/portage/const.py b/pym/portage/const.py
index 0af57d0e2..ec877b841 100644
--- a/pym/portage/const.py
+++ b/pym/portage/const.py
@@ -207,7 +207,7 @@ EAPI                     = 6
 HASHING_BLOCKSIZE        = 32768
 
 MANIFEST2_HASH_DEFAULTS = frozenset(["SHA256", "SHA512", "WHIRLPOOL"])
-MANIFEST2_REQUIRED_HASH  = "SHA512"
+MANIFEST2_HASH_DEFAULT  = "SHA512"
 
 MANIFEST2_IDENTIFIERS    = ("AUX", "MISC", "DIST", "EBUILD")
 
diff --git a/pym/portage/manifest.py b/pym/portage/manifest.py
index 36c82690c..4ec20515e 100644
--- a/pym/portage/manifest.py
+++ b/pym/portage/manifest.py
@@ -26,8 +26,7 @@ from portage import _unicode_encode
 from portage.exception import DigestException, FileNotFound, \
 	InvalidDataType, MissingParameter, PermissionDenied, \
 	PortageException, PortagePackageException
-from portage.const import (MANIFEST2_HASH_DEFAULTS,
-	MANIFEST2_IDENTIFIERS, MANIFEST2_REQUIRED_HASH)
+from portage.const import (MANIFEST2_HASH_DEFAULTS, MANIFEST2_IDENTIFIERS)
 from portage.localization import _
 
 _manifest_re = re.compile(
@@ -128,7 +127,7 @@ class Manifest(object):
 	parsers = (parseManifest2,)
 	def __init__(self, pkgdir, distdir=None, fetchlist_dict=None,
 		manifest1_compat=DeprecationWarning, from_scratch=False, thin=False,
-		allow_missing=False, allow_create=True, hashes=None,
+		allow_missing=False, allow_create=True, hashes=None, required_hashes=None,
 		find_invalid_path_char=None, strict_misc_digests=True):
 		""" Create new Manifest instance for package in pkgdir.
 		    Do not parse Manifest file if from_scratch == True (only for internal use)
@@ -148,15 +147,21 @@ class Manifest(object):
 		self.pkgdir = _unicode_decode(pkgdir).rstrip(os.sep) + os.sep
 		self.fhashdict = {}
 		self.hashes = set()
+		self.required_hashes = set()
 
 		if hashes is None:
 			hashes = MANIFEST2_HASH_DEFAULTS
+		if required_hashes is None:
+			required_hashes = hashes
 
 		self.hashes.update(hashes)
 		self.hashes.difference_update(hashname for hashname in \
 			list(self.hashes) if hashname not in get_valid_checksum_keys())
 		self.hashes.add("size")
-		self.hashes.add(MANIFEST2_REQUIRED_HASH)
+
+		self.required_hashes.update(required_hashes)
+		self.required_hashes.intersection_update(self.hashes)
+
 		for t in MANIFEST2_IDENTIFIERS:
 			self.fhashdict[t] = {}
 		if not from_scratch:
@@ -269,9 +274,11 @@ class Manifest(object):
 	def checkIntegrity(self):
 		for t in self.fhashdict:
 			for f in self.fhashdict[t]:
-				if MANIFEST2_REQUIRED_HASH not in self.fhashdict[t][f]:
-					raise MissingParameter(_("Missing %s checksum: %s %s") %
-						(MANIFEST2_REQUIRED_HASH, t, f))
+				diff = self.required_hashes.difference(
+						set(self.fhashdict[t][f]))
+				if diff:
+					raise MissingParameter(_("Missing %s checksum(s): %s %s") %
+						(' '.join(diff), t, f))
 
 	def write(self, sign=False, force=False):
 		""" Write Manifest instance to disk, optionally signing it. Returns
@@ -422,7 +429,7 @@ class Manifest(object):
 		self.fhashdict[ftype][fname] = {}
 		if hashdict != None:
 			self.fhashdict[ftype][fname].update(hashdict)
-		if not MANIFEST2_REQUIRED_HASH in self.fhashdict[ftype][fname]:
+		if self.required_hashes.difference(set(self.fhashdict[ftype][fname])):
 			self.updateFileHashes(ftype, fname, checkExisting=False, ignoreMissing=ignoreMissing)
 	
 	def removeFile(self, ftype, fname):
@@ -462,6 +469,7 @@ class Manifest(object):
 			fetchlist_dict=self.fetchlist_dict, from_scratch=True,
 			thin=self.thin, allow_missing=self.allow_missing,
 			allow_create=self.allow_create, hashes=self.hashes,
+			required_hashes=self.required_hashes,
 			find_invalid_path_char=self._find_invalid_path_char,
 			strict_misc_digests=self.strict_misc_digests)
 		pn = os.path.basename(self.pkgdir.rstrip(os.path.sep))
@@ -487,7 +495,7 @@ class Manifest(object):
 			requiredDistfiles = distlist.copy()
 		required_hash_types = set()
 		required_hash_types.add("size")
-		required_hash_types.add(MANIFEST2_REQUIRED_HASH)
+		required_hash_types.update(self.required_hashes)
 		for f in distlist:
 			fname = os.path.join(self.distdir, f)
 			mystat = None
diff --git a/pym/portage/package/ebuild/digestgen.py b/pym/portage/package/ebuild/digestgen.py
index 95d02db9b..40c1b7288 100644
--- a/pym/portage/package/ebuild/digestgen.py
+++ b/pym/portage/package/ebuild/digestgen.py
@@ -11,7 +11,6 @@ portage.proxy.lazyimport.lazyimport(globals(),
 )
 
 from portage import os
-from portage.const import MANIFEST2_REQUIRED_HASH
 from portage.dbapi.porttree import FetchlistDict
 from portage.dep import use_reduce
 from portage.exception import InvalidDependString, FileNotFound, \
@@ -58,6 +57,7 @@ def digestgen(myarchives=None, mysettings=None, myportdb=None):
 			mytree = os.path.realpath(mytree)
 			mf = mysettings.repositories.get_repo_for_location(mytree)
 
+		repo_required_hashes = mf.manifest_required_hashes
 		mf = mf.load_manifest(mysettings["O"], mysettings["DISTDIR"],
 			fetchlist_dict=fetchlist_dict)
 
@@ -72,7 +72,7 @@ def digestgen(myarchives=None, mysettings=None, myportdb=None):
 		# exist before and after the transition.
 		required_hash_types = set()
 		required_hash_types.add("size")
-		required_hash_types.add(MANIFEST2_REQUIRED_HASH)
+		required_hash_types.update(repo_required_hashes)
 		dist_hashes = mf.fhashdict.get("DIST", {})
 
 		# To avoid accidental regeneration of digests with the incorrect
diff --git a/pym/portage/repository/config.py b/pym/portage/repository/config.py
index 3be0e8bda..be31ed3b1 100644
--- a/pym/portage/repository/config.py
+++ b/pym/portage/repository/config.py
@@ -12,8 +12,7 @@ import re
 import portage
 from portage import eclass_cache, os
 from portage.checksum import get_valid_checksum_keys
-from portage.const import (MANIFEST2_REQUIRED_HASH,
-	PORTAGE_BASE_PATH, REPO_NAME_LOC, USER_CONFIG_PATH)
+from portage.const import (PORTAGE_BASE_PATH, REPO_NAME_LOC, USER_CONFIG_PATH)
 from portage.eapi import eapi_allows_directories_on_profile_level_and_repository_level
 from portage.env.loaders import KeyValuePairFileLoader
 from portage.util import (normalize_path, read_corresponding_eapi_file, shlex_split,
@@ -86,7 +85,7 @@ class RepoConfig(object):
 		'sync_depth', 'sync_hooks_only_on_change',
 		'sync_type', 'sync_umask', 'sync_uri', 'sync_user', 'thin_manifest',
 		'update_changelog', '_eapis_banned', '_eapis_deprecated',
-		'_masters_orig', 'module_specific_options',
+		'_masters_orig', 'module_specific_options', 'manifest_required_hashes',
 		)
 
 	def __init__(self, name, repo_opts, local_config=True):
@@ -227,6 +226,7 @@ class RepoConfig(object):
 		self.create_manifest = True
 		self.disable_manifest = False
 		self.manifest_hashes = None
+		self.manifest_required_hashes = None
 		self.update_changelog = False
 		self.cache_formats = None
 		self.portage1_profiles = True
@@ -262,7 +262,7 @@ class RepoConfig(object):
 			for value in ('allow-missing-manifest',
 				'allow-provide-virtual', 'cache-formats',
 				'create-manifest', 'disable-manifest', 'manifest-hashes',
-				'profile-formats',
+				'manifest-required-hashes', 'profile-formats',
 				'sign-commit', 'sign-manifest', 'thin-manifest', 'update-changelog'):
 				setattr(self, value.lower().replace("-", "_"), layout_data[value])
 
@@ -337,6 +337,7 @@ class RepoConfig(object):
 		kwds['allow_missing'] = self.allow_missing_manifest
 		kwds['allow_create'] = self.create_manifest
 		kwds['hashes'] = self.manifest_hashes
+		kwds['required_hashes'] = self.manifest_required_hashes
 		kwds['strict_misc_digests'] = self.strict_misc_digests
 		if self.disable_manifest:
 			kwds['from_scratch'] = True
@@ -1046,20 +1047,41 @@ def parse_layout_conf(repo_location, repo_name=None):
 	data['cache-formats'] = tuple(cache_formats)
 
 	manifest_hashes = layout_data.get('manifest-hashes')
+	manifest_required_hashes = layout_data.get('manifest-required-hashes')
+
+	if manifest_required_hashes is not None and manifest_hashes is None:
+		repo_name = _get_repo_name(repo_location, cached=repo_name)
+		warnings.warn((_("Repository named '%(repo_name)s' specifies "
+			"'manifest-required-hashes' setting without corresponding "
+			"'manifest-hashes'. Portage will default it to match "
+			"the required set but please add the missing entry "
+			"to: %(layout_filename)s") %
+			{"repo_name": repo_name or 'unspecified',
+			"layout_filename":layout_filename}),
+			SyntaxWarning)
+		manifest_hashes = manifest_required_hashes
+
 	if manifest_hashes is not None:
+		# require all the hashes unless specified otherwise
+		if manifest_required_hashes is None:
+			manifest_required_hashes = manifest_hashes
+
+		manifest_required_hashes = frozenset(manifest_required_hashes.upper().split())
 		manifest_hashes = frozenset(manifest_hashes.upper().split())
-		if MANIFEST2_REQUIRED_HASH not in manifest_hashes:
+		missing_required_hashes = manifest_required_hashes.difference(
+			manifest_hashes)
+		if missing_required_hashes:
 			repo_name = _get_repo_name(repo_location, cached=repo_name)
 			warnings.warn((_("Repository named '%(repo_name)s' has a "
 				"'manifest-hashes' setting that does not contain "
-				"the '%(hash)s' hash which is required by this "
-				"portage version. You will have to upgrade portage "
+				"the '%(hash)s' hashes which are listed in "
+				"'manifest-required-hashes'. Please fix that file "
 				"if you want to generate valid manifests for this "
 				"repository: %(layout_filename)s") %
 				{"repo_name": repo_name or 'unspecified',
-				"hash":MANIFEST2_REQUIRED_HASH,
+				"hash": ' '.join(missing_required_hashes),
 				"layout_filename":layout_filename}),
-				DeprecationWarning)
+				SyntaxWarning)
 		unsupported_hashes = manifest_hashes.difference(
 			get_valid_checksum_keys())
 		if unsupported_hashes:
@@ -1074,7 +1096,9 @@ def parse_layout_conf(repo_location, repo_name=None):
 				"hashes":" ".join(sorted(unsupported_hashes)),
 				"layout_filename":layout_filename}),
 				DeprecationWarning)
+
 	data['manifest-hashes'] = manifest_hashes
+	data['manifest-required-hashes'] = manifest_required_hashes
 
 	data['update-changelog'] = layout_data.get('update-changelog', 'false').lower() \
 		== 'true'
diff --git a/pym/portage/tests/ebuild/test_config.py b/pym/portage/tests/ebuild/test_config.py
index 1dd828538..dcb5ffe0d 100644
--- a/pym/portage/tests/ebuild/test_config.py
+++ b/pym/portage/tests/ebuild/test_config.py
@@ -228,6 +228,7 @@ class ConfigTestCase(TestCase):
 						"profile-formats = pms",
 						"thin-manifests = true",
 						"manifest-hashes = SHA256 SHA512 WHIRLPOOL",
+						"manifest-required-hashes = SHA512",
 						"# use implicit masters"
 					),
 			}
diff --git a/repoman/pym/repoman/repos.py b/repoman/pym/repoman/repos.py
index 11a6231de..e942a599e 100644
--- a/repoman/pym/repoman/repos.py
+++ b/repoman/pym/repoman/repos.py
@@ -100,18 +100,22 @@ class RepoSettings(object):
 			sys.exit(1)
 
 		manifest_hashes = self.repo_config.manifest_hashes
+		manifest_required_hashes = self.repo_config.manifest_required_hashes
 		if manifest_hashes is None:
 			manifest_hashes = portage.const.MANIFEST2_HASH_DEFAULTS
+			manifest_required_hashes = manifest_hashes
 
 		if options.mode in ("commit", "fix", "manifest"):
-			if portage.const.MANIFEST2_REQUIRED_HASH not in manifest_hashes:
+			missing_required_hashes = manifest_required_hashes.difference(
+				manifest_hashes)
+			if missing_required_hashes:
 				msg = (
 					"The 'manifest-hashes' setting in the '%s' repository's "
-					"metadata/layout.conf does not contain the '%s' hash which "
-					"is required by this portage version. You will have to "
-					"upgrade portage if you want to generate valid manifests for "
+					"metadata/layout.conf does not contain the '%s' hashes which "
+					"are listed in 'manifest-required-hashes'. Please fix that "
+					"file if you want to generate valid manifests for "
 					"this repository.") % (
-					self.repo_config.name, portage.const.MANIFEST2_REQUIRED_HASH)
+					self.repo_config.name, ' '.join(missing_required_hashes))
 				for line in textwrap.wrap(msg, 70):
 					logging.error(line)
 				sys.exit(1)
-- 
2.15.0