[gentoo-commits] proj/portage:master commit in: man/, pym/portage/package/ebuild/, pym/portage/dbapi/, ...

[Zac Medico]

commit:     accee1b7c61da284022f86d9ab39bcb492ea4023
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Tue Aug 21 20:22:19 2012 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Tue Aug 21 20:22:19 2012 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=accee1b7

Implement PORTAGE_CHECKSUM_FILTER for bug #432170

---
 man/make.conf.5                                    |   19 +++++++-
 pym/_emerge/EbuildFetcher.py                       |    4 +-
 pym/portage/checksum.py                            |   54 ++++++++++++++++++++
 pym/portage/dbapi/bintree.py                       |    6 ++-
 pym/portage/manifest.py                            |   15 +++--
 .../package/ebuild/_config/special_env_vars.py     |    1 +
 pym/portage/package/ebuild/digestcheck.py          |   13 +++--
 pym/portage/package/ebuild/fetch.py                |   13 +++--
 8 files changed, 107 insertions(+), 18 deletions(-)

diff --git a/man/make.conf.5 b/man/make.conf.5
index c617fbc..59f32b8 100644
--- a/man/make.conf.5
+++ b/man/make.conf.5
@@ -1,4 +1,4 @@
-.TH "MAKE.CONF" "5" "Jul 2012" "Portage VERSION" "Portage"
+.TH "MAKE.CONF" "5" "Aug 2012" "Portage VERSION" "Portage"
 .SH "NAME"
 make.conf \- custom settings for Portage
 .SH "SYNOPSIS"
@@ -697,6 +697,23 @@ for bzip2 compression operations. \fBPORTAGE_BZIP2_COMMAND\fR will also be
 called for extraction operation, with -d appended, unless the
 \fBPORTAGE_BUNZIP2_COMMAND\fR variable is set.
 .TP
+\fBPORTAGE_CHECKSUM_FILTER\fR = \fI[space delimited list of hash names]\fR
+This variable may be used to filter the hash functions that are used to
+verify integrity of files. Hash functions names are case\-insensitive, and
+the \fI*\fR and \fI\-*\fR wildcard tokens are supported.
+.br
+Defaults to the value of *.
+.br
+.I Examples:
+.nf
+# Use all available hash functions
+PORTAGE_CHECKSUM_FILTER="*"
+# Use any function except whirlpool
+PORTAGE_CHECKSUM_FILTER="* \-whirlpool"
+# Only use sha256
+PORTAGE_CHECKSUM_FILTER="\-* sha256"
+.fi
+.TP
 \fBPORTAGE_COMPRESS\fR = \fI"bzip2"\fR
 This variable contains the command used to compress documentation during the
 install phase.

diff --git a/pym/_emerge/EbuildFetcher.py b/pym/_emerge/EbuildFetcher.py
index c0a7fdd..bbcb6a9 100644
--- a/pym/_emerge/EbuildFetcher.py
+++ b/pym/_emerge/EbuildFetcher.py
@@ -13,6 +13,7 @@ from portage import os
 from portage import _encodings
 from portage import _unicode_encode
 from portage import _unicode_decode
+from portage.checksum import _hash_filter
 from portage.elog.messages import eerror
 from portage.package.ebuild.fetch import _check_distfile, fetch
 from portage.util._pty import _create_pty_or_pipe
@@ -57,6 +58,7 @@ class EbuildFetcher(SpawnProcess):
 			if st.st_size != expected_size:
 				return False
 
+		hash_filter = _hash_filter(settings.get("PORTAGE_CHECKSUM_FILTER", ""))
 		stdout_orig = sys.stdout
 		stderr_orig = sys.stderr
 		global_havecolor = portage.output.havecolor
@@ -78,7 +80,7 @@ class EbuildFetcher(SpawnProcess):
 						break
 					continue
 				ok, st = _check_distfile(os.path.join(distdir, filename),
-					mydigests, eout, show_errors=False)
+					mydigests, eout, show_errors=False, hash_filter=hash_filter)
 				if not ok:
 					success = False
 					break

diff --git a/pym/portage/checksum.py b/pym/portage/checksum.py
index daf4a0c..de4cc66 100644
--- a/pym/portage/checksum.py
+++ b/pym/portage/checksum.py
@@ -217,6 +217,60 @@ def _filter_unaccelarated_hashes(digests):
 
 	return digests
 
+class _hash_filter(object):
+	"""
+	Implements filtering for PORTAGE_CHECKSUM_FILTER.
+	"""
+
+	__slots__ = ('transparent', '_tokens',)
+
+	def __init__(self, filter_str):
+		tokens = filter_str.upper().split()
+		if not tokens or tokens[-1] == "*":
+			del tokens[:]
+		self.transparent = not tokens
+		tokens.reverse()
+		self._tokens = tuple(tokens)
+
+	def __call__(self, hash_name):
+		if self.transparent:
+			return True
+		matches = ("*", hash_name)
+		for token in self._tokens:
+			if token in matches:
+				return True
+			elif token[:1] == "-":
+				if token[1:] in matches:
+					return False
+		return False
+
+def _apply_hash_filter(digests, hash_filter):
+	"""
+	Return a new dict containing the filtered digests, or the same
+	dict if no changes are necessary. This will always preserve at
+	at least one digest, in order to ensure that they are not all
+	discarded.
+	"""
+	if hash_filter.transparent:
+		return digests
+
+	verifiable_hash_types = set(digests).intersection(hashfunc_map)
+	verifiable_hash_types.discard("size")
+	modified = False
+	if len(verifiable_hash_types) > 1:
+		for k in list(verifiable_hash_types):
+			if not hash_filter(k):
+				modified = True
+				verifiable_hash_types.remove(k)
+				if len(verifiable_hash_types) == 1:
+					break
+
+	if modified:
+		digests = dict((k, v) for (k, v) in digests.items()
+			if k == "size" or k in verifiable_hash_types)
+
+	return digests
+
 def verify_all(filename, mydict, calc_prelink=0, strict=0):
 	"""
 	Verify all checksums against a file.

diff --git a/pym/portage/dbapi/bintree.py b/pym/portage/dbapi/bintree.py
index 6c01867..a2fd5ea 100644
--- a/pym/portage/dbapi/bintree.py
+++ b/pym/portage/dbapi/bintree.py
@@ -5,7 +5,8 @@ __all__ = ["bindbapi", "binarytree"]
 
 import portage
 portage.proxy.lazyimport.lazyimport(globals(),
-	'portage.checksum:hashfunc_map,perform_multiple_checksums,verify_all',
+	'portage.checksum:hashfunc_map,perform_multiple_checksums,' + \
+		'verify_all,_apply_hash_filter,_hash_filter',
 	'portage.dbapi.dep_expand:dep_expand',
 	'portage.dep:dep_getkey,isjustname,isvalidatom,match_from_list',
 	'portage.output:EOutput,colorize',
@@ -1462,6 +1463,9 @@ class binarytree(object):
 		if not digests:
 			return False
 
+		hash_filter = _hash_filter(
+			self.settings.get("PORTAGE_CHECKSUM_FILTER", ""))
+		digests = _apply_hash_filter(digests, hash_filter)
 		eout = EOutput()
 		eout.quiet = self.settings.get("PORTAGE_QUIET") == "1"
 		ok, st = _check_distfile(pkg_path, digests, eout, show_errors=0)

diff --git a/pym/portage/manifest.py b/pym/portage/manifest.py
index a04b717..b2f1ff2 100644
--- a/pym/portage/manifest.py
+++ b/pym/portage/manifest.py
@@ -9,7 +9,7 @@ import warnings
 import portage
 portage.proxy.lazyimport.lazyimport(globals(),
 	'portage.checksum:hashfunc_map,perform_multiple_checksums,' + \
-		'verify_all,_filter_unaccelarated_hashes',
+		'verify_all,_apply_hash_filter,_filter_unaccelarated_hashes',
 	'portage.util:write_atomic',
 )
 
@@ -502,14 +502,17 @@ class Manifest(object):
 		for t in MANIFEST2_IDENTIFIERS:
 			self.checkTypeHashes(t, ignoreMissingFiles=ignoreMissingFiles)
 	
-	def checkTypeHashes(self, idtype, ignoreMissingFiles=False):
+	def checkTypeHashes(self, idtype, ignoreMissingFiles=False, hash_filter=None):
 		for f in self.fhashdict[idtype]:
-			self.checkFileHashes(idtype, f, ignoreMissing=ignoreMissingFiles)
+			self.checkFileHashes(idtype, f, ignoreMissing=ignoreMissingFiles,
+				hash_filter=hash_filter)
 	
-	def checkFileHashes(self, ftype, fname, ignoreMissing=False):
+	def checkFileHashes(self, ftype, fname, ignoreMissing=False, hash_filter=None):
+		digests = _filter_unaccelarated_hashes(self.fhashdict[ftype][fname])
+		if hash_filter is not None:
+			digests = _apply_hash_filter(digests, hash_filter)
 		try:
-			ok, reason = verify_all(self._getAbsname(ftype, fname),
-				_filter_unaccelarated_hashes(self.fhashdict[ftype][fname]))
+			ok, reason = verify_all(self._getAbsname(ftype, fname), digests)
 			if not ok:
 				raise DigestException(tuple([self._getAbsname(ftype, fname)]+list(reason)))
 			return ok, reason

diff --git a/pym/portage/package/ebuild/_config/special_env_vars.py b/pym/portage/package/ebuild/_config/special_env_vars.py
index 6ed6d05..c974eb9 100644
--- a/pym/portage/package/ebuild/_config/special_env_vars.py
+++ b/pym/portage/package/ebuild/_config/special_env_vars.py
@@ -150,6 +150,7 @@ environ_filter += [
 	"PORTAGE_BACKGROUND", "PORTAGE_BACKGROUND_UNMERGE",
 	"PORTAGE_BINHOST",
 	"PORTAGE_BINHOST_CHUNKSIZE", "PORTAGE_BUILDIR_LOCKED",
+	"PORTAGE_CHECKSUM_FILTER",
 	"PORTAGE_ELOG_CLASSES",
 	"PORTAGE_ELOG_MAILFROM", "PORTAGE_ELOG_MAILSUBJECT",
 	"PORTAGE_ELOG_MAILURI", "PORTAGE_ELOG_SYSTEM",

diff --git a/pym/portage/package/ebuild/digestcheck.py b/pym/portage/package/ebuild/digestcheck.py
index 8705639..1d59948 100644
--- a/pym/portage/package/ebuild/digestcheck.py
+++ b/pym/portage/package/ebuild/digestcheck.py
@@ -1,4 +1,4 @@
-# Copyright 2010-2011 Gentoo Foundation
+# Copyright 2010-2012 Gentoo Foundation
 # Distributed under the terms of the GNU General Public License v2
 
 __all__ = ['digestcheck']
@@ -6,6 +6,7 @@ __all__ = ['digestcheck']
 import warnings
 
 from portage import os, _encodings, _unicode_decode
+from portage.checksum import _hash_filter
 from portage.exception import DigestException, FileNotFound
 from portage.localization import _
 from portage.output import EOutput
@@ -28,6 +29,7 @@ def digestcheck(myfiles, mysettings, strict=False, justmanifest=None, mf=None):
 	if mysettings.get("EBUILD_SKIP_MANIFEST") == "1":
 		return 1
 	pkgdir = mysettings["O"]
+	hash_filter = _hash_filter(mysettings.get("PORTAGE_CHECKSUM_FILTER", ""))
 	if mf is None:
 		mf = mysettings.repositories.get_repo_for_location(
 			os.path.dirname(os.path.dirname(pkgdir)))
@@ -38,15 +40,16 @@ def digestcheck(myfiles, mysettings, strict=False, justmanifest=None, mf=None):
 		if not mf.thin and strict and "PORTAGE_PARALLEL_FETCHONLY" not in mysettings:
 			if mf.fhashdict.get("EBUILD"):
 				eout.ebegin(_("checking ebuild checksums ;-)"))
-				mf.checkTypeHashes("EBUILD")
+				mf.checkTypeHashes("EBUILD", hash_filter=hash_filter)
 				eout.eend(0)
 			if mf.fhashdict.get("AUX"):
 				eout.ebegin(_("checking auxfile checksums ;-)"))
-				mf.checkTypeHashes("AUX")
+				mf.checkTypeHashes("AUX", hash_filter=hash_filter)
 				eout.eend(0)
 			if mf.fhashdict.get("MISC"):
 				eout.ebegin(_("checking miscfile checksums ;-)"))
-				mf.checkTypeHashes("MISC", ignoreMissingFiles=True)
+				mf.checkTypeHashes("MISC", ignoreMissingFiles=True,
+					hash_filter=hash_filter)
 				eout.eend(0)
 		for f in myfiles:
 			eout.ebegin(_("checking %s ;-)") % f)
@@ -58,7 +61,7 @@ def digestcheck(myfiles, mysettings, strict=False, justmanifest=None, mf=None):
 				writemsg(_("\n!!! Missing digest for '%s'\n") % (f,),
 					noiselevel=-1)
 				return 0
-			mf.checkFileHashes(ftype, f)
+			mf.checkFileHashes(ftype, f, hash_filter=hash_filter)
 			eout.eend(0)
 	except FileNotFound as e:
 		eout.eend(1)

diff --git a/pym/portage/package/ebuild/fetch.py b/pym/portage/package/ebuild/fetch.py
index 60ed04d..8365ad2 100644
--- a/pym/portage/package/ebuild/fetch.py
+++ b/pym/portage/package/ebuild/fetch.py
@@ -26,7 +26,7 @@ portage.proxy.lazyimport.lazyimport(globals(),
 from portage import OrderedDict, os, selinux, shutil, _encodings, \
 	_shell_quote, _unicode_encode
 from portage.checksum import (hashfunc_map, perform_md5, verify_all,
-	_filter_unaccelarated_hashes)
+	_filter_unaccelarated_hashes, _hash_filter, _apply_hash_filter)
 from portage.const import BASH_BINARY, CUSTOM_MIRRORS_FILE, \
 	GLOBAL_CONFIG_PATH
 from portage.data import portage_gid, portage_uid, secpass, userpriv_groups
@@ -185,7 +185,7 @@ def _check_digests(filename, digests, show_errors=1):
 		return False
 	return True
 
-def _check_distfile(filename, digests, eout, show_errors=1):
+def _check_distfile(filename, digests, eout, show_errors=1, hash_filter=None):
 	"""
 	@return a tuple of (match, stat_obj) where match is True if filename
 	matches all given digests (if any) and stat_obj is a stat result, or
@@ -212,6 +212,8 @@ def _check_distfile(filename, digests, eout, show_errors=1):
 			return (False, st)
 	else:
 		digests = _filter_unaccelarated_hashes(digests)
+		if hash_filter is not None:
+			digests = _apply_hash_filter(digests, hash_filter)
 		if _check_digests(filename, digests, show_errors=show_errors):
 			eout.ebegin("%s %s ;-)" % (os.path.basename(filename),
 				" ".join(sorted(digests))))
@@ -355,6 +357,7 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0,
 		if try_mirrors:
 			mymirrors += [x.rstrip("/") for x in mysettings["GENTOO_MIRRORS"].split() if x]
 
+	hash_filter = _hash_filter(mysettings.get("PORTAGE_CHECKSUM_FILTER", ""))
 	skip_manifest = mysettings.get("EBUILD_SKIP_MANIFEST") == "1"
 	if skip_manifest:
 		allow_missing_digests = True
@@ -637,7 +640,7 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0,
 				eout = EOutput()
 				eout.quiet = mysettings.get("PORTAGE_QUIET") == "1"
 				match, mystat = _check_distfile(
-					myfile_path, pruned_digests, eout)
+					myfile_path, pruned_digests, eout, hash_filter=hash_filter)
 				if match:
 					# Skip permission adjustment for symlinks, since we don't
 					# want to modify anything outside of the primary DISTDIR,
@@ -709,7 +712,7 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0,
 					for x in ro_distdirs:
 						filename = os.path.join(x, myfile)
 						match, mystat = _check_distfile(
-							filename, pruned_digests, eout)
+							filename, pruned_digests, eout, hash_filter=hash_filter)
 						if match:
 							readonly_file = filename
 							break
@@ -796,6 +799,7 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0,
 							continue
 						else:
 							digests = _filter_unaccelarated_hashes(mydigests[myfile])
+							digests = _apply_hash_filter(digests, hash_filter)
 							verified_ok, reason = verify_all(myfile_path, digests)
 							if not verified_ok:
 								writemsg(_("!!! Previously fetched"
@@ -1053,6 +1057,7 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0,
 								# net connection. This way we have a chance to try to download
 								# from another mirror...
 								digests = _filter_unaccelarated_hashes(mydigests[myfile])
+								digests = _apply_hash_filter(digests, hash_filter)
 								verified_ok, reason = verify_all(myfile_path, digests)
 								if not verified_ok:
 									writemsg(_("!!! Fetched file: %s VERIFY FAILED!\n") % myfile,