[gentoo-portage-dev] [PATCH 1/3 v2] Have repoman check if the packages to unpack rare archive formats from SRC_URI are present in DEPEND (bug #205909).

[Tom Wijsman]

---
 bin/repoman         | 88 ++++++++++++++++++++++++++++++++++++++++++++++++++---
 man/repoman.1       |  4 +++
 pym/portage/eapi.py |  3 ++
 3 files changed, 91 insertions(+), 4 deletions(-)

diff --git a/bin/repoman b/bin/repoman
index d1542e9..eed1024 100755
--- a/bin/repoman
+++ b/bin/repoman
@@ -36,6 +36,14 @@ pym_path = osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), "pym")
 sys.path.insert(0, pym_path)
 import portage
 portage._internal_caller = True
+
+from portage._sets.profiles import PackagesSystemSet
+from portage import os
+
+config_root = os.environ.get("PORTAGE_CONFIGROOT")
+repoman_settings = portage.config(config_root=config_root, local_config=False)
+system_set_atoms = [atom.cp \
+	for atom in PackagesSystemSet(repoman_settings.profiles).getAtoms()]
 portage._disable_legacy_globals()
 
 try:
@@ -53,7 +61,6 @@ except (ImportError, SystemError, RuntimeError, Exception):
 		out.eerror(line)
 	sys.exit(1)
 
-from portage import os
 from portage import _encodings
 from portage import _unicode_encode
 import repoman.checks
@@ -78,7 +85,8 @@ from portage.output import ConsoleStyleFile, StyleWriter
 from portage.util import writemsg_level
 from portage.util._argparse import ArgumentParser
 from portage.package.ebuild.digestgen import digestgen
-from portage.eapi import eapi_has_iuse_defaults, eapi_has_required_use
+from portage.eapi import (eapi_has_iuse_defaults, eapi_has_required_use,
+	eapi_has_xz_utils)
 
 if sys.hexversion >= 0x3000000:
 	basestring = str
@@ -98,8 +106,6 @@ os.umask(0o22)
 # behave incrementally.
 repoman_incrementals = tuple(x for x in \
 	portage.const.INCREMENTALS if x != 'ACCEPT_KEYWORDS')
-config_root = os.environ.get("PORTAGE_CONFIGROOT")
-repoman_settings = portage.config(config_root=config_root, local_config=False)
 
 if repoman_settings.get("NOCOLOR", "").lower() in ("yes", "true") or \
 	repoman_settings.get('TERM') == 'dumb' or \
@@ -300,6 +306,7 @@ qahelp = {
 	"inherit.missing": "Ebuild uses functions from an eclass but does not inherit it",
 	"inherit.unused": "Ebuild inherits an eclass but does not use it",
 	"java.eclassesnotused": "With virtual/jdk in DEPEND you must inherit a java eclass",
+	"unpack.DEPEND.missing": "A rare archive format was used in SRC_URI, but its package to unpack it is missing in DEPEND.",
 	"wxwidgets.eclassnotused": "Ebuild DEPENDs on x11-libs/wxGTK without inheriting wxwidgets.eclass",
 	"KEYWORDS.dropped": "Ebuilds that appear to have dropped KEYWORDS for some arch",
 	"KEYWORDS.missing": "Ebuilds that have a missing or empty KEYWORDS variable",
@@ -399,6 +406,7 @@ qawarnings = set((
 "metadata.warning",
 "portage.internal",
 "repo.eapi.deprecated",
+"unpack.DEPEND.missing",
 "usage.obsolete",
 "upstream.workaround",
 "LIVEVCS.stable",
@@ -479,6 +487,65 @@ ruby_deprecated = frozenset([
 	"ruby_targets_ree18",
 ])
 
+def getNonSystemArchiveDepends(fetchlist, eapi):
+	"""
+	Compare each SRC_URI entry against archivers; if one of the
+	extensions match, we remember which archive depends are needed.
+
+	We don't pass back dependencies that are in the system set.
+
+	TODO:	Add functionality to support checking for deb2targz on platforms
+			where GNU binutils is absent; see PMS 5, section 11.3.3.13.
+	"""
+
+	def addMatchingFormatsToResult(formats, result):
+		for file_ext in formats:
+			format = formats[file_ext]
+
+			if format not in system_set_atoms:
+				for entry in fetchlist:
+					if file_ext.match(entry) is not None:
+						result.add(format)
+
+	result = set()
+
+	addMatchingFormatsToResult(getNonSystemArchiveDepends.archivers, result)
+
+	if eapi_has_xz_utils(eapi):
+		addMatchingFormatsToResult(getNonSystemArchiveDepends.archivers_xz, result)
+
+	return result
+
+getNonSystemArchiveDepends.archivers = {
+	re.compile('.*\.7[zZ]$'):"app-arch/p7zip",
+	re.compile('.*\.(bz2?|tbz2)$'):"app-arch/bzip2",
+	re.compile('.*\.jar$'):"app-arch/unzip",
+	re.compile('.*\.(LH[aA]|lha|lzh)$'):"app-arch/lha",
+	re.compile('.*\.lzma$'):"app-arch/lzma-utils",
+	re.compile('.*\.(rar|RAR)$'):"app-arch/unrar",
+	re.compile('.*\.(tar(\.(bz2?|gz|Z))?|tbz2|t[bg]z)?$'):"app-arch/tar",
+	re.compile('.*\.(gz|tar\.Z|t[bg]z|[zZ])$'):"app-arch/gzip",
+	re.compile('.*\.(zip|ZIP)$'):"app-arch/unzip",
+}
+
+getNonSystemArchiveDepends.archivers_xz = {
+	re.compile('.*\.tar.xz$'):"app-arch/tar",
+	re.compile('.*\.xz$'):"app-arch/xz-utils",
+}
+
+def checkArchiveDepends(atoms, catpkg, relative_path, \
+	system_set_atoms, needed_unpack_depends, stats, fails):
+	"""
+	We check whether the needed archive dependencies are present in DEPEND,
+	which were determined from SRC_URI.
+	"""
+	for entry in needed_unpack_depends[catpkg]:
+		if entry not in [atom.cp for atom in atoms if atom != "||"]:
+			stats['unpack.' + mytype + '.missing'] += 1
+			fails['unpack.' + mytype + '.missing'].append( \
+				relative_path + ": %s is missing in %s" % \
+				(entry, mytype))
+
 metadata_xml_encoding = 'UTF-8'
 metadata_xml_declaration = '<?xml version="1.0" encoding="%s"?>' % \
 	(metadata_xml_encoding,)
@@ -1559,6 +1626,7 @@ for x in effective_scanlist:
 	fetchlist_dict = portage.FetchlistDict(checkdir, repoman_settings, portdb)
 	myfiles_all = []
 	src_uri_error = False
+	needed_unpack_depends = {}
 	for mykey in fetchlist_dict:
 		try:
 			myfiles_all.extend(fetchlist_dict[mykey])
@@ -1573,7 +1641,13 @@ for x in effective_scanlist:
 				stats["SRC_URI.syntax"] += 1
 				fails["SRC_URI.syntax"].append(
 					"%s.ebuild SRC_URI: %s" % (mykey, e))
+
+		needed_unpack_depends[mykey] = \
+			getNonSystemArchiveDepends(fetchlist_dict[mykey], \
+				pkgs[y[:-7]]._metadata["EAPI"])
+
 	del fetchlist_dict
+
 	if not src_uri_error:
 		# This test can produce false positives if SRC_URI could not
 		# be parsed for one or more ebuilds. There's no point in
@@ -2010,6 +2084,10 @@ for x in effective_scanlist:
 				atoms = None
 				badsyntax.append(str(e))
 
+			if atoms and buildtime:
+				checkArchiveDepends(atoms, catpkg, relative_path, \
+					system_set_atoms, needed_unpack_depends, stats, fails)
+
 			if atoms and mytype.endswith("DEPEND"):
 				if runtime and \
 					"test?" in mydepstr.split():
@@ -2384,6 +2462,8 @@ for x in effective_scanlist:
 				"%s/metadata.xml: unused local USE-description: '%s'" % \
 				(x, myflag))
 
+	del needed_unpack_depends
+
 if options.if_modified == "y" and len(effective_scanlist) < 1:
 	logging.warn("--if-modified is enabled, but no modified packages were found!")
 
diff --git a/man/repoman.1 b/man/repoman.1
index a78f94e..fb89610 100644
--- a/man/repoman.1
+++ b/man/repoman.1
@@ -334,6 +334,10 @@ Ebuild inherits a deprecated eclass
 With virtual/jdk in DEPEND you must inherit a java eclass. Refer to
 \fIhttp://www.gentoo.org/proj/en/java/java\-devel.xml\fR for more information.
 .TP
+.B unpack.DEPEND.missing
+A rare archive format was used in SRC_URI, but the package to unpack it is
+missing from DEPEND.
+TP
 .B manifest.bad
 Manifest has missing or incorrect digests
 .TP
diff --git a/pym/portage/eapi.py b/pym/portage/eapi.py
index 4f77910..5f8919b 100644
--- a/pym/portage/eapi.py
+++ b/pym/portage/eapi.py
@@ -95,6 +95,9 @@ def eapi_has_hdepend(eapi):
 def eapi_has_targetroot(eapi):
 	return eapi in ("5-hdepend",)
 
+def eapi_has_xz_utils(eapi):
+	return eapi not in ("0", "1", "2")
+
 _eapi_attrs = collections.namedtuple('_eapi_attrs',
 	'dots_in_PN dots_in_use_flags exports_EBUILD_PHASE_FUNC '
 	'feature_flag_test feature_flag_targetroot '
-- 
1.8.5.2