[gentoo-commits] proj/portage:master commit in: pym/repoman/modules/scan/metadata/, pym/repoman/modules/scan/depend/, ...

[Brian Dolbec]

commit:     836cc3ffc55376348315577958e7212fefa38a2a
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Mon Apr 25 16:50:59 2016 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Mon Apr 25 16:58:53 2016 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=836cc3ff

ProfileDependsChecks: encapsulate DependChecks and LicenseChecks

 pym/repoman/modules/scan/depend/__init__.py       |  38 +-----
 pym/repoman/modules/scan/depend/_depend_checks.py | 150 ++++++++++++++++++++
 pym/repoman/modules/scan/depend/depend.py         | 158 ----------------------
 pym/repoman/modules/scan/depend/profile.py        |  17 ++-
 pym/repoman/modules/scan/metadata/__init__.py     |  17 ---
 pym/repoman/modules/scan/metadata/license.py      |  54 --------
 pym/repoman/scanner.py                            |   4 +-
 7 files changed, 167 insertions(+), 271 deletions(-)

diff --git a/pym/repoman/modules/scan/depend/__init__.py b/pym/repoman/modules/scan/depend/__init__.py
index 01bd116..6d12286 100644
--- a/pym/repoman/modules/scan/depend/__init__.py
+++ b/pym/repoman/modules/scan/depend/__init__.py
@@ -10,24 +10,6 @@ module_spec = {
 	'name': 'depend',
 	'description': doc,
 	'provides':{
-		'depend-module': {
-			'name': "depend",
-			'sourcefile': "depend",
-			'class': "DependChecks",
-			'description': doc,
-			'functions': ['check'],
-			'func_desc': {
-			},
-			'mod_kwargs': ['qatracker', 'portdb'
-			],
-			'func_kwargs': {
-				'baddepsyntax': ('Future', False),
-				'badlicsyntax': ('Future', False),
-				'ebuild': (None, None),
-				'pkg': (None, None),
-				'unknown_pkgs': ('Future', 'UNSET'),
-			},
-		},
 		'profile-module': {
 			'name': "profile",
 			'sourcefile': "profile",
@@ -37,30 +19,12 @@ module_spec = {
 			'func_desc': {
 			},
 			'mod_kwargs': ['qatracker', 'portdb', 'profiles', 'options',
-				'repo_settings', 'include_arches', 'caches',
+				'repo_metadata', 'repo_settings', 'include_arches', 'caches',
 				'repoman_incrementals', 'env', 'have', 'dev_keywords'
 			],
 			'func_kwargs': {
-				'baddepsyntax': (None, None),
 				'ebuild': (None, None),
 				'pkg': (None, None),
-				'unknown_pkgs': (None, None),
-			},
-		},
-		'unknown-module': {
-			'name': "unknown",
-			'sourcefile': "unknown",
-			'class': "DependUnknown",
-			'description': doc,
-			'functions': ['check'],
-			'func_desc': {
-			},
-			'mod_kwargs': ['qatracker',
-			],
-			'func_kwargs': {
-				'baddepsyntax': (None, None),
-				'ebuild': (None, None),
-				'unknown_pkgs': ('Future', 'UNSET'),
 			},
 		},
 	}

diff --git a/pym/repoman/modules/scan/depend/_depend_checks.py b/pym/repoman/modules/scan/depend/_depend_checks.py
new file mode 100644
index 0000000..4e1d216
--- /dev/null
+++ b/pym/repoman/modules/scan/depend/_depend_checks.py
@@ -0,0 +1,150 @@
+# -*- coding:utf-8 -*-
+
+
+from _emerge.Package import Package
+
+from repoman.check_missingslot import check_missingslot
+# import our initialized portage instance
+from repoman._portage import portage
+from repoman.qa_data import suspect_virtual, suspect_rdepend
+
+
+def _depend_checks(ebuild, pkg, portdb, qatracker, repo_metadata):
+	'''Checks the ebuild dependencies for errors
+
+	@param pkg: Package in which we check (object).
+	@param ebuild: Ebuild which we check (object).
+	@param portdb: portdb instance
+	@param qatracker: QATracker instance
+	@param repo_metadata: dictionary of various repository items.
+	@returns: (unknown_pkgs, badlicsyntax)
+	'''
+
+	unknown_pkgs = set()
+
+	inherited_java_eclass = "java-pkg-2" in ebuild.inherited or \
+		"java-pkg-opt-2" in ebuild.inherited,
+	inherited_wxwidgets_eclass = "wxwidgets" in ebuild.inherited
+	# operator_tokens = set(["||", "(", ")"])
+	type_list, badsyntax = [], []
+	for mytype in Package._dep_keys + ("LICENSE", "PROPERTIES", "PROVIDE"):
+		mydepstr = ebuild.metadata[mytype]
+
+		buildtime = mytype in Package._buildtime_keys
+		runtime = mytype in Package._runtime_keys
+		token_class = None
+		if mytype.endswith("DEPEND"):
+			token_class = portage.dep.Atom
+
+		try:
+			atoms = portage.dep.use_reduce(
+				mydepstr, matchall=1, flat=True,
+				is_valid_flag=pkg.iuse.is_valid_flag, token_class=token_class)
+		except portage.exception.InvalidDependString as e:
+			atoms = None
+			badsyntax.append(str(e))
+
+		if atoms and mytype.endswith("DEPEND"):
+			if runtime and \
+				"test?" in mydepstr.split():
+				qatracker.add_error(
+					mytype + '.suspect',
+					"%s: 'test?' USE conditional in %s" %
+					(ebuild.relative_path, mytype))
+
+			for atom in atoms:
+				if atom == "||":
+					continue
+
+				is_blocker = atom.blocker
+
+				# Skip dependency.unknown for blockers, so that we
+				# don't encourage people to remove necessary blockers,
+				# as discussed in bug 382407. We use atom.without_use
+				# due to bug 525376.
+				if not is_blocker and \
+					not portdb.xmatch("match-all", atom.without_use) and \
+					not atom.cp.startswith("virtual/"):
+					unknown_pkgs.add((mytype, atom.unevaluated_atom))
+
+				if pkg.category != "virtual":
+					if not is_blocker and \
+						atom.cp in suspect_virtual:
+						qatracker.add_error(
+							'virtual.suspect', ebuild.relative_path +
+							": %s: consider using '%s' instead of '%s'" %
+							(mytype, suspect_virtual[atom.cp], atom))
+					if not is_blocker and \
+						atom.cp.startswith("perl-core/"):
+						qatracker.add_error('dependency.perlcore',
+							ebuild.relative_path +
+							": %s: please use '%s' instead of '%s'" %
+							(mytype,
+							atom.replace("perl-core/","virtual/perl-"),
+							atom))
+
+				if buildtime and \
+					not is_blocker and \
+					not inherited_java_eclass and \
+					atom.cp == "virtual/jdk":
+					qatracker.add_error(
+						'java.eclassesnotused', ebuild.relative_path)
+				elif buildtime and \
+					not is_blocker and \
+					not inherited_wxwidgets_eclass and \
+					atom.cp == "x11-libs/wxGTK":
+					qatracker.add_error(
+						'wxwidgets.eclassnotused',
+						"%s: %ss on x11-libs/wxGTK without inheriting"
+						" wxwidgets.eclass" % (ebuild.relative_path, mytype))
+				elif runtime:
+					if not is_blocker and \
+						atom.cp in suspect_rdepend:
+						qatracker.add_error(
+							mytype + '.suspect',
+							ebuild.relative_path + ": '%s'" % atom)
+
+				if atom.operator == "~" and \
+					portage.versions.catpkgsplit(atom.cpv)[3] != "r0":
+					qacat = 'dependency.badtilde'
+					qatracker.add_error(
+						qacat, "%s: %s uses the ~ operator"
+						" with a non-zero revision: '%s'" %
+						(ebuild.relative_path, mytype, atom))
+
+				check_missingslot(atom, mytype, ebuild.eapi, portdb, qatracker,
+					ebuild.relative_path, ebuild.metadata)
+
+		type_list.extend([mytype] * (len(badsyntax) - len(type_list)))
+
+	for m, b in zip(type_list, badsyntax):
+		if m.endswith("DEPEND"):
+			qacat = "dependency.syntax"
+		else:
+			qacat = m + ".syntax"
+		qatracker.add_error(
+			qacat, "%s: %s: %s" % (ebuild.relative_path, m, b))
+
+	# data required for some other tests
+	badlicsyntax = len([z for z in type_list if z == "LICENSE"])
+	badprovsyntax = len([z for z in type_list if z == "PROVIDE"])
+	baddepsyntax = len(type_list) != badlicsyntax + badprovsyntax
+	badlicsyntax = badlicsyntax > 0
+	#badprovsyntax = badprovsyntax > 0
+
+	# Parse the LICENSE variable, remove USE conditions and flatten it.
+	licenses = portage.dep.use_reduce(
+		ebuild.metadata["LICENSE"], matchall=1, flat=True)
+
+	# Check each entry to ensure that it exists in ${PORTDIR}/licenses/.
+	for lic in licenses:
+		# Need to check for "||" manually as no portage
+		# function will remove it without removing values.
+		if lic not in repo_metadata['liclist'] and lic != "||":
+			qatracker.add_error("LICENSE.invalid",
+				"%s: %s" % (ebuild.relative_path, lic))
+		elif lic in repo_metadata['lic_deprecated']:
+			qatracker.add_error("LICENSE.deprecated",
+				"%s: %s" % (ebuild.relative_path, lic))
+
+	return unknown_pkgs, baddepsyntax

diff --git a/pym/repoman/modules/scan/depend/depend.py b/pym/repoman/modules/scan/depend/depend.py
deleted file mode 100644
index 44a21e9..0000000
--- a/pym/repoman/modules/scan/depend/depend.py
+++ /dev/null
@@ -1,158 +0,0 @@
-# -*- coding:utf-8 -*-
-
-
-from _emerge.Package import Package
-
-from repoman.check_missingslot import check_missingslot
-# import our initialized portage instance
-from repoman._portage import portage
-from repoman.modules.scan.scanbase import ScanBase
-from repoman.qa_data import suspect_virtual, suspect_rdepend
-
-
-class DependChecks(ScanBase):
-	'''Perform dependency checks'''
-
-	def __init__(self, **kwargs):
-		'''
-		@param portdb: portdb instance
-		@param qatracker: QATracker instance
-		'''
-		self.qatracker = kwargs.get('qatracker')
-		self.portdb = kwargs.get('portdb')
-
-	def check(self, **kwargs):
-		'''Checks the ebuild dependencies for errors
-
-		@param pkg: Package in which we check (object).
-		@param ebuild: Ebuild which we check (object).
-		@returns: boolean
-		'''
-		ebuild = kwargs.get('ebuild').get()
-		pkg = kwargs.get('pkg').get()
-
-		unknown_pkgs = set()
-
-		inherited_java_eclass = "java-pkg-2" in ebuild.inherited or \
-			"java-pkg-opt-2" in ebuild.inherited,
-		inherited_wxwidgets_eclass = "wxwidgets" in ebuild.inherited
-		# operator_tokens = set(["||", "(", ")"])
-		type_list, badsyntax = [], []
-		for mytype in Package._dep_keys + ("LICENSE", "PROPERTIES", "PROVIDE"):
-			mydepstr = ebuild.metadata[mytype]
-
-			buildtime = mytype in Package._buildtime_keys
-			runtime = mytype in Package._runtime_keys
-			token_class = None
-			if mytype.endswith("DEPEND"):
-				token_class = portage.dep.Atom
-
-			try:
-				atoms = portage.dep.use_reduce(
-					mydepstr, matchall=1, flat=True,
-					is_valid_flag=pkg.iuse.is_valid_flag, token_class=token_class)
-			except portage.exception.InvalidDependString as e:
-				atoms = None
-				badsyntax.append(str(e))
-
-			if atoms and mytype.endswith("DEPEND"):
-				if runtime and \
-					"test?" in mydepstr.split():
-					self.qatracker.add_error(
-						mytype + '.suspect',
-						"%s: 'test?' USE conditional in %s" %
-						(ebuild.relative_path, mytype))
-
-				for atom in atoms:
-					if atom == "||":
-						continue
-
-					is_blocker = atom.blocker
-
-					# Skip dependency.unknown for blockers, so that we
-					# don't encourage people to remove necessary blockers,
-					# as discussed in bug 382407. We use atom.without_use
-					# due to bug 525376.
-					if not is_blocker and \
-						not self.portdb.xmatch("match-all", atom.without_use) and \
-						not atom.cp.startswith("virtual/"):
-						unknown_pkgs.add((mytype, atom.unevaluated_atom))
-
-					if kwargs.get('catdir') != "virtual":
-						if not is_blocker and \
-							atom.cp in suspect_virtual:
-							self.qatracker.add_error(
-								'virtual.suspect', ebuild.relative_path +
-								": %s: consider using '%s' instead of '%s'" %
-								(mytype, suspect_virtual[atom.cp], atom))
-						if not is_blocker and \
-							atom.cp.startswith("perl-core/"):
-							self.qatracker.add_error('dependency.perlcore',
-								ebuild.relative_path +
-								": %s: please use '%s' instead of '%s'" %
-								(mytype,
-								atom.replace("perl-core/","virtual/perl-"),
-								atom))
-
-					if buildtime and \
-						not is_blocker and \
-						not inherited_java_eclass and \
-						atom.cp == "virtual/jdk":
-						self.qatracker.add_error(
-							'java.eclassesnotused', ebuild.relative_path)
-					elif buildtime and \
-						not is_blocker and \
-						not inherited_wxwidgets_eclass and \
-						atom.cp == "x11-libs/wxGTK":
-						self.qatracker.add_error(
-							'wxwidgets.eclassnotused',
-							"%s: %ss on x11-libs/wxGTK without inheriting"
-							" wxwidgets.eclass" % (ebuild.relative_path, mytype))
-					elif runtime:
-						if not is_blocker and \
-							atom.cp in suspect_rdepend:
-							self.qatracker.add_error(
-								mytype + '.suspect',
-								ebuild.relative_path + ": '%s'" % atom)
-
-					if atom.operator == "~" and \
-						portage.versions.catpkgsplit(atom.cpv)[3] != "r0":
-						qacat = 'dependency.badtilde'
-						self.qatracker.add_error(
-							qacat, "%s: %s uses the ~ operator"
-							" with a non-zero revision: '%s'" %
-							(ebuild.relative_path, mytype, atom))
-
-					check_missingslot(atom, mytype, ebuild.eapi, self.portdb, self.qatracker,
-						ebuild.relative_path, ebuild.metadata)
-
-			type_list.extend([mytype] * (len(badsyntax) - len(type_list)))
-
-		for m, b in zip(type_list, badsyntax):
-			if m.endswith("DEPEND"):
-				qacat = "dependency.syntax"
-			else:
-				qacat = m + ".syntax"
-			self.qatracker.add_error(
-				qacat, "%s: %s: %s" % (ebuild.relative_path, m, b))
-
-		# data required for some other tests
-		badlicsyntax = len([z for z in type_list if z == "LICENSE"])
-		badprovsyntax = len([z for z in type_list if z == "PROVIDE"])
-		baddepsyntax = len(type_list) != badlicsyntax + badprovsyntax
-		badlicsyntax = badlicsyntax > 0
-		#badprovsyntax = badprovsyntax > 0
-
-		# update the dynamic data
-		dyn_unknown = kwargs.get('unknown_pkgs')
-		dyn_unknown.set(unknown_pkgs)
-		dyn_badlicsyntax = kwargs.get('badlicsyntax')
-		dyn_badlicsyntax.set(badlicsyntax, ignore_InvalidState=True)
-		dyn_baddepsyntax = kwargs.get('baddepsyntax')
-		dyn_baddepsyntax.set(baddepsyntax, ignore_InvalidState=True)
-		return False
-
-	@property
-	def runInEbuilds(self):
-		'''Ebuild level scans'''
-		return (True, [self.check])

diff --git a/pym/repoman/modules/scan/depend/profile.py b/pym/repoman/modules/scan/depend/profile.py
index 0388374..5b32f66 100644
--- a/pym/repoman/modules/scan/depend/profile.py
+++ b/pym/repoman/modules/scan/depend/profile.py
@@ -9,6 +9,7 @@ from _emerge.Package import Package
 # import our initialized portage instance
 from repoman._portage import portage
 from repoman.modules.scan.scanbase import ScanBase
+from repoman.modules.scan.depend._depend_checks import _depend_checks
 from repoman.modules.scan.depend._gen_arches import _gen_arches
 from portage.dep import Atom
 
@@ -34,6 +35,7 @@ class ProfileDependsChecks(ScanBase):
 		@param env: the environment
 		@param have: dictionary instance
 		@param dev_keywords: developer profile keywords
+		@param repo_metadata: dictionary of various repository items.
 		'''
 		self.qatracker = kwargs.get('qatracker')
 		self.portdb = kwargs.get('portdb')
@@ -46,6 +48,7 @@ class ProfileDependsChecks(ScanBase):
 		self.env = kwargs.get('env')
 		self.have = kwargs.get('have')
 		self.dev_keywords = kwargs.get('dev_keywords')
+		self.repo_metadata = kwargs.get('repo_metadata')
 
 	def check(self, **kwargs):
 		'''Perform profile dependant dependancy checks
@@ -59,8 +62,8 @@ class ProfileDependsChecks(ScanBase):
 		'''
 		ebuild = kwargs.get('ebuild').get()
 		pkg = kwargs.get('pkg').get()
-		baddepsyntax = kwargs.get('baddepsyntax').get()
-		unknown_pkgs = kwargs.get('unknown_pkgs').get()
+		unknown_pkgs, baddepsyntax = _depend_checks(
+			ebuild, pkg, self.portdb, self.qatracker, self.repo_metadata)
 
 		relevant_profiles = []
 		for keyword, arch, groups in _gen_arches(ebuild, self.options,
@@ -226,6 +229,16 @@ class ProfileDependsChecks(ScanBase):
 								"%s: %s: %s(%s)\n%s"
 								% (ebuild.relative_path, mytype, keyword,
 									prof, pformat(atoms, indent=6)))
+
+		if not baddepsyntax and unknown_pkgs:
+			type_map = {}
+			for mytype, atom in unknown_pkgs:
+				type_map.setdefault(mytype, set()).add(atom)
+			for mytype, atoms in type_map.items():
+				self.qatracker.add_error(
+					"dependency.unknown", "%s: %s: %s"
+					% (ebuild.relative_path, mytype, ", ".join(sorted(atoms))))
+
 		return False
 
 	@property

diff --git a/pym/repoman/modules/scan/metadata/__init__.py b/pym/repoman/modules/scan/metadata/__init__.py
index 1a6c0ed..c297676 100644
--- a/pym/repoman/modules/scan/metadata/__init__.py
+++ b/pym/repoman/modules/scan/metadata/__init__.py
@@ -61,23 +61,6 @@ module_spec = {
 				'pkg': ('Future', 'UNSET'),
 			},
 		},
-		'license-metadata': {
-			'name': "license",
-			'sourcefile': "license",
-			'class': "LicenseChecks",
-			'description': doc,
-			'functions': ['check'],
-			'func_desc': {
-			},
-			'mod_kwargs': ['qatracker', 'repo_metadata',
-			],
-			'func_kwargs': {
-				'badlicsyntax': (None, None),
-				'ebuild': (None, None),
-				'xpkg': (None, None),
-				'y_ebuild': (None, None),
-			},
-		},
 		'restrict-metadata': {
 			'name': "restrict",
 			'sourcefile': "restrict",

diff --git a/pym/repoman/modules/scan/metadata/license.py b/pym/repoman/modules/scan/metadata/license.py
deleted file mode 100644
index 4a67c70..0000000
--- a/pym/repoman/modules/scan/metadata/license.py
+++ /dev/null
@@ -1,54 +0,0 @@
-
-'''license.py
-Perform checks on the LICENSE variable.
-'''
-
-# import our initialized portage instance
-from repoman._portage import portage
-from repoman.modules.scan.scanbase import ScanBase
-
-
-class LicenseChecks(ScanBase):
-	'''Perform checks on the LICENSE variable.'''
-
-	def __init__(self, **kwargs):
-		'''
-		@param qatracker: QATracker instance
-		@param repo_metadata: dictionary of various repository items.
-		'''
-		self.qatracker = kwargs.get('qatracker')
-		self.repo_metadata = kwargs.get('repo_metadata')
-
-	def check(self, **kwargs):
-		'''
-		@param xpkg: Package in which we check (string).
-		@param ebuild: Ebuild which we check (object).
-		@param y_ebuild: Ebuild which we check (string).
-		'''
-		xpkg = kwargs.get('xpkg')
-		ebuild = kwargs.get('ebuild').get()
-		y_ebuild = kwargs.get('y_ebuild')
-		if not kwargs.get('badlicsyntax').get():
-			# Parse the LICENSE variable, remove USE conditions and flatten it.
-			licenses = portage.dep.use_reduce(
-				ebuild.metadata["LICENSE"], matchall=1, flat=True)
-
-			# Check each entry to ensure that it exists in ${PORTDIR}/licenses/.
-			for lic in licenses:
-				# Need to check for "||" manually as no portage
-				# function will remove it without removing values.
-				if lic not in self.repo_metadata['liclist'] and lic != "||":
-					self.qatracker.add_error("LICENSE.invalid",
-						"%s/%s.ebuild: %s" % (xpkg, y_ebuild, lic))
-				elif lic in self.repo_metadata['lic_deprecated']:
-					self.qatracker.add_error("LICENSE.deprecated",
-						"%s: %s" % (ebuild.relative_path, lic))
-		return False
-
-	@property
-	def runInPkgs(self):
-		return (False, [])
-
-	@property
-	def runInEbuilds(self):
-		return (True, [self.check])

diff --git a/pym/repoman/scanner.py b/pym/repoman/scanner.py
index 2224fa3..f6dcc1b 100644
--- a/pym/repoman/scanner.py
+++ b/pym/repoman/scanner.py
@@ -380,13 +380,11 @@ class Scanner(object):
 				('thirdpartymirrors', 'ThirdPartyMirrors'),
 				('description', 'DescriptionChecks'),
 				('keywords', 'KeywordChecks'),
-				('depend', 'DependChecks'),
 				('use_flags', 'USEFlagChecks'), ('ruby', 'RubyEclassChecks'),
-				('license', 'LicenseChecks'), ('restrict', 'RestrictChecks'),
+				('restrict', 'RestrictChecks'),
 				('mtime', 'MtimeChecks'), ('multicheck', 'MultiCheck'),
 				# Options.is_forced() is used to bypass further checks
 				('options', 'Options'), ('profile', 'ProfileDependsChecks'),
-				('unknown', 'DependUnknown'),
 				]:
 				if mod[0] and mod[1] not in self.modules:
 					mod_class = MODULE_CONTROLLER.get_class(mod[0])