[gentoo-commits] proj/portage:master commit in: pym/repoman/modules/scan/arches/, pym/repoman/modules/scan/eclasses/, ...

[Brian Dolbec]

commit:     a05acb46919638ef60a4303c2bc6d92e9e6c0771
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Sun Apr 24 01:34:37 2016 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Mon Apr 25 15:28:53 2016 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=a05acb46

More new data structure change fixes

Fix arches not being cleared every ebuild.

 pym/repoman/modules/scan/arches/arches.py     |  1 +
 pym/repoman/modules/scan/depend/depend.py     |  1 +
 pym/repoman/modules/scan/ebuild/__init__.py   |  2 +-
 pym/repoman/modules/scan/ebuild/ebuild.py     |  2 +-
 pym/repoman/modules/scan/ebuild/isebuild.py   | 11 ++++--
 pym/repoman/modules/scan/eclasses/live.py     |  5 ++-
 pym/repoman/modules/scan/keywords/__init__.py |  4 +-
 pym/repoman/modules/scan/metadata/__init__.py |  2 +-
 pym/repoman/modules/scan/metadata/unused.py   |  2 +-
 pym/repoman/modules/scan/use/use_flags.py     | 13 +++---
 pym/repoman/scanner.py                        | 57 +++++++++++++++++----------
 11 files changed, 62 insertions(+), 38 deletions(-)

diff --git a/pym/repoman/modules/scan/arches/arches.py b/pym/repoman/modules/scan/arches/arches.py
index 8bc7a24..4df25a8 100644
--- a/pym/repoman/modules/scan/arches/arches.py
+++ b/pym/repoman/modules/scan/arches/arches.py
@@ -69,6 +69,7 @@ class ArchChecks(ScanBase):
 				arches.add(('**', '**', ('**',)))
 		# update the dynamic data
 		dyn_arches = kwargs.get('arches')
+		#dyn_arches.clear()
 		dyn_arches.update(arches)
 		return False
 

diff --git a/pym/repoman/modules/scan/depend/depend.py b/pym/repoman/modules/scan/depend/depend.py
index a5f91d3..cd1fb33 100644
--- a/pym/repoman/modules/scan/depend/depend.py
+++ b/pym/repoman/modules/scan/depend/depend.py
@@ -145,6 +145,7 @@ class DependChecks(ScanBase):
 
 		# update the dynamic data
 		dyn_unknown = kwargs.get('unknown_pkgs')
+		dyn_unknown.clear()
 		dyn_unknown.update(unknown_pkgs)
 		self.set_result_pass([
 			(kwargs.get('badlicsyntax'), badlicsyntax),

diff --git a/pym/repoman/modules/scan/ebuild/__init__.py b/pym/repoman/modules/scan/ebuild/__init__.py
index ea21ece..efb2feb 100644
--- a/pym/repoman/modules/scan/ebuild/__init__.py
+++ b/pym/repoman/modules/scan/ebuild/__init__.py
@@ -21,7 +21,7 @@ module_spec = {
 			'mod_kwargs': ['portdb', 'qatracker', 'repo_settings'
 			],
 			'func_kwargs': {'checkdirlist': 'list', 'checkdir': None, 'xpkg': None,
-				'validity_fuse': None, 'can_force': None, 'pkgs': 'dict',
+				'validity_future': 'Future', 'can_force': None, 'pkgs': 'dict',
 			},
 		},
 		'ebuild-module': {

diff --git a/pym/repoman/modules/scan/ebuild/ebuild.py b/pym/repoman/modules/scan/ebuild/ebuild.py
index b015985..b02cdcd 100644
--- a/pym/repoman/modules/scan/ebuild/ebuild.py
+++ b/pym/repoman/modules/scan/ebuild/ebuild.py
@@ -86,7 +86,7 @@ class Ebuild(ScanBase):
 			self.qatracker.add_error(
 				"ebuild.notadded", self.xpkg + "/" + self.y_ebuild + ".ebuild")
 		# update the dynamic data
-		self.set_result_raise([('ebuild', self)])
+		self.set_result_raise([(kwargs.get('ebuild'), self)])
 		return False
 
 	def set_pkg_data(self, **kwargs):

diff --git a/pym/repoman/modules/scan/ebuild/isebuild.py b/pym/repoman/modules/scan/ebuild/isebuild.py
index 3e288a3..24680e2 100644
--- a/pym/repoman/modules/scan/ebuild/isebuild.py
+++ b/pym/repoman/modules/scan/ebuild/isebuild.py
@@ -13,6 +13,7 @@ from portage import os
 from repoman.qa_data import no_exec, allvars
 from repoman.modules.scan.scanbase import ScanBase
 
+
 class IsEbuild(ScanBase):
 	'''Performs basic tests to confirm it is an ebuild'''
 
@@ -88,9 +89,13 @@ class IsEbuild(ScanBase):
 			self.continue_ = True
 			self.set_result_pass([(can_force, False)])
 		# set our updated data
-		self.set_result_raise([
-			(kwargs.get('pkgs'), pkgs),
-			])
+		dyn_pkgs = kwargs.get('pkgs')
+		# clear() sets it to None,
+		# we don't want to kill the pointer reference
+		# just set it back to an empty dict()
+		for key in list(dyn_pkgs):
+			dyn_pkgs.pop(key)
+		dyn_pkgs.update(pkgs)
 		return self.continue_
 
 	@property

diff --git a/pym/repoman/modules/scan/eclasses/live.py b/pym/repoman/modules/scan/eclasses/live.py
index e5bb7dc..85e829c 100644
--- a/pym/repoman/modules/scan/eclasses/live.py
+++ b/pym/repoman/modules/scan/eclasses/live.py
@@ -25,10 +25,11 @@ class LiveEclassChecks(ScanBase):
 
 		@returns: dictionary, including {live_ebuild}
 		'''
+		ebuild = kwargs.get('ebuild').result()
 		# update the dynamic data
 		dyn_live = kwargs.get('live_ebuild')
-		dyn_live.update(LIVE_ECLASSES.intersection(
-				kwargs.get('ebuild').inherited))
+		#dyn_live.clear()
+		dyn_live.update(LIVE_ECLASSES.intersection(ebuild.inherited))
 		return False
 
 	def check(self, **kwargs):

diff --git a/pym/repoman/modules/scan/keywords/__init__.py b/pym/repoman/modules/scan/keywords/__init__.py
index 2d92dba..2b3f6ca 100644
--- a/pym/repoman/modules/scan/keywords/__init__.py
+++ b/pym/repoman/modules/scan/keywords/__init__.py
@@ -20,8 +20,8 @@ module_spec = {
 			},
 			'mod_kwargs': ['qatracker', 'options', 'repo_metadata', 'profiles',
 			],
-			'func_kwargs': {'pkg': None, 'xpkg': None, 'ebuild': None,
-				'y_ebuild': None, 'changed': None, 'live_ebuild': None,
+			'func_kwargs': {'pkg': 'Future', 'xpkg': None, 'ebuild': 'Future',
+				'y_ebuild': None, 'changed': None, 'live_ebuild': 'set',
 			},
 		},
 	}

diff --git a/pym/repoman/modules/scan/metadata/__init__.py b/pym/repoman/modules/scan/metadata/__init__.py
index af9771b..9ccf377 100644
--- a/pym/repoman/modules/scan/metadata/__init__.py
+++ b/pym/repoman/modules/scan/metadata/__init__.py
@@ -48,7 +48,7 @@ module_spec = {
 			},
 			'mod_kwargs': ['qatracker',
 			],
-			'func_kwargs': {'ebuild': None, 'pkg': None,
+			'func_kwargs': {'ebuild': None, 'pkg': 'Future',
 			},
 		},
 		'license-metadata': {

diff --git a/pym/repoman/modules/scan/metadata/unused.py b/pym/repoman/modules/scan/metadata/unused.py
index 0184d78..fdca418 100644
--- a/pym/repoman/modules/scan/metadata/unused.py
+++ b/pym/repoman/modules/scan/metadata/unused.py
@@ -23,7 +23,7 @@ class UnusedCheck(ScanBase):
 		@param validity_future: Future instance
 		'''
 		xpkg = kwargs.get('xpkg')
-		muselist = kwargs.get('muselist').result()
+		muselist = self.get_result(kwargs.get('muselist'), set())
 		used_useflags = kwargs.get('used_useflags')
 		valid_state = self.get_result(kwargs['validity_future'], True)
 		# check if there are unused local USE-descriptions in metadata.xml

diff --git a/pym/repoman/modules/scan/use/use_flags.py b/pym/repoman/modules/scan/use/use_flags.py
index 70f7583..b2b1cc0 100644
--- a/pym/repoman/modules/scan/use/use_flags.py
+++ b/pym/repoman/modules/scan/use/use_flags.py
@@ -42,20 +42,21 @@ class USEFlagChecks(ScanBase):
 		package = kwargs.get('xpkg')
 		ebuild = kwargs.get('ebuild').result()
 		y_ebuild = kwargs.get('y_ebuild')
-		localUseFlags = kwargs.get('muselist').result()
+		localUseFlags = self.get_result(kwargs.get('muselist'), set())
+		dyn_ebuild_used = kwargs.get('ebuild_UsedUseFlags')
+		dyn_used = kwargs.get('used_useflags')
 		# reset state variables for the run
 		self.useFlags = []
 		self.defaultUseFlags = []
 		self.usedUseFlags = set()
+		dyn_ebuild_used.clear()
+		# perform the checks
 		self._checkGlobal(pkg)
 		self._checkMetadata(package, ebuild, y_ebuild, localUseFlags)
 		self._checkRequiredUSE(pkg, ebuild)
 		# update the dynamic data
-		used_useflags = kwargs.get('used_useflags').union(self.usedUseFlags)
-		dyn_ebuild_used = kwargs.get('ebuild_UsedUseFlags')
-		dyn_ebuild_used.update(self.usedUseFlags)
-		dyn_used = kwargs.get('used_useflags')
-		dyn_used.update(used_useflags)
+		dyn_ebuild_used.union(self.usedUseFlags)
+		dyn_used.update(self.usedUseFlags)
 		return False
 
 

diff --git a/pym/repoman/scanner.py b/pym/repoman/scanner.py
index 3d2c69c..eaf023b 100644
--- a/pym/repoman/scanner.py
+++ b/pym/repoman/scanner.py
@@ -203,11 +203,6 @@ class Scanner(object):
 		}
 		# initialize the plugin checks here
 		self.modules = {}
-		for mod in ['manifests', 'isebuild', 'keywords', 'files', 'vcsstatus',
-					'fetches', 'pkgmetadata']:
-			mod_class = MODULE_CONTROLLER.get_class(mod)
-			logging.debug("Initializing class name: %s", mod_class.__name__)
-			self.modules[mod_class.__name__] = mod_class(**self.set_kwargs(mod))
 
 	def set_kwargs(self, mod):
 		'''Creates a limited set of kwargs to pass to the module's __init__()
@@ -231,17 +226,25 @@ class Scanner(object):
 		'''
 		func_kwargs = MODULE_CONTROLLER.modules[mod]['func_kwargs']
 		# determine new keys
-		required = set(func_kwargs.viewkeys())
-		exist = set(dynamic_data.viewkeys())
-		new = exist.difference(required)
+		required = set(list(func_kwargs))
+		exist = set(list(dynamic_data))
+		new = required.difference(exist)
 		# update dynamic_data with initialized entries
 		for key in new:
-			dynamic_data[key] = DATA_TYPES[func_kwargs['key']]()
+			logging.debug("set_func_kwargs(); adding: %s, %s",
+				key, func_kwargs[key])
+			dynamic_data[key] = DATA_TYPES[func_kwargs[key]]()
 		kwargs = {}
 		for key in required:
 			kwargs[key] = dynamic_data[key]
 		return kwargs
 
+	def reset_futures(self, dynamic_data):
+		for key in list(dynamic_data):
+			#if key in ['ebuild', 'pkg']:  # and isinstance(dynamic_data[key], Future):
+			if isinstance(dynamic_data[key], Future) and key not in ['muselist']:
+				dynamic_data[key] = Future()
+
 	def scan_pkgs(self, can_force):
 		for xpkg in self.effective_scanlist:
 			xpkg_continue = False
@@ -270,16 +273,26 @@ class Scanner(object):
 				'repolevel': self.repolevel,
 				'catdir': catdir,
 				'pkgdir': pkgdir,
-				'validity_future': Future()
+				'validity_future': Future(),
+				'y_ebuild': None,
+				# this needs to be reset at the pkg level only,
+				# easiest is to just initialize it here
+				'muselist': Future(),
 				}
 			# need to set it up for ==> self.modules or some other ordered list
-			for mod in ['Manifests', 'IsEbuild', 'KeywordChecks', 'FileChecks',
-						'VCSStatus', 'FetchChecks', 'PkgMetadata']:
-				logging.debug("scan_pkgs; module: %s", mod)
-				do_it, functions = self.modules[mod].runInPkgs
+			for mod in [('manifests', 'Manifests'), ('isebuild', 'IsEbuild'),
+						('keywords', 'KeywordChecks'), ('files', 'FileChecks'),
+						('vcsstatus', 'VCSStatus'), ('fetches', 'FetchChecks'),
+						('pkgmetadata', 'PkgMetadata'),
+						]:
+				mod_class = MODULE_CONTROLLER.get_class(mod[0])
+				logging.debug("Initializing class name: %s", mod_class.__name__)
+				self.modules[mod_class.__name__] = mod_class(**self.set_kwargs(mod[0]))
+				logging.debug("scan_pkgs; module: %s", mod[1])
+				do_it, functions = self.modules[mod[1]].runInPkgs
 				if do_it:
 					for func in functions:
-						_continue = func(**self.set_func_kwargs(mod, dynamic_data))
+						_continue = func(**self.set_func_kwargs(mod[0], dynamic_data))
 						if _continue:
 							# If we can't access all the metadata then it's totally unsafe to
 							# commit since there's no way to generate a correct Manifest.
@@ -312,6 +325,7 @@ class Scanner(object):
 		dynamic_data['used_useflags'] = set()
 
 		for y_ebuild in ebuildlist:
+			self.reset_futures(dynamic_data)
 			dynamic_data['y_ebuild'] = y_ebuild
 			y_ebuild_continue = False
 
@@ -320,7 +334,8 @@ class Scanner(object):
 			for mod in [('ebuild', 'Ebuild'), ('live', 'LiveEclassChecks'),
 				('eapi', 'EAPIChecks'), ('ebuild_metadata', 'EbuildMetadata'),
 				('thirdpartymirrors', 'ThirdPartyMirrors'),
-				('description', 'DescriptionChecks'), (None, 'KeywordChecks'),
+				('description', 'DescriptionChecks'),
+				('keywords', 'KeywordChecks'),
 				('arches', 'ArchChecks'), ('depend', 'DependChecks'),
 				('use_flags', 'USEFlagChecks'), ('ruby', 'RubyEclassChecks'),
 				('license', 'LicenseChecks'), ('restrict', 'RestrictChecks'),
@@ -329,17 +344,17 @@ class Scanner(object):
 				('options', 'Options'), ('profile', 'ProfileDependsChecks'),
 				('unknown', 'DependUnknown'),
 				]:
-				if mod[0]:
+				if mod[0] and mod[1] not in self.modules:
 					mod_class = MODULE_CONTROLLER.get_class(mod[0])
 					logging.debug("Initializing class name: %s", mod_class.__name__)
-					self.modules[mod[1]] = mod_class(**self.set_kwargs(mod))
+					self.modules[mod[1]] = mod_class(**self.set_kwargs(mod[0]))
 				logging.debug("scan_ebuilds: module: %s", mod[1])
 				do_it, functions = self.modules[mod[1]].runInEbuilds
 				logging.debug("do_it: %s, functions: %s", do_it, [x.__name__ for x in functions])
 				if do_it:
 					for func in functions:
 						logging.debug("\tRunning function: %s", func)
-						_continue = func(**self.set_func_kwargs(mod, dynamic_data))
+						_continue = func(**self.set_func_kwargs(mod[0], dynamic_data))
 						if _continue:
 							# If we can't access all the metadata then it's totally unsafe to
 							# commit since there's no way to generate a correct Manifest.
@@ -363,14 +378,14 @@ class Scanner(object):
 			if mod[0]:
 				mod_class = MODULE_CONTROLLER.get_class(mod[0])
 				logging.debug("Initializing class name: %s", mod_class.__name__)
-				self.modules[mod[1]] = mod_class(**self.set_kwargs(mod))
+				self.modules[mod[1]] = mod_class(**self.set_kwargs(mod[0]))
 			logging.debug("scan_ebuilds final checks: module: %s", mod[1])
 			do_it, functions = self.modules[mod[1]].runInFinal
 			logging.debug("do_it: %s, functions: %s", do_it, [x.__name__ for x in functions])
 			if do_it:
 				for func in functions:
 					logging.debug("\tRunning function: %s", func)
-					_continue = func(**self.set_func_kwargs(mod, dynamic_data))
+					_continue = func(**self.set_func_kwargs(mod[0], dynamic_data))
 					if _continue:
 						xpkg_complete = True
 						# logging.debug("\t>>> Continuing")