[gentoo-commits] proj/portage:repoman commit in: repoman/pym/repoman/, repoman/pym/repoman/modules/scan/

[Brian Dolbec]

commit:     3b4030e3e2f85bb92463dd89f9fa2f514a49b56a
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Mon Jul 10 17:39:04 2017 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Mon Jul 10 17:50:19 2017 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=3b4030e3

repoman: Create a new ModuleConfig class

This class will load the module configuration data from the target repo.
Move the Modules class handling code to this new class.
Update scanner.py to use the new ModuleConfig class for Modules class
calls.
Move the hard-coded loop lists to use the ModuleConfig instance
configure/dynamically determined lists.

TODO: add masters inheritance config stacking.

 repoman/pym/repoman/modules/scan/module.py | 84 ++++++++++++++++++++++++++++++
 repoman/pym/repoman/scanner.py             | 83 ++++++++++++-----------------
 2 files changed, 118 insertions(+), 49 deletions(-)

diff --git a/repoman/pym/repoman/modules/scan/module.py b/repoman/pym/repoman/modules/scan/module.py
new file mode 100644
index 000000000..f7f135f73
--- /dev/null
+++ b/repoman/pym/repoman/modules/scan/module.py
@@ -0,0 +1,84 @@
+
+'''
+moudules/scan/module.py
+Module loading and run list generator
+'''
+
+import logging
+import os
+import yaml
+
+from portage.module import InvalidModuleName, Modules
+from portage.util import stack_lists
+
+MODULES_PATH = os.path.dirname(__file__)
+# initial development debug info
+logging.debug("module path: %s", MODULES_PATH)
+
+
+class ModuleConfig(object):
+	'''Holds the scan modules configuration information and
+	creates the ordered list of modulles to run'''
+
+	def __init__(self, configpaths):
+		'''Module init
+
+		@param configpaths: ordered list of filepaths to load
+		'''
+		self.configpaths = configpaths
+
+		self.controller = Modules(path=MODULES_PATH, namepath="repoman.modules.scan")
+		logging.debug("module_names: %s", self.controller.module_names)
+
+		self._configs = None
+		self.enabled = []
+		self.pkgs_loop = []
+		self.ebuilds_loop = []
+		self.final_loop = []
+		self.modules_forced = ['ebuild', 'mtime']
+		self.load_configs()
+		for loop in ['pkgs', 'ebuilds', 'final']:
+			logging.debug("ModuleConfig; Processing loop %s", loop)
+			setattr(self, '%s_loop' % loop, self._determine_list(loop))
+
+
+	def load_configs(self, configpaths=None):
+		'''load the config files in order'''
+		if configpaths:
+			self.configpaths = configpaths
+		elif not self.configpaths:
+			logging.error("ModuleConfig; Error: No repository.yml files defined")
+		configs = []
+		for path in self.configpaths:
+			logging.debug("ModuleConfig; Processing: %s", path)
+			if os.path.exists(path):
+				try:
+					with open(path, 'r') as inputfile:
+						configs.append(yaml.safe_load(inputfile.read()))
+				except IOError as error:
+					logging,error("Failed to load file: %s", inputfile)
+					logging.exception(error)
+			logging.debug("ModuleConfig; completed : %s", path)
+		logging.debug("ModuleConfig; new _configs: %s", configs)
+		self._configs = configs
+
+	def _determine_list(self, loop):
+		'''Determine the ordered list from the config data and
+		the moule_runsIn value in the module_spec
+
+		@returns: list of modules
+		'''
+		lists = [c['modules'] for c in self._configs]
+		stacked = self.modules_forced + stack_lists(lists)
+		mlist = []
+		try:
+			for mod in stacked:
+				logging.debug("ModuleConfig; checking loop %s, module: %s, in: %s",
+					loop, mod, self.controller.get_spec(mod, 'module_runsIn'))
+				if loop in self.controller.get_spec(mod, 'module_runsIn'):
+					mlist.append(mod)
+		except InvalidModuleName:
+			logging.error("ModuleConfig; unkown module: %s, skipping", mod)
+
+		logging.debug("ModuleConfig; mlist: %s", mlist)
+		return mlist

diff --git a/repoman/pym/repoman/scanner.py b/repoman/pym/repoman/scanner.py
index e13d4f68b..ffce701be 100644
--- a/repoman/pym/repoman/scanner.py
+++ b/repoman/pym/repoman/scanner.py
@@ -15,20 +15,10 @@ from repoman.modules.commit import repochecks
 from repoman.modules.commit import manifest
 from repoman.profile import check_profiles, dev_profile_keywords, setup_profile
 from repoman.repos import repo_metadata
+from repoman.modules.scan.module import ModuleConfig
 from repoman.modules.scan.scan import scan
 from repoman.modules.vcs.vcs import vcs_files_to_cps
 
-from portage.module import Modules
-
-MODULES_PATH = os.path.join(os.path.dirname(__file__), "modules", "scan")
-# initial development debug info
-logging.debug("module path: %s", MODULES_PATH)
-
-MODULE_CONTROLLER = Modules(path=MODULES_PATH, namepath="repoman.modules.scan")
-
-MODULE_NAMES = MODULE_CONTROLLER.module_names[:]
-# initial development debug info
-logging.debug("module_names: %s", MODULE_NAMES)
 
 DATA_TYPES = {'dict': dict, 'Future': ExtendedFuture, 'list': list, 'set': set}
 
@@ -124,6 +114,12 @@ class Scanner(object):
 		if self.vcs_settings.vcs is None:
 			self.options.echangelog = 'n'
 
+		# Initialize the ModuleConfig class here
+		# TODO Add layout.conf masters repository.yml config to the list to load/stack
+		self.moduleconfig = ModuleConfig([
+				'/home/bdolbec/git/gentoo/metadata/repoman/repository.yml',
+				])
+
 		checks = {}
 		# The --echangelog option causes automatic ChangeLog generation,
 		# which invalidates changelog.ebuildadded and changelog.missing
@@ -212,7 +208,7 @@ class Scanner(object):
 		@returns: dictionary
 		'''
 		kwargs = {}
-		for key in MODULE_CONTROLLER.modules[mod]['mod_kwargs']:
+		for key in self.moduleconfig.controller.modules[mod]['mod_kwargs']:
 			kwargs[key] = self.kwargs[key]
 		return kwargs
 
@@ -224,7 +220,7 @@ class Scanner(object):
 		@param dynamic_data: dictionary structure
 		@returns: dictionary
 		'''
-		func_kwargs = MODULE_CONTROLLER.modules[mod]['func_kwargs']
+		func_kwargs = self.moduleconfig.controller.modules[mod]['func_kwargs']
 		# determine new keys
 		required = set(list(func_kwargs))
 		exist = set(list(dynamic_data))
@@ -324,19 +320,16 @@ class Scanner(object):
 				'validity_future',
 				]
 			# need to set it up for ==> self.modules or some other ordered list
-			for mod in [('manifests', 'Manifests'), ('ebuild', 'Ebuild'),
-						('keywords', 'KeywordChecks'), ('files', 'FileChecks'),
-						('fetches', 'FetchChecks'),
-						('pkgmetadata', 'PkgMetadata'),
-						]:
-				mod_class = MODULE_CONTROLLER.get_class(mod[0])
+			logging.debug("***** starting pkgs_loop: %s", self.moduleconfig.pkgs_loop)
+			for mod in self.moduleconfig.pkgs_loop:
+				mod_class = self.moduleconfig.controller.get_class(mod)
 				logging.debug("Initializing class name: %s", mod_class.__name__)
-				self.modules[mod_class.__name__] = mod_class(**self.set_kwargs(mod[0]))
-				logging.debug("scan_pkgs; module: %s", mod[1])
-				do_it, functions = self.modules[mod[1]].runInPkgs
+				self.modules[mod_class.__name__] = mod_class(**self.set_kwargs(mod))
+				logging.debug("scan_pkgs; module: %s", mod_class.__name__)
+				do_it, functions = self.modules[mod_class.__name__].runInPkgs
 				if do_it:
 					for func in functions:
-						_continue = func(**self.set_func_kwargs(mod[0], dynamic_data))
+						_continue = func(**self.set_func_kwargs(mod, dynamic_data))
 						if _continue:
 							# If we can't access all the metadata then it's totally unsafe to
 							# commit since there's no way to generate a correct Manifest.
@@ -373,28 +366,19 @@ class Scanner(object):
 
 			# initialize per ebuild plugin checks here
 			# need to set it up for ==> self.modules_list or some other ordered list
-			for mod in [('ebuild', 'Ebuild'), ('live', 'LiveEclassChecks'),
-				('eapi', 'EAPIChecks'), ('ebuild_metadata', 'EbuildMetadata'),
-				('fetches', 'FetchChecks'),
-				('description', 'DescriptionChecks'),
-				('keywords', 'KeywordChecks'),
-				('pkgmetadata', 'PkgMetadata'), ('ruby', 'RubyEclassChecks'),
-				('restrict', 'RestrictChecks'),
-				('mtime', 'MtimeChecks'), ('multicheck', 'MultiCheck'),
-				# Options.is_forced() is used to bypass further checks
-				('options', 'Options'), ('profile', 'ProfileDependsChecks'),
-				]:
-				if mod[0] and mod[1] not in self.modules:
-					mod_class = MODULE_CONTROLLER.get_class(mod[0])
-					logging.debug("Initializing class name: %s", mod_class.__name__)
-					self.modules[mod[1]] = mod_class(**self.set_kwargs(mod[0]))
-				logging.debug("scan_ebuilds: module: %s", mod[1])
-				do_it, functions = self.modules[mod[1]].runInEbuilds
+			for mod in self.moduleconfig.ebuilds_loop:
+				if mod:
+					mod_class = self.moduleconfig.controller.get_class(mod)
+					if mod_class.__name__ not in self.modules:
+						logging.debug("Initializing class name: %s", mod_class.__name__)
+						self.modules[mod_class.__name__] = mod_class(**self.set_kwargs(mod))
+				logging.debug("scan_ebuilds: module: %s", mod_class.__name__)
+				do_it, functions = self.modules[mod_class.__name__].runInEbuilds
 				logging.debug("do_it: %s, functions: %s", do_it, [x.__name__ for x in functions])
 				if do_it:
 					for func in functions:
 						logging.debug("\tRunning function: %s", func)
-						_continue = func(**self.set_func_kwargs(mod[0], dynamic_data))
+						_continue = func(**self.set_func_kwargs(mod, dynamic_data))
 						if _continue:
 							# If we can't access all the metadata then it's totally unsafe to
 							# commit since there's no way to generate a correct Manifest.
@@ -414,18 +398,19 @@ class Scanner(object):
 		# initialize per pkg plugin final checks here
 		# need to set it up for ==> self.modules_list or some other ordered list
 		xpkg_complete = False
-		for mod in [('pkgmetadata', 'PkgMetadata'), ('keywords', 'KeywordChecks')]:
-			if mod[0] and mod[1] not in self.modules:
-				mod_class = MODULE_CONTROLLER.get_class(mod[0])
-				logging.debug("Initializing class name: %s", mod_class.__name__)
-				self.modules[mod[1]] = mod_class(**self.set_kwargs(mod[0]))
-			logging.debug("scan_ebuilds final checks: module: %s", mod[1])
-			do_it, functions = self.modules[mod[1]].runInFinal
+		for mod in self.moduleconfig.final_loop:
+			if mod:
+				mod_class = self.moduleconfig.controller.get_class(mod)
+				if mod_class.__name__ not in self.modules:
+					logging.debug("Initializing class name: %s", mod_class.__name__)
+					self.modules[mod_class.__name__] = mod_class(**self.set_kwargs(mod))
+			logging.debug("scan_ebuilds final checks: module: %s", mod_class.__name__)
+			do_it, functions = self.modules[mod_class.__name__].runInFinal
 			logging.debug("do_it: %s, functions: %s", do_it, [x.__name__ for x in functions])
 			if do_it:
 				for func in functions:
 					logging.debug("\tRunning function: %s", func)
-					_continue = func(**self.set_func_kwargs(mod[0], dynamic_data))
+					_continue = func(**self.set_func_kwargs(mod, dynamic_data))
 					if _continue:
 						xpkg_complete = True
 						# logging.debug("\t>>> Continuing")