[gentoo-commits] proj/tinderbox-cluster:master commit in: tbc/pym/, tbc/pym/repoman/

[Magnus Granberg]

commit:     801894358cc5569b03844bcd043931d441fb8508
Author:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Thu Jul 30 11:36:57 2015 +0000
Commit:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Thu Jul 30 11:36:57 2015 +0000
URL:        https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=80189435

add repoman full scan and split repoman/main to more functions

 tbc/pym/package.py           |   9 +-
 tbc/pym/qachecks.py          |  79 ++++++------
 tbc/pym/repoman/main.py      | 278 +++++++++++++++++++++----------------------
 tbc/pym/repoman/utilities.py |   4 +-
 4 files changed, 191 insertions(+), 179 deletions(-)

diff --git a/tbc/pym/package.py b/tbc/pym/package.py
index ba58e2e..0a19f15 100644
--- a/tbc/pym/package.py
+++ b/tbc/pym/package.py
@@ -8,7 +8,7 @@ from portage.xml.metadata import MetaDataXML
 from tbc.flags import tbc_use_flags
 from tbc.text import get_ebuild_cvs_revision, get_log_text_dict
 from tbc.flags import tbc_use_flags
-from tbc.qachecks import digestcheck, check_repoman
+from tbc.qachecks import digestcheck, check_repoman, repoman_full
 from tbc.sqlquerys import add_logs, get_package_info, get_config_info, \
 	add_new_build_job, add_new_ebuild_sql, get_ebuild_id_list, add_old_ebuild, \
 	get_package_metadata_sql, update_package_metadata, update_manifest_sql, \
@@ -282,6 +282,10 @@ class tbc_package(object):
 		manifest_checksum_tree = self.get_manifest_checksum_tree(pkgdir, cp, repo, mytree)
 		if manifest_checksum_tree is None:
 			return
+
+		# Check cp with repoman repoman full
+		repoman_full(self._session, pkgdir, self._config_id)
+
 		package_id = add_new_package_sql(self._session, cp, repo)
 		
 		package_metadataDict = self.get_package_metadataDict(pkgdir, package_id)
@@ -336,6 +340,9 @@ class tbc_package(object):
 			log_msg = "U %s:%s" % (cp, repo)
 			add_logs(self._session, log_msg, "info", self._config_id)
 
+			# Check cp with repoman repoman full
+			repoman_full(self._session, pkgdir, self._config_id)
+
 			# Get the ebuild list for cp
 			old_ebuild_id_list = []
 			ebuild_list_tree = self._myportdb.cp_list(cp, use_cache=1, mytree=mytree)

diff --git a/tbc/pym/qachecks.py b/tbc/pym/qachecks.py
index 8c522c6..2e25d86 100644
--- a/tbc/pym/qachecks.py
+++ b/tbc/pym/qachecks.py
@@ -15,6 +15,8 @@ from portage.exception import DigestException, FileNotFound, ParseError, Permiss
 from _emerge.Package import Package
 from _emerge.RootConfig import RootConfig
 from tbc.repoman.checks.ebuilds.checks import run_checks
+from tbc.repoman.main import repoman_main
+from tbc.sqlquerys import get_configmetadata_info, get_config_info, get_setup_info
 import portage
 
 # Copy of portage.digestcheck() but without the writemsg() stuff
@@ -142,38 +144,47 @@ def check_file_in_manifest(pkgdir, mysettings, portdb, cpv, build_use_flags_list
 	return
 
 def check_repoman(mysettings, myportdb, cpv, repo):
-		# We run repoman run_checks on the ebuild
-		ebuild_version_tree = portage.versions.cpv_getversion(cpv)
-		element = portage.versions.cpv_getkey(cpv).split('/')
-		categories = element[0]
-		package = element[1]
-		pkgdir = myportdb.getRepositoryPath(repo) + "/" + categories + "/" + package
-		full_path = pkgdir + "/" + package + "-" + ebuild_version_tree + ".ebuild"
-		root = '/'
-		trees = {
-		root : {'porttree' : portage.portagetree(root, settings=mysettings)}
-		}
-		root_config = RootConfig(mysettings, trees[root], None)
-		allvars = set(x for x in portage.auxdbkeys if not x.startswith("UNUSED_"))
-		allvars.update(Package.metadata_keys)
-		allvars = sorted(allvars)
-		myaux = dict(zip(allvars, myportdb.aux_get(cpv, allvars, myrepo=repo)))
-		pkg = Package(cpv=cpv, metadata=myaux, root_config=root_config, type_name='ebuild')
-		fails = []
+	# We run repoman run_checks on the ebuild
+	ebuild_version_tree = portage.versions.cpv_getversion(cpv)
+	element = portage.versions.cpv_getkey(cpv).split('/')
+	categories = element[0]
+	package = element[1]
+	pkgdir = myportdb.getRepositoryPath(repo) + "/" + categories + "/" + package
+	full_path = pkgdir + "/" + package + "-" + ebuild_version_tree + ".ebuild"
+	root = '/'
+	trees = {
+	root : {'porttree' : portage.portagetree(root, settings=mysettings)}
+	}
+	root_config = RootConfig(mysettings, trees[root], None)
+	allvars = set(x for x in portage.auxdbkeys if not x.startswith("UNUSED_"))
+	allvars.update(Package.metadata_keys)
+	allvars = sorted(allvars)
+	myaux = dict(zip(allvars, myportdb.aux_get(cpv, allvars, myrepo=repo)))
+	pkg = Package(cpv=cpv, metadata=myaux, root_config=root_config, type_name='ebuild')
+	fails = []
+	try:
+		# All ebuilds should have utf_8 encoding.
+		f = codecs.open(_unicode_encode(full_path,
+		encoding = _encodings['fs'], errors = 'strict'),
+		mode = 'r', encoding = _encodings['repo.content'])
 		try:
-			# All ebuilds should have utf_8 encoding.
-			f = codecs.open(_unicode_encode(full_path,
-			encoding = _encodings['fs'], errors = 'strict'),
-			mode = 'r', encoding = _encodings['repo.content'])
-			try:
-				for check_name, e in run_checks(f, pkg):
-					fails.append(check_name + ": " + e + "\n")
-			finally:
-				f.close()
-		except UnicodeDecodeError:
-			# A file.UTF8 failure will have already been recorded above.
-			pass
-		# fails will have a list with repoman errors
-		if fails == []:
-			return False
-		return fails
\ No newline at end of file
+			for check_name, e in run_checks(f, pkg):
+				fails.append(check_name + ": " + e + "\n")
+		finally:
+			f.close()
+	except UnicodeDecodeError:
+		# A file.UTF8 failure will have already been recorded above.
+		pass
+	# fails will have a list with repoman errors
+	if fails == []:
+		return False
+	return fails
+
+def repoman_full(session, pkgdir, config_id):
+	ConfigsMetaData = get_configmetadata_info(session, config_id)
+	ConfigInfo = get_config_info(session, config_id)
+	SetupInfo = get_setup_info(session, config_id)
+	config_root = ConfigsMetaData.RepoPath + '/' + ConfigInfo.Hostname + "/" + SetupInfo.Setup
+	argscmd = []
+	argscmd.append('full')
+	repoman_main(argscmd, config_root=config_root, pkgdir=pkgdir)

diff --git a/tbc/pym/repoman/main.py b/tbc/pym/repoman/main.py
index 2196009..5bdee0e 100755
--- a/tbc/pym/repoman/main.py
+++ b/tbc/pym/repoman/main.py
@@ -1,5 +1,4 @@
-#!/usr/bin/python -bO
-# Copyright 1999-2014 Gentoo Foundation
+# Copyright 1999-2015 Gentoo Foundation
 # Distributed under the terms of the GNU General Public License v2
 
 from __future__ import print_function, unicode_literals
@@ -165,113 +164,16 @@ def need_signature(filename):
 				return False
 			raise
 
-def repoman_main(argv):
-	# Repoman sets it's own ACCEPT_KEYWORDS and we don't want it to
-	# behave incrementally.
-	repoman_incrementals = tuple(
-		x for x in portage.const.INCREMENTALS if x != 'ACCEPT_KEYWORDS')
-	config_root = os.environ.get("PORTAGE_CONFIGROOT")
-	repoman_settings = portage.config(config_root=config_root, local_config=False)
-
-	if repoman_settings.get("NOCOLOR", "").lower() in ("yes", "true") or \
-		repoman_settings.get('TERM') == 'dumb' or \
-		not sys.stdout.isatty():
-		nocolor()
-
-	options, arguments = parse_args(
-		argv, qahelp, repoman_settings.get("REPOMAN_DEFAULT_OPTS", ""))
-
-	if options.version:
-		print("Portage", portage.VERSION)
-		sys.exit(0)
-
-	if options.experimental_inherit == 'y':
-		# This is experimental, so it's non-fatal.
-		qawarnings.add("inherit.missing")
-		checks_init(experimental_inherit=True)
-
-	# Set this to False when an extraordinary issue (generally
-	# something other than a QA issue) makes it impossible to
-	# commit (like if Manifest generation fails).
-	can_force = True
-
-	portdir, portdir_overlay, mydir = utilities.FindPortdir(repoman_settings)
-	if portdir is None:
-		sys.exit(1)
-
-	myreporoot = os.path.basename(portdir_overlay)
-	myreporoot += mydir[len(portdir_overlay):]
-	##################
-
-	vcs_settings = VCSSettings(options, repoman_settings)
-
-	##################
-
-	repo_settings = RepoSettings(
-		config_root, portdir, portdir_overlay,
-		repoman_settings, vcs_settings, options, qawarnings)
-
-	repoman_settings = repo_settings.repoman_settings
-
-	portdb = repo_settings.portdb
-	##################
-
-	if options.echangelog is None and repo_settings.repo_config.update_changelog:
-		options.echangelog = 'y'
-
-	if vcs_settings.vcs is None:
-		options.echangelog = 'n'
-
-	# The --echangelog option causes automatic ChangeLog generation,
-	# which invalidates changelog.ebuildadded and changelog.missing
-	# checks.
-	# Note: Some don't use ChangeLogs in distributed SCMs.
-	# It will be generated on server side from scm log,
-	# before package moves to the rsync server.
-	# This is needed because they try to avoid merge collisions.
-	# Gentoo's Council decided to always use the ChangeLog file.
-	# TODO: shouldn't this just be switched on the repo, iso the VCS?
-	is_echangelog_enabled = options.echangelog in ('y', 'force')
-	vcs_settings.vcs_is_cvs_or_svn = vcs_settings.vcs in ('cvs', 'svn')
-	check_changelog = not is_echangelog_enabled and vcs_settings.vcs_is_cvs_or_svn
+def repoman_scan(repoman_settings, repo_settings, vcs_settings, options, repolevel):
 
-	if 'digest' in repoman_settings.features and options.digest != 'n':
-		options.digest = 'y'
-
-	logging.debug("vcs: %s" % (vcs_settings.vcs,))
-	logging.debug("repo config: %s" % (repo_settings.repo_config,))
-	logging.debug("options: %s" % (options,))
-
-	# It's confusing if these warnings are displayed without the user
-	# being told which profile they come from, so disable them.
-	env = os.environ.copy()
-	env['FEATURES'] = env.get('FEATURES', '') + ' -unknown-features-warn'
-
-	categories = []
-	for path in repo_settings.repo_config.eclass_db.porttrees:
-		categories.extend(portage.util.grabfile(
-			os.path.join(path, 'profiles', 'categories')))
-	repoman_settings.categories = frozenset(
-		portage.util.stack_lists([categories], incremental=1))
-	categories = repoman_settings.categories
-
-	portdb.settings = repoman_settings
-	# We really only need to cache the metadata that's necessary for visibility
-	# filtering. Anything else can be discarded to reduce memory consumption.
-	portdb._aux_cache_keys.clear()
-	portdb._aux_cache_keys.update(
-		["EAPI", "IUSE", "KEYWORDS", "repository", "SLOT"])
-
-	reposplit = myreporoot.split(os.path.sep)
-	repolevel = len(reposplit)
-
-	###################
-
-	if options.mode == 'commit':
-		repochecks.commit_check(repolevel, reposplit)
-		repochecks.conflict_check(vcs_settings, options)
+	if options.mode == "manifest":
+		pass
+	elif options.pretend:
+		print(green("\nRepoMan does a once-over of the neighborhood..."))
+	else:
+		print(green("\nRepoMan scours the neighborhood..."))
 
-	###################
+	qatracker = QATracker()
 
 	# Make startdir relative to the canonical repodir, so that we can pass
 	# it to digestgen and it won't have to be canonicalized again.
@@ -281,7 +183,9 @@ def repoman_main(argv):
 		startdir = normalize_path(mydir)
 		startdir = os.path.join(
 			repo_settings.repodir, *startdir.split(os.sep)[-2 - repolevel + 3:])
-	###################
+
+	have_pmasked = False
+	have_dev_keywords = False
 
 	# get lists of valid keywords, licenses, and use
 	new_data = repo_metadata(repo_settings.portdb, repoman_settings)
@@ -305,28 +209,12 @@ def repoman_main(argv):
 
 	####################
 
-	dev_keywords = dev_keywords(profiles)
-
-	qatracker = QATracker()
-
-
-	if options.mode == "manifest":
-		pass
-	elif options.pretend:
-		print(green("\nRepoMan does a once-over of the neighborhood..."))
-	else:
-		print(green("\nRepoMan scours the neighborhood..."))
-
-	#####################
-
-	changed = Changes(options)
-	changed.scan(vcs_settings)
-
-	######################
+	dev_keyword = dev_keywords(profiles)
 
-	have_pmasked = False
-	have_dev_keywords = False
-	dofail = 0
+	# Disable the "ebuild.notadded" check when not in commit mode and
+	# running `svn status` in every package dir will be too expensive.
+	check_ebuild_notadded = not \
+		(vcs_settings.vcs == "svn" and repolevel < 3 and options.mode != "commit")
 
 	# NOTE: match-all caches are not shared due to potential
 	# differences between profiles in _get_implicit_iuse.
@@ -339,19 +227,6 @@ def repoman_main(argv):
 		include_arches = set()
 		include_arches.update(*[x.split() for x in options.include_arches])
 
-	# Disable the "ebuild.notadded" check when not in commit mode and
-	# running `svn status` in every package dir will be too expensive.
-
-	check_ebuild_notadded = not \
-		(vcs_settings.vcs == "svn" and repolevel < 3 and options.mode != "commit")
-
-	effective_scanlist = scanlist
-	if options.if_modified == "y":
-		effective_scanlist = sorted(vcs_files_to_cps(
-			chain(changed.changed, changed.new, changed.removed),
-			repolevel, reposplit, categories))
-
-	######################
 	# initialize our checks classes here before the big xpkg loop
 	manifester = Manifests(options, qatracker, repoman_settings)
 	is_ebuild = IsEbuild(repoman_settings, repo_settings, portdb, qatracker)
@@ -814,7 +689,7 @@ def repoman_main(argv):
 
 					if not have_dev_keywords:
 						have_dev_keywords = \
-							bool(dev_keywords.intersection(keywords))
+							bool(dev_keyword.intersection(keywords))
 
 					if prof.status == "dev":
 						suffix = suffix + "indev"
@@ -890,6 +765,123 @@ def repoman_main(argv):
 					"metadata.warning",
 					"%s/metadata.xml: unused local USE-description: '%s'"
 					% (xpkg, myflag))
+	return qatracker
+
+def repoman_main(argv, config_root=None, pkgdir=None):
+	# Repoman sets it's own ACCEPT_KEYWORDS and we don't want it to
+	# behave incrementally.
+	repoman_incrementals = tuple(
+		x for x in portage.const.INCREMENTALS if x != 'ACCEPT_KEYWORDS')
+	if config_root is None:
+		config_root = os.environ.get("PORTAGE_CONFIGROOT")
+	repoman_settings = portage.config(config_root=config_root, local_config=False)
+
+	if repoman_settings.get("NOCOLOR", "").lower() in ("yes", "true") or \
+		repoman_settings.get('TERM') == 'dumb' or \
+		not sys.stdout.isatty():
+		nocolor()
+
+	options, arguments = parse_args(
+		argv, qahelp, repoman_settings.get("REPOMAN_DEFAULT_OPTS", ""))
+
+	if options.version:
+		print("Portage", portage.VERSION)
+		sys.exit(0)
+
+	if options.experimental_inherit == 'y':
+		# This is experimental, so it's non-fatal.
+		qawarnings.add("inherit.missing")
+		checks_init(experimental_inherit=True)
+
+	# Set this to False when an extraordinary issue (generally
+	# something other than a QA issue) makes it impossible to
+	# commit (like if Manifest generation fails).
+	can_force = True
+
+	portdir, portdir_overlay, mydir = utilities.FindPortdir(repoman_settings, pkgdir)
+	if portdir is None:
+		sys.exit(1)
+
+	myreporoot = os.path.basename(portdir_overlay)
+	myreporoot += mydir[len(portdir_overlay):]
+	##################
+
+	vcs_settings = VCSSettings(options, repoman_settings)
+
+	##################
+
+	repo_settings = RepoSettings(
+		config_root, portdir, portdir_overlay,
+		repoman_settings, vcs_settings, options, qawarnings)
+
+	repoman_settings = repo_settings.repoman_settings
+
+	portdb = repo_settings.portdb
+	##################
+
+	if options.echangelog is None and repo_settings.repo_config.update_changelog:
+		options.echangelog = 'y'
+
+	if vcs_settings.vcs is None:
+		options.echangelog = 'n'
+
+	# The --echangelog option causes automatic ChangeLog generation,
+	# which invalidates changelog.ebuildadded and changelog.missing
+	# checks.
+	# Note: Some don't use ChangeLogs in distributed SCMs.
+	# It will be generated on server side from scm log,
+	# before package moves to the rsync server.
+	# This is needed because they try to avoid merge collisions.
+	# Gentoo's Council decided to always use the ChangeLog file.
+	# TODO: shouldn't this just be switched on the repo, iso the VCS?
+	is_echangelog_enabled = options.echangelog in ('y', 'force')
+	vcs_settings.vcs_is_cvs_or_svn = vcs_settings.vcs in ('cvs', 'svn')
+	check_changelog = not is_echangelog_enabled and vcs_settings.vcs_is_cvs_or_svn
+
+	if 'digest' in repoman_settings.features and options.digest != 'n':
+		options.digest = 'y'
+
+	logging.debug("vcs: %s" % (vcs_settings.vcs,))
+	logging.debug("repo config: %s" % (repo_settings.repo_config,))
+	logging.debug("options: %s" % (options,))
+
+	# It's confusing if these warnings are displayed without the user
+	# being told which profile they come from, so disable them.
+	env = os.environ.copy()
+	env['FEATURES'] = env.get('FEATURES', '') + ' -unknown-features-warn'
+
+	categories = []
+	for path in repo_settings.repo_config.eclass_db.porttrees:
+		categories.extend(portage.util.grabfile(
+			os.path.join(path, 'profiles', 'categories')))
+	repoman_settings.categories = frozenset(
+		portage.util.stack_lists([categories], incremental=1))
+	categories = repoman_settings.categories
+
+	portdb.settings = repoman_settings
+	# We really only need to cache the metadata that's necessary for visibility
+	# filtering. Anything else can be discarded to reduce memory consumption.
+	portdb._aux_cache_keys.clear()
+	portdb._aux_cache_keys.update(
+		["EAPI", "IUSE", "KEYWORDS", "repository", "SLOT"])
+
+	reposplit = myreporoot.split(os.path.sep)
+	repolevel = len(reposplit)
+
+	if options.mode == 'commit':
+		repochecks.commit_check(repolevel, reposplit)
+		repochecks.conflict_check(vcs_settings, options)
+
+	changed = Changes(options)
+	changed.scan(vcs_settings)
+
+	effective_scanlist = scanlist
+	if options.if_modified == "y":
+		effective_scanlist = sorted(vcs_files_to_cps(
+			chain(changed.changed, changed.new, changed.removed),
+			repolevel, reposplit, categories))
+
+	qatracker = repoman_scan(repoman_settings, repo_settings, vcs_settings, options, repolevel)
 
 	if options.if_modified == "y" and len(effective_scanlist) < 1:
 		logging.warn("--if-modified is enabled, but no modified packages were found!")

diff --git a/tbc/pym/repoman/utilities.py b/tbc/pym/repoman/utilities.py
index 5e43851..b1871db 100644
--- a/tbc/pym/repoman/utilities.py
+++ b/tbc/pym/repoman/utilities.py
@@ -240,7 +240,7 @@ def get_commit_message_with_stdin():
 	return commitmessage
 
 
-def FindPortdir(settings):
+def FindPortdir(settings, pkdir):
 	""" Try to figure out what repo we are in and whether we are in a regular
 	tree or an overlay.
 
@@ -276,6 +276,8 @@ def FindPortdir(settings):
 		# the current working directory (from the shell).
 		location = pwd
 
+	if not pkdir is None:
+		location = pkgdir
 	location = normalize_path(location)
 
 	path_ids = {}