[gentoo-commits] proj/tinderbox-cluster:master commit in: tbc/pym/, tbc/pym/repoman/

[Magnus Granberg]

commit:     2410346473e1d00de202eac126d995a148b9609f
Author:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Sat Aug  1 20:49:08 2015 +0000
Commit:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Sat Aug  1 20:49:08 2015 +0000
URL:        https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=24103464

add repoman full to host and guest

 tbc/pym/build_log.py    | 41 ++++++++++++++++++++++++++++++-----------
 tbc/pym/db_mapping.py   |  8 ++++++++
 tbc/pym/package.py      | 32 ++++++++++++++------------------
 tbc/pym/qachecks.py     | 13 ++++++++++++-
 tbc/pym/repoman/main.py | 10 ++++++----
 tbc/pym/sqlquerys.py    | 15 ++++++++++++++-
 6 files changed, 84 insertions(+), 35 deletions(-)

diff --git a/tbc/pym/build_log.py b/tbc/pym/build_log.py
index 9c8bbfa..71c14b5 100644
--- a/tbc/pym/build_log.py
+++ b/tbc/pym/build_log.py
@@ -20,18 +20,38 @@ portage.proxy.lazyimport.lazyimport(globals(),
 	'tbc.actions:action_info,load_emerge_config',
 )
 
-from tbc.qachecks import check_repoman
+from tbc.qachecks import check_repoman, repoman_full
 from tbc.text import get_log_text_dict
-from tbc.package import tbc_package
 from tbc.readconf import read_config_settings
 from tbc.flags import tbc_use_flags
 from tbc.ConnectionManager import NewConnection
 from tbc.sqlquerys import add_logs, get_config_id, get_ebuild_id_db, add_new_buildlog, \
 	get_package_info, get_build_job_id, get_use_id, get_config_info, get_hilight_info, get_error_info_list, \
 	add_e_info, get_fail_times, add_fail_times, update_fail_times, del_old_build_jobs, add_old_ebuild, \
-	update_buildjobs_status, update_manifest_sql, add_repoman_qa, get_config_id_fqdn, get_setup_info
+	update_buildjobs_status, update_manifest_sql, add_repoman_qa, get_config_id_fqdn, get_setup_info, \
+	add_repoman_log
 from sqlalchemy.orm import sessionmaker
 
+def repoman_check_full(session, pkgdir, package_id, config_id):
+	# Check cp with repoman repoman full
+	status = repoman_full(session, pkgdir, config_id)
+	repoman_hash = hashlib.sha256()
+	repoman_log = ""
+	if status:
+		for k, v in status.items():
+			repoman_line = k + '/n'
+			repoman_hash.update(repoman_line.encode('utf-8'))
+			repoman_log = repoman_log + repoman_line
+			for line in v:
+				repoman_line = line + '/n'
+				repoman_hash.update(repoman_line.encode('utf-8'))
+				repoman_log = repoman_log + repoman_line
+		add_repoman_log(session, package_id, repoman_log, repoman_hash.hexdigest())
+		return repoman_log
+	else:
+		return status
+
+
 def get_build_dict_db(session, config_id, settings, tbc_settings_dict, pkg):
 	myportdb = portage.portdbapi(mysettings=settings)
 	cpvr_list = catpkgsplit(pkg.cpv, silent=1)
@@ -48,6 +68,7 @@ def get_build_dict_db(session, config_id, settings, tbc_settings_dict, pkg):
 	build_dict['cpv'] = pkg.cpv
 	build_dict['categories'] = categories
 	build_dict['package'] = package
+	build_dict['repo'] = repo
 	build_dict['config_id'] = config_id
 	init_useflags = tbc_use_flags(settings, myportdb, pkg.cpv)
 	iuse_flags_list, final_use_list = init_useflags.get_flags_pkg(pkg, settings)
@@ -77,12 +98,6 @@ def get_build_dict_db(session, config_id, settings, tbc_settings_dict, pkg):
 			log_msg = "%s:%s Don't have any ebuild_id!" % (pkg.cpv, repo,)
 			add_logs(session, log_msg, "info", config_id)
 			update_manifest_sql(session, build_dict['package_id'], "0")
-			init_package = tbc_package(session, settings, myportdb, config_id)
-			init_package.update_package_db(build_dict['package_id'])
-			ebuild_id_list, status = get_ebuild_id_db(session, build_dict['checksum'], build_dict['package_id'])
-			if status and ebuild_id_list is None:
-				log_msg = "%s:%s Don't have any ebuild_id!" % (pkg.cpv, repo,)
-				add_logs(session, log_msg, "error", config_id)
 		else:
 			old_ebuild_id_list = []
 			for ebuild_id in ebuild_id_list:
@@ -204,8 +219,12 @@ def get_buildlog_info(session, settings, pkg, build_dict):
 					qa_error_list.append(logfile_text_dict[i])
 				i = i +1
 
-	# Run repoman check_repoman()
-	repoman_error_list = check_repoman(settings, myportdb, build_dict['cpv'], pkg.repo)
+	# Run repoman full
+	element = portage.versions.cpv_getkey(build_dict['cpv']).split('/')
+	categories = element[0]
+	package = element[1]
+	pkgdir = myportdb.getRepositoryPath(build_dict['repo']) + "/" + categories + "/" + package
+	repoman_error_list = repoman_check_full(session, pkgdir, build_dict['package_id'], config_id)
 	build_log_dict = {}
 	build_log_dict['fail'] = False
 	if repoman_error_list:

diff --git a/tbc/pym/db_mapping.py b/tbc/pym/db_mapping.py
index a296c2e..dbae5a8 100644
--- a/tbc/pym/db_mapping.py
+++ b/tbc/pym/db_mapping.py
@@ -230,6 +230,14 @@ class BuildLogsRepomanQa(Base):
 	SummeryText = Column('summery_text', Text)
 	__tablename__ = 'build_logs_repoman_qa'
 
+class PackagesRepoman(Base):
+	Id = Column('id', Integer, primary_key=True)
+	PackageId = Column('package_id', Integer, ForeignKey('packages.package_id'))
+	RepomanText = Column('repoman_text', Text)
+	RepomanHash = Column('repoman_hash', String(100))
+	TimeStamp = Column('time_stamp', DateTime, nullable=False, default=datetime.datetime.utcnow)
+	__tablename__ = 'packages_repoman'
+
 class ErrorsInfo(Base):
 	ErrorId = Column('error_id', Integer, primary_key=True)
 	ErrorName = Column('error_name', String)

diff --git a/tbc/pym/package.py b/tbc/pym/package.py
index 0a19f15..657c90d 100644
--- a/tbc/pym/package.py
+++ b/tbc/pym/package.py
@@ -3,12 +3,14 @@
 
 from __future__ import print_function
 import re
+import hashlib
 import portage
 from portage.xml.metadata import MetaDataXML
 from tbc.flags import tbc_use_flags
 from tbc.text import get_ebuild_cvs_revision, get_log_text_dict
 from tbc.flags import tbc_use_flags
-from tbc.qachecks import digestcheck, check_repoman, repoman_full
+from tbc.qachecks import digestcheck, check_repoman
+from tbc.build_log import check_repoman_full
 from tbc.sqlquerys import add_logs, get_package_info, get_config_info, \
 	add_new_build_job, add_new_ebuild_sql, get_ebuild_id_list, add_old_ebuild, \
 	get_package_metadata_sql, update_package_metadata, update_manifest_sql, \
@@ -283,11 +285,13 @@ class tbc_package(object):
 		if manifest_checksum_tree is None:
 			return
 
-		# Check cp with repoman repoman full
-		repoman_full(self._session, pkgdir, self._config_id)
-
 		package_id = add_new_package_sql(self._session, cp, repo)
 		
+		# Check cp with repoman full
+		status = check_repoman_full(self._session, pkgdir, package_id, self._config_id)
+		if status:
+					log_msg = "Repoman %s:%s ... Fail." % (cpv, repo)
+					add_logs(self._session, log_msg, "error", self._config_id)
 		package_metadataDict = self.get_package_metadataDict(pkgdir, package_id)
 		# Get the ebuild list for cp
 		ebuild_list_tree = self._myportdb.cp_list(cp, use_cache=1, mytree=mytree)
@@ -308,11 +312,7 @@ class tbc_package(object):
 			# take package descriptions from the ebuilds
 			if package_metadataDict[package_id]['metadata_xml_descriptions'] != packageDict[cpv]['ebuild_version_descriptions_tree']:
 				package_metadataDict[package_id]['metadata_xml_descriptions'] = packageDict[cpv]['ebuild_version_descriptions_tree']
-			if packageDict[cpv]['checksum'] != "0":
-				repoman_fail = check_repoman(self._mysettings, self._myportdb, cpv, repo)
-				if repoman_fail:
-					log_msg = "Repoman %s:%s ... Fail." % (cpv, repo)
-					add_logs(self._session, log_msg, "error", self._config_id)
+
 		self.add_package(packageDict, package_metadataDict, package_id, new_ebuild_id_list, old_ebuild_id_list, manifest_checksum_tree)
 		log_msg = "C %s:%s ... Done." % (cp, repo)
 		add_logs(self._session, log_msg, "info", self._config_id)
@@ -340,8 +340,11 @@ class tbc_package(object):
 			log_msg = "U %s:%s" % (cp, repo)
 			add_logs(self._session, log_msg, "info", self._config_id)
 
-			# Check cp with repoman repoman full
-			repoman_full(self._session, pkgdir, self._config_id)
+			# Check cp with repoman full
+			status = check_repoman_full(self._session, pkgdir, package_id, self._config_id)
+			if status:
+					log_msg = "Repoman %s:%s ... Fail." % (cpv, repo)
+					add_logs(self._session, log_msg, "error", self._config_id)
 
 			# Get the ebuild list for cp
 			old_ebuild_id_list = []
@@ -399,13 +402,6 @@ class tbc_package(object):
 				else:
 					ebuild_version_manifest_checksum_db = checksums_db
 
-				# Check with repoman
-				if (ebuild_version_manifest_checksum_db is None or ebuild_version_checksum_tree != ebuild_version_manifest_checksum_db) and ebuild_version_checksum_tree != "0":
-					repoman_fail = check_repoman(self._mysettings, self._myportdb, cpv, repo)
-					if repoman_fail:
-						log_msg = "Repoman %s:%s ... Fail." % (cpv, repo)
-						add_logs(self._session, log_msg, "error", self._config_id)
-
 				# Check if the checksum have change
 				if ebuild_version_manifest_checksum_db is None:
 					# N = New ebuild

diff --git a/tbc/pym/qachecks.py b/tbc/pym/qachecks.py
index 2e25d86..766b572 100644
--- a/tbc/pym/qachecks.py
+++ b/tbc/pym/qachecks.py
@@ -186,5 +186,16 @@ def repoman_full(session, pkgdir, config_id):
 	SetupInfo = get_setup_info(session, config_id)
 	config_root = ConfigsMetaData.RepoPath + '/' + ConfigInfo.Hostname + "/" + SetupInfo.Setup
 	argscmd = []
+	argscmd.append('--xmlparse')
 	argscmd.append('full')
-	repoman_main(argscmd, config_root=config_root, pkgdir=pkgdir)
+	qatracker, qawarnings = repoman_main(argscmd, config_root=config_root, pkgdir=pkgdir)
+	adict = {}
+	for key in qatracker.fails.items():
+		alist = []
+		for foo in key[1]:
+			alist.append(foo)
+			adict[key[0]] = alist
+	if adict == {}:
+		return False
+	return adict
+

diff --git a/tbc/pym/repoman/main.py b/tbc/pym/repoman/main.py
index 2353455..a30704a 100644
--- a/tbc/pym/repoman/main.py
+++ b/tbc/pym/repoman/main.py
@@ -789,7 +789,9 @@ def repoman_scan(repoman_settings, repo_settings, vcs_settings, portdb, options,
 					"metadata.warning",
 					"%s/metadata.xml: unused local USE-description: '%s'"
 					% (xpkg, myflag))
-
+	
+	suggest_ignore_masked = False
+	suggest_include_dev = False
 	if have_pmasked and not (options.without_mask or options.ignore_masked):
 		suggest_ignore_masked = True
 	if have_dev_keywords and not options.include_dev:
@@ -973,7 +975,7 @@ def repoman_main(argv, config_root=None, pkgdir=None):
 			utilities.repoman_sez(
 				"\"Make your QA payment on time"
 				" and you'll never see the likes of me.\"\n")
-			sys.exit(1)
+			return qatracker, qawarnings
 	else:
 		if dofail and can_force and options.force and not options.pretend:
 			utilities.repoman_sez(
@@ -988,7 +990,7 @@ def repoman_main(argv, config_root=None, pkgdir=None):
 			utilities.repoman_sez(
 				"\"Make your QA payment on time"
 				" and you'll never see the likes of me.\"\n")
-			sys.exit(1)
+			return qatracker, qawarnings
 
 		if options.pretend:
 			utilities.repoman_sez(
@@ -1627,4 +1629,4 @@ def repoman_main(argv, config_root=None, pkgdir=None):
 				" that he forgot to commit anything")
 		utilities.repoman_sez(
 			"\"If everyone were like you, I'd be out of business!\"\n")
-	sys.exit(0)
+	return qatracker, qawarnings

diff --git a/tbc/pym/sqlquerys.py b/tbc/pym/sqlquerys.py
index 8fdf901..4d2a5e5 100644
--- a/tbc/pym/sqlquerys.py
+++ b/tbc/pym/sqlquerys.py
@@ -8,7 +8,8 @@ from tbc.db_mapping import Configs, Logs, ConfigsMetaData, Jobs, BuildJobs, Pack
 	Uses, ConfigsEmergeOptions, EmergeOptions, HiLight, BuildLogs, BuildLogsConfig, BuildJobsUse, BuildJobsRedo, \
 	HiLightCss, BuildLogsHiLight, BuildLogsEmergeOptions, BuildLogsErrors, ErrorsInfo, EmergeInfo, BuildLogsUse, \
 	BuildJobsEmergeOptions, EbuildsMetadata, EbuildsIUse, Restrictions, EbuildsRestrictions, EbuildsKeywords, \
-	Keywords, PackagesMetadata, Emails, PackagesEmails, Setups, BuildLogsRepomanQa, CategoriesMetadata
+	Keywords, PackagesMetadata, Emails, PackagesEmails, Setups, BuildLogsRepomanQa, CategoriesMetadata, \
+	PackagesRepoman
 from sqlalchemy.orm.exc import NoResultFound, MultipleResultsFound
 from sqlalchemy import and_, or_
 
@@ -566,3 +567,15 @@ def get_ebuild_restrictions(session, ebuild_id):
 			restrictions.append(session.query(Restrictions).filter_by(RestrictionId = EbuildsRestrictionsInfo.RestrictionId).one())
 		return restrictions
 	return restrictions.append(session.query(Restrictions).filter_by(RestrictionId = EbuildsRestrictionsInfo.RestrictionId).one())
+
+def add_repoman_log(session, package_id, repoman_log, repoman_hash):
+	try:
+		PackagesRepomanInfo = session.query(PackagesRepoman).filter_by(PackageId = package_id).one()
+	except NoResultFound as e:
+		session.add(PackagesRepoman(PackageId = package_id, RepomanText = repoman_log, RepomanHash = repoman_hash))
+		session.commit()
+	else:
+		if PackagesRepomanInfo.RepomanHash != repoman_hash:
+			PackagesRepomanInfo.RepomanHash = repoman_hash
+			PackagesRepomanInfo.RepomanText = repoman_log
+			session.commit()