[gentoo-commits] proj/tinderbox-cluster:master commit in: pym/tbc/

[Magnus Granberg]

commit:     85d70df173a81e8257172d39be1ae971be54422b
Author:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Tue Jan 26 20:30:38 2016 +0000
Commit:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Tue Jan 26 20:30:38 2016 +0000
URL:        https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=85d70df1

check all ebuilds in package instead of only check Manifest

 pym/tbc/build_log.py  |   5 +-
 pym/tbc/db_mapping.py |   1 -
 pym/tbc/package.py    | 151 +++++++++++++++++++++-----------------------------
 pym/tbc/sqlquerys.py  |   5 --
 pym/tbc/sync.py       |  20 +++++--
 5 files changed, 80 insertions(+), 102 deletions(-)

diff --git a/pym/tbc/build_log.py b/pym/tbc/build_log.py
index a16cdc6..ec4aaa7 100644
--- a/pym/tbc/build_log.py
+++ b/pym/tbc/build_log.py
@@ -29,7 +29,7 @@ from tbc.ConnectionManager import NewConnection
 from tbc.sqlquerys import get_config_id, get_ebuild_id_db, add_new_buildlog, \
 	get_package_info, get_build_job_id, get_use_id, get_config_info, get_hilight_info, get_error_info_list, \
 	add_e_info, get_fail_times, add_fail_times, update_fail_times, del_old_build_jobs, add_old_ebuild, \
-	update_buildjobs_status, update_mtime_sql, add_repoman_qa, get_config_id_fqdn, get_setup_info, \
+	update_buildjobs_status, add_repoman_qa, get_config_id_fqdn, get_setup_info, \
 	add_repoman_log
 from tbc.log import write_log
 
@@ -112,8 +112,7 @@ def get_build_dict_db(session, config_id, settings, tbc_settings_dict, pkg):
 	if status:
 		if ebuild_id_list is None:
 			log_msg = "%s:%s Don't have any ebuild_id!" % (pkg.cpv, repo,)
-			add_logs(session, log_msg, "info", config_id)
-			update_mtime_sql(session, build_dict['package_id'], 0)
+			add_logs(session, log_msg, "error", config_id)
 		else:
 			old_ebuild_id_list = []
 			for ebuild_id in ebuild_id_list:

diff --git a/pym/tbc/db_mapping.py b/pym/tbc/db_mapping.py
index 2636bb6..d9fcf25 100644
--- a/pym/tbc/db_mapping.py
+++ b/pym/tbc/db_mapping.py
@@ -84,7 +84,6 @@ class Packages(Base):
 	CategoryId = Column('category_id', Integer, ForeignKey('categories.category_id'))
 	Package = Column('package',String(150))
 	RepoId = Column('repo_id', Integer, ForeignKey('repos.repo_id'))
-	Mtime = Column('mtime', DateTime, default=0)
 	Active = Column('active', Boolean, default=False)
 	TimeStamp = Column('time_stamp', DateTime, nullable=False, default=datetime.datetime.utcnow)
 	__tablename__ = 'packages'

diff --git a/pym/tbc/package.py b/pym/tbc/package.py
index 65a0f40..68574cf 100644
--- a/pym/tbc/package.py
+++ b/pym/tbc/package.py
@@ -17,7 +17,7 @@ from tbc.build_log import check_repoman_full
 from tbc.log import write_log
 from tbc.sqlquerys import get_package_info, get_config_info, \
 	add_new_build_job, add_new_ebuild_sql, get_ebuild_id_list, add_old_ebuild, \
-	get_package_metadata_sql, update_package_metadata, update_mtime_sql, \
+	get_package_metadata_sql, update_package_metadata, \
 	get_package_info_from_package_id, get_config_all_info, add_new_package_sql, \
 	get_ebuild_checksums, get_ebuild_id_db, get_configmetadata_info, get_setup_info, \
 	get_ebuild_info_ebuild_id, get_ebuild_restrictions, add_old_package
@@ -210,7 +210,7 @@ class tbc_package(object):
 		package_metadataDict[package_id] = attDict
 		return package_metadataDict
 
-	def add_package(self, packageDict, package_metadataDict, package_id, new_ebuild_id_list, old_ebuild_id_list, manifest_mtime_tree):
+	def add_package(self, packageDict, package_metadataDict, package_id, new_ebuild_id_list, old_ebuild_id_list):
 		# Use packageDict to update the db
 		ebuild_id_list = add_new_ebuild_sql(self._session, packageDict)
 
@@ -227,9 +227,6 @@ class tbc_package(object):
 		# update package metadata
 		update_package_metadata(self._session, package_metadataDict)
 
-		# update the cp manifest checksum
-		update_mtime_sql(self._session, package_id, manifest_mtime_tree)
-
 		# Get the best cpv for the configs and add it to config_cpv_listDict
 		PackageInfo, CategoryInfo, RepoInfo = get_package_info_from_package_id(self._session, package_id)
 		cp = CategoryInfo.Category + '/' + PackageInfo.Package
@@ -244,18 +241,6 @@ class tbc_package(object):
 		# Add the ebuild to the build jobs table if needed
 		self.add_new_build_job_db(ebuild_id_list, packageDict, config_cpv_listDict)
 
-	def get_manifest_mtime_tree(self, pkgdir, cp, repo, mytree):
-		# Get the cp manifest file mtime.
-		try:
-			mtime = os.path.getmtime(pkgdir + "/Manifest")
-		except:
-			log_msg = "QA: Can't checksum the Manifest file. :%s:%s" % (cp, repo,)
-			write_log(self._session, log_msg, "warning", self._config_id, 'packages.get_manifest_mtime_tree')
-			log_msg = "C %s:%s ... Fail." % (cp, repo)
-			write_log(self._session, log_msg, "warning", self._config_id, 'packages.get_manifest_mtime_tree')
-			return False
-		return datetime.datetime.fromtimestamp(mtime).replace(microsecond=0)
-
 	def add_new_package_db(self, cp, repo):
 		# Add new categories package ebuild to tables package and ebuilds
 		# C = Checking
@@ -268,9 +253,6 @@ class tbc_package(object):
 		mytree = []
 		mytree.append(repodir)
 		pkgdir = repodir + "/" + cp # Get RepoDIR + cp
-		manifest_mtime_tree = self.get_manifest_mtime_tree(pkgdir, cp, repo, mytree)
-		if not manifest_mtime_tree:
-			return None
 		
 		package_id = add_new_package_sql(self._session, cp, repo)
 
@@ -332,83 +314,78 @@ class tbc_package(object):
 		
 		mytree = []
 		mytree.append(repodir)
-		manifest_mtime_tree = self.get_manifest_mtime_tree(pkgdir, cp, repo, mytree)
-		if not manifest_mtime_tree:
+
+		# Get the ebuild list for cp
+		old_ebuild_id_list = []
+		ebuild_list_tree = self._myportdb.cp_list(cp, use_cache=1, mytree=mytree)
+		if ebuild_list_tree == []:
+			log_msg = "QA: Can't get the ebuilds list. %s:%s" % (cp, repo,)
+			write_log(self._session, log_msg, "error", self._config_id, 'packages.update_package_db')
+			log_msg = "C %s:%s ... Fail." % (cp, repo)
+			write_log(self._session, log_msg, "warning", self._config_id, 'packages.update_package_db')
 			return None
 
-		# if we NOT have the same mtime in the db update the package
-		if PackageInfo.Mtime is None or manifest_mtime_tree > PackageInfo.Mtime:
+		package_metadataDict = self.get_package_metadataDict(pkgdir, repodir, package_id, cp)
+		packageDict ={}
+		new_ebuild_id_list = []
+		package_updated = False
+		for cpv in sorted(ebuild_list_tree):
 
-			# U = Update
-			log_msg = "U %s:%s" % (cp, repo)
-			write_log(self._session, log_msg, "info", self._config_id, 'packages.update_package_db')
+			# split out ebuild version
+			ebuild_version_tree = portage.versions.cpv_getversion(cpv)
 
-			# Get the ebuild list for cp
-			old_ebuild_id_list = []
-			ebuild_list_tree = self._myportdb.cp_list(cp, use_cache=1, mytree=mytree)
-			if ebuild_list_tree == []:
-				log_msg = "QA: Can't get the ebuilds list. %s:%s" % (cp, repo,)
-				write_log(self._session, log_msg, "error", self._config_id, 'packages.update_package_db')
-				log_msg = "C %s:%s ... Fail." % (cp, repo)
-				write_log(self._session, log_msg, "warning", self._config_id, 'packages.update_package_db')
-				return None
+			# Get packageDict for cpv
+			packageDict[cpv] = self.get_packageDict(pkgdir, cpv, repo)
+
+			# take package descriptions from the ebuilds
+			if package_metadataDict[package_id]['metadata_xml_descriptions'] != packageDict[cpv]['ebuild_version_descriptions_tree']:
+				package_metadataDict[package_id]['metadata_xml_descriptions'] = packageDict[cpv]['ebuild_version_descriptions_tree']
+
+			# Get the checksum of the ebuild in tree and db
+			ebuild_version_checksum_tree = packageDict[cpv]['checksum']
+			checksums_db, fail = get_ebuild_checksums(self._session, package_id, ebuild_version_tree)
+
+			# check if we have dupes of the checksum from db
+			if checksums_db is None:
+				ebuild_version_manifest_checksum_db = None
+			elif fail:
+				dupe_ebuild_id_list = []
+				for checksum in checksums_db:
+					ebuilds_id , status = get_ebuild_id_db(self._session, checksum, package_id)
+					for ebuild_id in ebuilds_id:
+						log_msg = "U %s:%s:%s Dups of checksums" % (cpv, repo, ebuild_id,)
+						write_log(self._session, log_msg, "warning", self._config_id, 'packages.update_package_db')
+						dupe_ebuild_id_list.append(ebuild_id)
+				add_old_ebuild(self._session, dupe_ebuild_id_list)
+				ebuild_version_manifest_checksum_db = None
+			else:
+				ebuild_version_manifest_checksum_db = checksums_db
+
+			# Check if the checksum have change
+			if ebuild_version_manifest_checksum_db is None:
+				# N = New ebuild
+				log_msg = "N %s:%s" % (cpv, repo,)
+				write_log(self._session, log_msg, "info", self._config_id, 'packages.update_package_db')
+				packageDict[cpv]['new'] = True
+				package_updated = True
+			elif  ebuild_version_checksum_tree != ebuild_version_manifest_checksum_db:
+				# U = Updated ebuild
+				log_msg = "U %s:%s" % (cpv, repo,)
+				write_log(self._session, log_msg, "info", self._config_id, 'packages.update_package_db')
+				package_updated = True
+			else:
+				# Remove cpv from packageDict and add ebuild to new ebuils list
+				del packageDict[cpv]
+				ebuild_id , status = get_ebuild_id_db(self._session, ebuild_version_checksum_tree, package_id)
+				new_ebuild_id_list.append(ebuild_id)
+		self.add_package(packageDict, package_metadataDict, package_id, new_ebuild_id_list, old_ebuild_id_list, manifest_mtime_tree)
 
+		if package_updated:
 			# Check cp with repoman full
 			status = check_repoman_full(self._session, pkgdir, package_id, self._config_id)
 			if status:
 				log_msg = "Repoman %s::%s ... Fail." % (cp, repo)
 				write_log(self._session, log_msg, "warning", self._config_id, 'packages.update_package_db')
-			package_metadataDict = self.get_package_metadataDict(pkgdir, repodir, package_id, cp)
-			packageDict ={}
-			new_ebuild_id_list = []
-			for cpv in sorted(ebuild_list_tree):
-
-				# split out ebuild version
-				ebuild_version_tree = portage.versions.cpv_getversion(cpv)
-
-				# Get packageDict for cpv
-				packageDict[cpv] = self.get_packageDict(pkgdir, cpv, repo)
-
-				# take package descriptions from the ebuilds
-				if package_metadataDict[package_id]['metadata_xml_descriptions'] != packageDict[cpv]['ebuild_version_descriptions_tree']:
-					package_metadataDict[package_id]['metadata_xml_descriptions'] = packageDict[cpv]['ebuild_version_descriptions_tree']
-
-				# Get the checksum of the ebuild in tree and db
-				ebuild_version_checksum_tree = packageDict[cpv]['checksum']
-				checksums_db, fail = get_ebuild_checksums(self._session, package_id, ebuild_version_tree)
-
-				# check if we have dupes of the checksum from db
-				if checksums_db is None:
-					ebuild_version_manifest_checksum_db = None
-				elif fail:
-					dupe_ebuild_id_list = []
-					for checksum in checksums_db:
-						ebuilds_id , status = get_ebuild_id_db(self._session, checksum, package_id)
-						for ebuild_id in ebuilds_id:
-							log_msg = "U %s:%s:%s Dups of checksums" % (cpv, repo, ebuild_id,)
-							write_log(self._session, log_msg, "warning", self._config_id, 'packages.update_package_db')
-							dupe_ebuild_id_list.append(ebuild_id)
-					add_old_ebuild(self._session, dupe_ebuild_id_list)
-					ebuild_version_manifest_checksum_db = None
-				else:
-					ebuild_version_manifest_checksum_db = checksums_db
-
-				# Check if the checksum have change
-				if ebuild_version_manifest_checksum_db is None:
-					# N = New ebuild
-					log_msg = "N %s:%s" % (cpv, repo,)
-					write_log(self._session, log_msg, "info", self._config_id, 'packages.update_package_db')
-					packageDict[cpv]['new'] = True
-				elif  ebuild_version_checksum_tree != ebuild_version_manifest_checksum_db:
-					# U = Updated ebuild
-					log_msg = "U %s:%s" % (cpv, repo,)
-					write_log(self._session, log_msg, "info", self._config_id, 'packages.update_package_db')
-				else:
-					# Remove cpv from packageDict and add ebuild to new ebuils list
-					del packageDict[cpv]
-					ebuild_id , status = get_ebuild_id_db(self._session, ebuild_version_checksum_tree, package_id)
-					new_ebuild_id_list.append(ebuild_id)
-			self.add_package(packageDict, package_metadataDict, package_id, new_ebuild_id_list, old_ebuild_id_list, manifest_mtime_tree)
 
 		log_msg = "C %s:%s ... Done." % (cp, repo)
 		write_log(self._session, log_msg, "info", self._config_id, 'packages.update_package_db')

diff --git a/pym/tbc/sqlquerys.py b/pym/tbc/sqlquerys.py
index e43b8fc..4161461 100644
--- a/pym/tbc/sqlquerys.py
+++ b/pym/tbc/sqlquerys.py
@@ -515,11 +515,6 @@ def update_package_metadata(session, package_metadataDict):
 				EmailInfo = update_email_info(session, email)
 				PackagesEmailInfo = update_package_email_info(session, EmailInfo.EmailId, k)
 
-def update_mtime_sql(session, package_id, manifest_mtime_tree):
-	PackagesInfo = session.query(Packages).filter_by(PackageId = package_id).one()
-	PackagesInfo.Mtime = manifest_mtime_tree
-	session.commit()
-
 def get_package_info_from_package_id(session, package_id):
 	PackageInfo = session.query(Packages).filter_by(PackageId = package_id).one()
 	CategoryInfo = session.query(Categories).filter_by(CategoryId = PackageInfo.CategoryId).one()

diff --git a/pym/tbc/sync.py b/pym/tbc/sync.py
index 1732f1f..2d22e74 100644
--- a/pym/tbc/sync.py
+++ b/pym/tbc/sync.py
@@ -70,9 +70,10 @@ def git_sync_main(session):
 	except:
 		pass
 
-	# check git diffs witch Manifests get updated and pass that to a dict
+	# check git diffs witch get updated and pass that to a dict
 	# fetch and merge the repo
 	repo_cp_dict = {}
+	search_list = [ '^metadata', '^eclass', '^licenses', '^profiles', '^scripts',]
 	for repo_dir in git_repos_list(myportdb):
 		reponame = myportdb.getRepositoryName(repo_dir)
 		repo = git.Repo(repo_dir)
@@ -82,14 +83,21 @@ def git_sync_main(session):
 		if not repouptodate:
 			cp_list = []
 			attr = {}
-			# We check for Manifest changes and add the package to a list
+			# We check for dir changes and add the package to a list
 			repo_diff = repo.git.diff('origin', '--name-only')
-			write_log(session, 'Git diff: %s' % (repo_diff,), "debug", config_id, 'sync.git_sync_main')
+			write_log(session, 'Git dir diff:\n%s' % (repo_diff,), "debug", config_id, 'sync.git_sync_main')
 			for diff_line in repo_diff.splitlines():
-				if re.search("Manifest$", diff_line):
-						cp = re.sub('/Manifest', '', diff_line)
-						cp_list.append(cp)
+                                find_search = True
+                                for search_line in search_list:
+                                        if re.search(search_line, diff_line):
+                                                find_search = False
+                                if find_search:
+                                        splited_diff_line = re.split('/', diff_line)
+                                        cp = splited_diff_line[0] + '/' + splited_diff_line[1]
+                                        if not cp in cp_list:
+                                                cp_list.append(cp)
 			attr['cp_list'] = cp_list
+			write_log(session, 'Git CP Diff: %s' % (cp_list,), "debug", config_id, 'sync.git_sync_main')
 			repo_cp_dict[reponame] = attr
 			git_merge(repo, info_list[0])
 		else: