[gentoo-commits] dev/zorry:master commit in: gobs/bin/, gobs/pym/

[Magnus Granberg]

commit:     7a6764b06e02586172b5bb34c3989d490289ddda
Author:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Sat Apr 28 16:00:52 2012 +0000
Commit:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Sat Apr 28 16:00:52 2012 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=dev/zorry.git;a=commit;h=7a6764b0

add logging and multiprocessing to gobs_setup_profile

---
 gobs/bin/gobs_setup_profile    |    8 +++
 gobs/pym/init_setup_profile.py |  122 +++++++++++++++++++++++----------------
 2 files changed, 80 insertions(+), 50 deletions(-)

diff --git a/gobs/bin/gobs_setup_profile b/gobs/bin/gobs_setup_profile
index 902ffe8..9a1dd04 100755
--- a/gobs/bin/gobs_setup_profile
+++ b/gobs/bin/gobs_setup_profile
@@ -3,9 +3,17 @@
 # Distributed under the terms of the GNU General Public License v2
 
 from gobs.init_setup_profile import setup_profile_main
+from gobs.readconf import get_conf_settings
+reader = get_conf_settings()
+gobs_settings_dict=reader.read_gobs_settings_all()
+
+import logging
 
 def main():
 	# Main
+	# Logging
+	logging.basicConfig(filename=gobs_settings_dict['gobs_logfile'], \
+	format='%(levelname)s: %(asctime)s %(message)s', level=logging.INFO)
 	setup_profile_main(args=None)
 	
 if __name__ == "__main__":

diff --git a/gobs/pym/init_setup_profile.py b/gobs/pym/init_setup_profile.py
index e647e1f..a18865e 100644
--- a/gobs/pym/init_setup_profile.py
+++ b/gobs/pym/init_setup_profile.py
@@ -7,6 +7,7 @@
 
 import sys
 import os
+import logging
 
 # Get the options from the config file set in gobs.readconf
 from gobs.readconf import get_conf_settings
@@ -19,9 +20,42 @@ CM=connectionManager(gobs_settings_dict)
 if CM.getName()=='pgsql':
   from gobs.pgsql import *
 
-from gobs.check_setup import check_make_conf, git_pull
+from gobs.check_setup import check_make_conf
+from gobs.sync import git_pull
 from gobs.package import gobs_package
 import portage
+import multiprocessing
+def add_cpv_query_pool(init_package, config_id_list, package_line)
+	conn=CM.getConnection()
+	# FIXME: remove the check for gobs when in tree
+	if package_line != "dev-python/gobs":
+		build_dict = {}
+		packageDict = {}
+		ebuild_id_list = []
+		# split the cp to categories and package
+		element = package_line.split('/')
+		categories = element[0]
+		package = element[1]
+		logging.info"C %s/%s", categories, package			# C = Checking
+		pkgdir = mysettings['PORTDIR'] + "/" + categories + "/" + package
+		config_cpv_listDict = init_package.config_match_ebuild(categories, package, config_id_list)
+		if config_cpv_listDict != {}:
+			cpv = categories + "/" + package + "-" + config_cpv_listDict[config_id]['ebuild_version']
+			attDict = {}
+			attDict['categories'] = categories
+			attDict['package'] = package
+			attDict['ebuild_version_tree'] = config_cpv_listDict[config_id]['ebuild_version']
+			packageDict[cpv] = attDict
+			build_dict['checksum'] = portage.checksum.sha256hash(pkgdir + "/" + package + "-" + config_cpv_listDict[config_id]['ebuild_version'] + ".ebuild")[0]
+			build_dict['package_id'] = have_package_db(conn, categories, package)[0]
+			build_dict['ebuild_version'] = config_cpv_listDict[config_id]['ebuild_version']
+			ebuild_id = get_ebuild_id_db_checksum(conn, build_dict)
+			if ebuild_id is not None:
+				ebuild_id_list.append(ebuild_id)
+				init_package.add_new_ebuild_buildquery_db(ebuild_id_list, packageDict, config_cpv_listDict)
+		logging.info"C %s/%s ... Done.", categories, package
+	CM.putConnection(conn)
+	return
 
 def setup_profile_main(args=None):
 	"""
@@ -31,56 +65,44 @@ def setup_profile_main(args=None):
 	conn=CM.getConnection()
 	if args is None:
 		args = sys.argv[1:]
-		if args[0] == "-add":
-			git_pull()
-			check_make_conf()
-			print "Check configs done"
-			# Get default config from the configs table  and default_config=1
-			config_id = args[1]
-			default_config_root = "/var/lib/gobs/" + gobs_settings_dict['gobs_gitreponame'] + "/" + config_id + "/"
-			# Set config_root (PORTAGE_CONFIGROOT)  to default_config_root
-			mysettings = portage.config(config_root = default_config_root)
-			myportdb = portage.portdbapi(mysettings=mysettings)
-			init_package = gobs_package(mysettings, myportdb)
-			# get the cp list
-			package_list_tree = package_list_tree = myportdb.cp_all()
-			print "Setting default config to:", config_id
-			config_id_list = []
-			config_id_list.append(config_id)
-			for package_line in sorted(package_list_tree):
-				# FIXME: remove the check for gobs when in tree
-				if package_line != "dev-python/gobs":
-					build_dict = {}
-					packageDict = {}
-					ebuild_id_list = []
-					# split the cp to categories and package
-					element = package_line.split('/')
-					categories = element[0]
-					package = element[1]
-					print "C", categories + "/" + package			# C = Checking
-					pkgdir = mysettings['PORTDIR'] + "/" + categories + "/" + package
-					config_cpv_listDict = init_package.config_match_ebuild(categories, package, config_id_list)
-					if config_cpv_listDict != {}:
-						cpv = categories + "/" + package + "-" + config_cpv_listDict[config_id]['ebuild_version']
-                                                attDict = {}
-                                                attDict['categories'] = categories
-                                                attDict['package'] = package
-                                                attDict['ebuild_version_tree'] = config_cpv_listDict[config_id]['ebuild_version']
-                                                packageDict[cpv] = attDict
-						build_dict['checksum'] = portage.checksum.sha256hash(pkgdir + "/" + package + "-" + config_cpv_listDict[config_id]['ebuild_version'] + ".ebuild")[0]
-						build_dict['package_id'] = have_package_db(conn, categories, package)[0]
-						build_dict['ebuild_version'] = config_cpv_listDict[config_id]['ebuild_version']
-						ebuild_id = get_ebuild_id_db_checksum(conn, build_dict)
-						if ebuild_id is not None:
-							ebuild_id_list.append(ebuild_id)
-							init_package.add_new_ebuild_buildquery_db(ebuild_id_list, packageDict, config_cpv_listDict)
+	if args[0] == "-add":
+		config_id = args[1]
+		logging.info("Adding build querys for %s:", config_id)
+		git_pull()
+		check_make_conf()
+		logging.info("Check configs done")
+		# Get default config from the configs table  and default_config=1
+		default_config_root = "/var/lib/gobs/" + gobs_settings_dict['gobs_gitreponame'] + "/" + config_id + "/"
+		# Set config_root (PORTAGE_CONFIGROOT)  to default_config_root
+		mysettings = portage.config(config_root = default_config_root)
+		myportdb = portage.portdbapi(mysettings=mysettings)
+		init_package = gobs_package(mysettings, myportdb)
+		# get the cp list
+		package_list_tree = package_list_tree = myportdb.cp_all()
+		logging.info("Setting default config to:", config_id)
+		config_id_list = []
+		config_id_list.append(config_id)
+		# Use all exept 2 cores when multiprocessing
+		pool_cores= multiprocessing.cpu_count()
+		if pool_cores >= 3:
+			use_pool_cores = pool_cores - 2
+		else:
+			use_pool_cores = 1
+		pool = multiprocessing.Pool(processes=use_pool_cores)
+		for package_line in sorted(package_list_tree):
+			pool.apply_async(add_cpv_query_pool, (init_package, config_id_list, package_line,))
+		pool.close()
+		pool.join()
+		logging.info("Adding build querys for: %s ... Done.", config_id)
 
-		if args[0] == "-del":
-			config_id = args[1]
-			querue_id_list = get_queue_id_list_config(conn, config_id)
-			if querue_id_list is not None:
-				for querue_id in querue_id_list:
-					del_old_queue(conn, querue_id)
+	if args[0] == "-del":
+		config_id = args[1]
+		logging.info("Removeing build querys for %s:", config_id)
+		querue_id_list = get_queue_id_list_config(conn, config_id)
+		if querue_id_list is not None:
+			for querue_id in querue_id_list:
+				del_old_queue(conn, querue_id)
+		logging.info("Removeing build querys for: %s ... Done.", config_id)
 	CM.putConnection(conn)
 
 		
\ No newline at end of file