[gentoo-commits] proj/catalyst:pending commit in: catalyst/, catalyst/targets/

[Brian Dolbec]

commit:     ee9831262599e0ee02d8374e677c94462f87b006
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Fri Jan  3 18:42:26 2014 +0000
Commit:     Brian Dolbec <brian.dolbec <AT> gmail <DOT> com>
CommitDate: Sat Mar 22 18:01:29 2014 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/catalyst.git;a=commit;h=ee983126

Initial separation and creation of a hash_utils.py module

---
 catalyst/hash_utils.py                   | 137 +++++++++++++++++++++++++++++++
 catalyst/main.py                         |  30 ++++---
 catalyst/support.py                      |  62 --------------
 catalyst/targets/generic_stage_target.py |  36 ++++----
 catalyst/targets/livecd_stage2_target.py |   4 +-
 catalyst/targets/stage2_target.py        |   7 +-
 6 files changed, 184 insertions(+), 92 deletions(-)

diff --git a/catalyst/hash_utils.py b/catalyst/hash_utils.py
new file mode 100644
index 0000000..b575ace
--- /dev/null
+++ b/catalyst/hash_utils.py
@@ -0,0 +1,137 @@
+
+import os
+from collections import namedtuple
+from subprocess import Popen, PIPE
+
+from support import CatalystError
+
+
+# Use HashMap.fields for the value legend
+# fields = ["func", "cmd", "args", "id"]
+HASH_DEFINITIONS = {
+	"adler32"  :["calc_hash2", "shash", ["-a", "ADLER32"], "ADLER32"],
+	"crc32"    :["calc_hash2", "shash", ["-a", "CRC32"], "CRC32"],
+	"crc32b"   :["calc_hash2", "shash", ["-a", "CRC32B"], "CRC32B"],
+	"gost"     :["calc_hash2", "shash", ["-a", "GOST"], "GOST"],
+	"haval128" :["calc_hash2", "shash", ["-a", "HAVAL128"], "HAVAL128"],
+	"haval160" :["calc_hash2", "shash", ["-a", "HAVAL160"], "HAVAL160"],
+	"haval192" :["calc_hash2", "shash", ["-a", "HAVAL192"], "HAVAL192"],
+	"haval224" :["calc_hash2", "shash", ["-a", "HAVAL224"], "HAVAL224"],
+	"haval256" :["calc_hash2", "shash", ["-a", "HAVAL256"], "HAVAL256"],
+	"md2"      :["calc_hash2", "shash", ["-a", "MD2"], "MD2"],
+	"md4"      :["calc_hash2", "shash", ["-a", "MD4"], "MD4"],
+	"md5"      :["calc_hash2", "shash", ["-a", "MD5"], "MD5"],
+	"ripemd128":["calc_hash2", "shash", ["-a", "RIPEMD128"], "RIPEMD128"],
+	"ripemd160":["calc_hash2", "shash", ["-a", "RIPEMD160"], "RIPEMD160"],
+	"ripemd256":["calc_hash2", "shash", ["-a", "RIPEMD256"], "RIPEMD256"],
+	"ripemd320":["calc_hash2", "shash", ["-a", "RIPEMD320"], "RIPEMD320"],
+	"sha1"     :["calc_hash2", "shash", ["-a", "SHA1"], "SHA1"],
+	"sha224"   :["calc_hash2", "shash", ["-a", "SHA224"], "SHA224"],
+	"sha256"   :["calc_hash2", "shash", ["-a", "SHA256"], "SHA256"],
+	"sha384"   :["calc_hash2", "shash", ["-a", "SHA384"], "SHA384"],
+	"sha512"   :["calc_hash2", "shash", ["-a", "SHA512"], "SHA512"],
+	"snefru128":["calc_hash2", "shash", ["-a", "SNEFRU128"], "SNEFRU128"],
+	"snefru256":["calc_hash2", "shash", ["-a", "SNEFRU256"], "SNEFRU256"],
+	"tiger"    :["calc_hash2", "shash", ["-a", "TIGER"], "TIGER"],
+	"tiger128" :["calc_hash2", "shash", ["-a", "TIGER128"], "TIGER128"],
+	"tiger160" :["calc_hash2", "shash", ["-a", "TIGER160"], "TIGER160"],
+	"whirlpool":["calc_hash2", "shash", ["-a", "WHIRLPOOL"], "WHIRLPOOL"],
+	}
+
+
+class HashMap(object):
+	'''Class for handling
+	Catalyst's hash generation'''
+
+	fields = ["func", "cmd", "args", "id"]
+
+
+	def __init__(self, hashes=None):
+		'''Class init
+
+		@param hashes: dictionary of Key:[function, cmd, cmd_args, Print string]
+		@param fields: list of ordered field names for the hashes
+			eg: ["func", "cmd", "args", "id"]
+		'''
+		if hashes is None:
+			hashes = {}
+		self.hash_map = {}
+
+		# create the hash definition namedtuple classes
+		for name in list(hashes):
+			obj = namedtuple(name, self.fields)
+			obj.__slots__ = ()
+			self.hash_map[name] = obj._make(hashes[name])
+		del obj
+
+
+	def generate_hash(self, file_, hash_="crc32", verbose=False):
+		'''Prefered method of generating a hash for the passed in file_
+
+		@param file_: the file to generate the hash for
+		@param hash_: the hash algorythm to use
+		@param verbose: boolean
+		@returns the hash result
+		'''
+		try:
+			return getattr(self, self.hash_map[hash_].func)(
+				file_,
+				hash_,
+				verbose
+				)
+		except:
+			raise CatalystError,"Error generating hash, is appropriate " + \
+				"utility installed on your system?"
+
+
+	def calc_hash(self, file_, hash_, verbose=False):
+		'''
+		Calculate the hash for "file_"
+
+		@param file_: the file to generate the hash for
+		@param hash_: the hash algorythm to use
+		@param verbose: boolean
+		@returns the hash result
+		'''
+		_hash = self.hash_map[hash_]
+		args = [_hash.cmd]
+		args.extend(_hash.args)
+		args.append(file_)
+		source = Popen(args, stdout=PIPE)
+		mylines = source.communicate()[0]
+		mylines=mylines[0].split()
+		result=mylines[0]
+		if verbose:
+			print _hash.id + " (%s) = %s" % (file_, result)
+		return result
+
+
+	def calc_hash2(self, file_, hash_type, verbose=False):
+		'''
+		Calculate the hash for "file_"
+
+		@param file_: the file to generate the hash for
+		@param hash_: the hash algorythm to use
+		@param verbose: boolean
+		@returns the hash result
+		'''
+		_hash = self.hash_map[hash_type]
+		args = [_hash.cmd]
+		args.extend(_hash.args)
+		args.append(file_)
+		#print("DEBUG: calc_hash2; args =", args)
+		source = Popen(args, stdout=PIPE)
+		output = source.communicate()
+		lines = output[0].split('\n')
+		#print("DEBUG: calc_hash2; output =", output)
+		header = lines[0]
+		h_f = lines[1].split()
+		hash_result = h_f[0]
+		short_file = os.path.split(h_f[1])[1]
+		result = header + "\n" + hash_result + "  " + short_file + "\n"
+		if verbose:
+			print header + " (%s) = %s" % (short_file, result)
+		return result
+
+
+

diff --git a/catalyst/main.py b/catalyst/main.py
index 6b90989..7bcf2cb 100644
--- a/catalyst/main.py
+++ b/catalyst/main.py
@@ -22,7 +22,10 @@ from . import __version__
 import catalyst.config
 import catalyst.util
 from catalyst.support import (required_build_targets,
-	valid_build_targets, CatalystError, hash_map, find_binary, LockInUse)
+	valid_build_targets, CatalystError, find_binary, LockInUse)
+
+from hash_utils import HashMap, HASH_DEFINITIONS
+
 
 
 conf_values={}
@@ -345,40 +348,43 @@ def main():
 	# import configuration file and import our main module using those settings
 	parse_config(myconfig)
 
-	# Start checking that digests are valid now that the hash_map was imported
-	# from catalyst.support
+	# initialze our hash and contents generators
+	hash_map = HashMap(HASH_DEFINITIONS)
+	conf_values["hash_map"] = hash_map
+
+	# Start checking that digests are valid now that hash_map is initialized
 	if "digests" in conf_values:
 		for i in conf_values["digests"].split():
-			if i not in hash_map:
+			if i not in HASH_DEFINITIONS:
 				print
 				print i+" is not a valid digest entry"
 				print "Valid digest entries:"
-				print hash_map.keys()
+				print HASH_DEFINITIONS.keys()
 				print
 				print "Catalyst aborting...."
 				sys.exit(2)
-			if find_binary(hash_map[i][1]) == None:
+			if find_binary(hash_map.hash_map[i].cmd) == None:
 				print
-				print "digest="+i
-				print "\tThe "+hash_map[i][1]+\
+				print "digest=" + i
+				print "\tThe " + hash_map.hash_map[i].cmd + \
 					" binary was not found. It needs to be in your system path"
 				print
 				print "Catalyst aborting...."
 				sys.exit(2)
 	if "hash_function" in conf_values:
-		if conf_values["hash_function"] not in hash_map:
+		if conf_values["hash_function"] not in HASH_DEFINITIONS:
 			print
 			print conf_values["hash_function"]+\
 				" is not a valid hash_function entry"
 			print "Valid hash_function entries:"
-			print hash_map.keys()
+			print HASH_DEFINITIONS.keys()
 			print
 			print "Catalyst aborting...."
 			sys.exit(2)
-		if find_binary(hash_map[conf_values["hash_function"]][1]) == None:
+		if find_binary(hash_map.hash_map[conf_values["hash_function"]].cmd) == None:
 			print
 			print "hash_function="+conf_values["hash_function"]
-			print "\tThe "+hash_map[conf_values["hash_function"]][1]+\
+			print "\tThe "+hash_map.hash_map[conf_values["hash_function"]].cmd + \
 				" binary was not found. It needs to be in your system path"
 			print
 			print "Catalyst aborting...."

diff --git a/catalyst/support.py b/catalyst/support.py
index 5e7ce92..308d9c0 100644
--- a/catalyst/support.py
+++ b/catalyst/support.py
@@ -114,68 +114,6 @@ contents_map={
 	"isoinfo-f":[calc_contents,"isoinfo -f -i %(file)s"],
 }
 
-def generate_hash(file,hash_function="crc32",verbose=False):
-	try:
-		return hash_map[hash_function][0](file,hash_map[hash_function][1],hash_map[hash_function][2],\
-			hash_map[hash_function][3],verbose)
-	except:
-		raise CatalystError,"Error generating hash, is appropriate utility installed on your system?"
-
-def calc_hash(file,cmd,cmd_args,id_string="MD5",verbose=False):
-	a=os.popen(cmd+" "+cmd_args+" "+file)
-	mylines=a.readlines()
-	a.close()
-	mylines=mylines[0].split()
-	result=mylines[0]
-	if verbose:
-		print id_string+" (%s) = %s" % (file, result)
-	return result
-
-def calc_hash2(file,cmd,cmd_args,id_string="MD5",verbose=False):
-	a=os.popen(cmd+" "+cmd_args+" "+file)
-	header=a.readline()
-	mylines=a.readline().split()
-	hash=mylines[0]
-	short_file=os.path.split(mylines[1])[1]
-	a.close()
-	result=header+hash+"  "+short_file+"\n"
-	if verbose:
-		print header+" (%s) = %s" % (short_file, result)
-	return result
-
-# This has map must be defined after the function calc_hash
-# It is possible to call different functions from this but they must be defined
-# before hash_map
-# Key,function,cmd,cmd_args,Print string
-hash_map={
-	 "adler32":[calc_hash2,"shash","-a ADLER32","ADLER32"],\
-	 "crc32":[calc_hash2,"shash","-a CRC32","CRC32"],\
-	 "crc32b":[calc_hash2,"shash","-a CRC32B","CRC32B"],\
-	 "gost":[calc_hash2,"shash","-a GOST","GOST"],\
-	 "haval128":[calc_hash2,"shash","-a HAVAL128","HAVAL128"],\
-	 "haval160":[calc_hash2,"shash","-a HAVAL160","HAVAL160"],\
-	 "haval192":[calc_hash2,"shash","-a HAVAL192","HAVAL192"],\
-	 "haval224":[calc_hash2,"shash","-a HAVAL224","HAVAL224"],\
-	 "haval256":[calc_hash2,"shash","-a HAVAL256","HAVAL256"],\
-	 "md2":[calc_hash2,"shash","-a MD2","MD2"],\
-	 "md4":[calc_hash2,"shash","-a MD4","MD4"],\
-	 "md5":[calc_hash2,"shash","-a MD5","MD5"],\
-	 "ripemd128":[calc_hash2,"shash","-a RIPEMD128","RIPEMD128"],\
-	 "ripemd160":[calc_hash2,"shash","-a RIPEMD160","RIPEMD160"],\
-	 "ripemd256":[calc_hash2,"shash","-a RIPEMD256","RIPEMD256"],\
-	 "ripemd320":[calc_hash2,"shash","-a RIPEMD320","RIPEMD320"],\
-	 "sha1":[calc_hash2,"shash","-a SHA1","SHA1"],\
-	 "sha224":[calc_hash2,"shash","-a SHA224","SHA224"],\
-	 "sha256":[calc_hash2,"shash","-a SHA256","SHA256"],\
-	 "sha384":[calc_hash2,"shash","-a SHA384","SHA384"],\
-	 "sha512":[calc_hash2,"shash","-a SHA512","SHA512"],\
-	 "snefru128":[calc_hash2,"shash","-a SNEFRU128","SNEFRU128"],\
-	 "snefru256":[calc_hash2,"shash","-a SNEFRU256","SNEFRU256"],\
-	 "tiger":[calc_hash2,"shash","-a TIGER","TIGER"],\
-	 "tiger128":[calc_hash2,"shash","-a TIGER128","TIGER128"],\
-	 "tiger160":[calc_hash2,"shash","-a TIGER160","TIGER160"],\
-	 "whirlpool":[calc_hash2,"shash","-a WHIRLPOOL","WHIRLPOOL"],\
-	 }
 
 def read_from_clst(file):
 	line = ''

diff --git a/catalyst/targets/generic_stage_target.py b/catalyst/targets/generic_stage_target.py
index eaf2c1f..b6a6200 100644
--- a/catalyst/targets/generic_stage_target.py
+++ b/catalyst/targets/generic_stage_target.py
@@ -428,10 +428,11 @@ class generic_stage_target(generic_target):
 			if os.path.isfile(self.settings["source_path"]):
 				# XXX: Is this even necessary if the previous check passes?
 				if os.path.exists(self.settings["source_path"]):
-					self.settings["source_path_hash"]=\
-						generate_hash(self.settings["source_path"],\
-						hash_function=self.settings["hash_function"],\
-						verbose=False)
+					self.settings["source_path_hash"] = \
+						self.settings["hash_map"].generate_hash(
+							self.settings["source_path"],
+							hash_ = self.settings["hash_function"],
+							verbose = False)
 		print "Source path set to "+self.settings["source_path"]
 		if os.path.isdir(self.settings["source_path"]):
 			print "\tIf this is not desired, remove this directory or turn off"
@@ -457,18 +458,22 @@ class generic_stage_target(generic_target):
 			self.settings["snapshot"] + ".tar.xz")
 
 		if os.path.exists(self.settings["snapshot_path"]):
-			self.settings["snapshot_path_hash"]=\
-				generate_hash(self.settings["snapshot_path"],\
-				hash_function=self.settings["hash_function"],verbose=False)
+			self.settings["snapshot_path_hash"] = \
+				self.settings["hash_map"].generate_hash(
+					self.settings["snapshot_path"],
+					hash_ = self.settings["hash_function"],
+					verbose = False)
 		else:
 			self.settings["snapshot_path"]=normpath(self.settings["storedir"]+\
 				"/snapshots/" + self.settings["snapshot_name"] +
 				self.settings["snapshot"] + ".tar.bz2")
 
 			if os.path.exists(self.settings["snapshot_path"]):
-				self.settings["snapshot_path_hash"]=\
-					generate_hash(self.settings["snapshot_path"],\
-					hash_function=self.settings["hash_function"],verbose=False)
+				self.settings["snapshot_path_hash"] = \
+					self.settings["hash_map"].generate_hash(
+						self.settings["snapshot_path"],
+						hash_ = self.settings["hash_function"],
+						verbose = False)
 
 	def set_snapcache_path(self):
 		if "SNAPCACHE" in self.settings:
@@ -1716,6 +1721,7 @@ class generic_stage_target(generic_target):
 		if os.path.exists(file+".DIGESTS"):
 			os.remove(file+".DIGESTS")
 		if "digests" in self.settings:
+			hash_map = self.settings["hash_map"]
 			if os.path.exists(file):
 				myf=open(file+".DIGESTS","w")
 				keys={}
@@ -1726,14 +1732,14 @@ class generic_stage_target(generic_target):
 				for f in [file, file+'.CONTENTS']:
 					if os.path.exists(f):
 						if "all" in array:
-							for k in hash_map.keys():
-								hash=generate_hash(f,hash_function=k,verbose=\
-									"VERBOSE" in self.settings)
+							for k in list(hash_map.hash_map):
+								hash = hash_map.generate_hash(f, hash_ = k,
+									verbose = "VERBOSE" in self.settings)
 								myf.write(hash)
 						else:
 							for j in array:
-								hash=generate_hash(f,hash_function=j,verbose=\
-									"VERBOSE" in self.settings)
+								hash = hash_map.generate_hash(f, hash_ = j,
+									verbose = "VERBOSE" in self.settings)
 								myf.write(hash)
 				myf.close()
 

diff --git a/catalyst/targets/livecd_stage2_target.py b/catalyst/targets/livecd_stage2_target.py
index 1bfd820..e784844 100644
--- a/catalyst/targets/livecd_stage2_target.py
+++ b/catalyst/targets/livecd_stage2_target.py
@@ -35,7 +35,9 @@ class livecd_stage2_target(generic_stage_target):
 	def set_source_path(self):
 		self.settings["source_path"]=normpath(self.settings["storedir"]+"/builds/"+self.settings["source_subpath"]+".tar.bz2")
 		if os.path.isfile(self.settings["source_path"]):
-			self.settings["source_path_hash"]=generate_hash(self.settings["source_path"])
+			self.settings["source_path_hash"] = \
+				self.settings["hash_map"].generate_hash(
+					self.settings["source_path"])
 		else:
 			self.settings["source_path"]=normpath(self.settings["storedir"]+"/tmp/"+self.settings["source_subpath"]+"/")
 		if not os.path.exists(self.settings["source_path"]):

diff --git a/catalyst/targets/stage2_target.py b/catalyst/targets/stage2_target.py
index 15acdee..6377f5d 100644
--- a/catalyst/targets/stage2_target.py
+++ b/catalyst/targets/stage2_target.py
@@ -23,8 +23,11 @@ class stage2_target(generic_stage_target):
 			if os.path.isfile(self.settings["source_path"]):
 				if os.path.exists(self.settings["source_path"]):
 				# XXX: Is this even necessary if the previous check passes?
-					self.settings["source_path_hash"]=generate_hash(self.settings["source_path"],\
-						hash_function=self.settings["hash_function"],verbose=False)
+					self.settings["source_path_hash"] = \
+						self.settings["hash_map"].generate_hash(
+							self.settings["source_path"],\
+							hash_=self.settings["hash_function"],
+							verbose=False)
 		print "Source path set to "+self.settings["source_path"]
 		if os.path.isdir(self.settings["source_path"]):
 			print "\tIf this is not desired, remove this directory or turn of seedcache in the options of catalyst.conf"