1 |
--- |
2 |
catalyst/hash_utils.py | 137 +++++++++++++++++++++++++++++++ |
3 |
catalyst/main.py | 30 ++++--- |
4 |
catalyst/support.py | 62 -------------- |
5 |
catalyst/targets/generic_stage_target.py | 36 ++++---- |
6 |
catalyst/targets/livecd_stage2_target.py | 4 +- |
7 |
catalyst/targets/stage2_target.py | 7 +- |
8 |
6 files changed, 184 insertions(+), 92 deletions(-) |
9 |
create mode 100644 catalyst/hash_utils.py |
10 |
|
11 |
diff --git a/catalyst/hash_utils.py b/catalyst/hash_utils.py |
12 |
new file mode 100644 |
13 |
index 0000000..b575ace |
14 |
--- /dev/null |
15 |
+++ b/catalyst/hash_utils.py |
16 |
@@ -0,0 +1,137 @@ |
17 |
+ |
18 |
+import os |
19 |
+from collections import namedtuple |
20 |
+from subprocess import Popen, PIPE |
21 |
+ |
22 |
+from support import CatalystError |
23 |
+ |
24 |
+ |
25 |
+# Use HashMap.fields for the value legend |
26 |
+# fields = ["func", "cmd", "args", "id"] |
27 |
+HASH_DEFINITIONS = { |
28 |
+ "adler32" :["calc_hash2", "shash", ["-a", "ADLER32"], "ADLER32"], |
29 |
+ "crc32" :["calc_hash2", "shash", ["-a", "CRC32"], "CRC32"], |
30 |
+ "crc32b" :["calc_hash2", "shash", ["-a", "CRC32B"], "CRC32B"], |
31 |
+ "gost" :["calc_hash2", "shash", ["-a", "GOST"], "GOST"], |
32 |
+ "haval128" :["calc_hash2", "shash", ["-a", "HAVAL128"], "HAVAL128"], |
33 |
+ "haval160" :["calc_hash2", "shash", ["-a", "HAVAL160"], "HAVAL160"], |
34 |
+ "haval192" :["calc_hash2", "shash", ["-a", "HAVAL192"], "HAVAL192"], |
35 |
+ "haval224" :["calc_hash2", "shash", ["-a", "HAVAL224"], "HAVAL224"], |
36 |
+ "haval256" :["calc_hash2", "shash", ["-a", "HAVAL256"], "HAVAL256"], |
37 |
+ "md2" :["calc_hash2", "shash", ["-a", "MD2"], "MD2"], |
38 |
+ "md4" :["calc_hash2", "shash", ["-a", "MD4"], "MD4"], |
39 |
+ "md5" :["calc_hash2", "shash", ["-a", "MD5"], "MD5"], |
40 |
+ "ripemd128":["calc_hash2", "shash", ["-a", "RIPEMD128"], "RIPEMD128"], |
41 |
+ "ripemd160":["calc_hash2", "shash", ["-a", "RIPEMD160"], "RIPEMD160"], |
42 |
+ "ripemd256":["calc_hash2", "shash", ["-a", "RIPEMD256"], "RIPEMD256"], |
43 |
+ "ripemd320":["calc_hash2", "shash", ["-a", "RIPEMD320"], "RIPEMD320"], |
44 |
+ "sha1" :["calc_hash2", "shash", ["-a", "SHA1"], "SHA1"], |
45 |
+ "sha224" :["calc_hash2", "shash", ["-a", "SHA224"], "SHA224"], |
46 |
+ "sha256" :["calc_hash2", "shash", ["-a", "SHA256"], "SHA256"], |
47 |
+ "sha384" :["calc_hash2", "shash", ["-a", "SHA384"], "SHA384"], |
48 |
+ "sha512" :["calc_hash2", "shash", ["-a", "SHA512"], "SHA512"], |
49 |
+ "snefru128":["calc_hash2", "shash", ["-a", "SNEFRU128"], "SNEFRU128"], |
50 |
+ "snefru256":["calc_hash2", "shash", ["-a", "SNEFRU256"], "SNEFRU256"], |
51 |
+ "tiger" :["calc_hash2", "shash", ["-a", "TIGER"], "TIGER"], |
52 |
+ "tiger128" :["calc_hash2", "shash", ["-a", "TIGER128"], "TIGER128"], |
53 |
+ "tiger160" :["calc_hash2", "shash", ["-a", "TIGER160"], "TIGER160"], |
54 |
+ "whirlpool":["calc_hash2", "shash", ["-a", "WHIRLPOOL"], "WHIRLPOOL"], |
55 |
+ } |
56 |
+ |
57 |
+ |
58 |
+class HashMap(object): |
59 |
+ '''Class for handling |
60 |
+ Catalyst's hash generation''' |
61 |
+ |
62 |
+ fields = ["func", "cmd", "args", "id"] |
63 |
+ |
64 |
+ |
65 |
+ def __init__(self, hashes=None): |
66 |
+ '''Class init |
67 |
+ |
68 |
+ @param hashes: dictionary of Key:[function, cmd, cmd_args, Print string] |
69 |
+ @param fields: list of ordered field names for the hashes |
70 |
+ eg: ["func", "cmd", "args", "id"] |
71 |
+ ''' |
72 |
+ if hashes is None: |
73 |
+ hashes = {} |
74 |
+ self.hash_map = {} |
75 |
+ |
76 |
+ # create the hash definition namedtuple classes |
77 |
+ for name in list(hashes): |
78 |
+ obj = namedtuple(name, self.fields) |
79 |
+ obj.__slots__ = () |
80 |
+ self.hash_map[name] = obj._make(hashes[name]) |
81 |
+ del obj |
82 |
+ |
83 |
+ |
84 |
+ def generate_hash(self, file_, hash_="crc32", verbose=False): |
85 |
+ '''Prefered method of generating a hash for the passed in file_ |
86 |
+ |
87 |
+ @param file_: the file to generate the hash for |
88 |
+ @param hash_: the hash algorythm to use |
89 |
+ @param verbose: boolean |
90 |
+ @returns the hash result |
91 |
+ ''' |
92 |
+ try: |
93 |
+ return getattr(self, self.hash_map[hash_].func)( |
94 |
+ file_, |
95 |
+ hash_, |
96 |
+ verbose |
97 |
+ ) |
98 |
+ except: |
99 |
+ raise CatalystError,"Error generating hash, is appropriate " + \ |
100 |
+ "utility installed on your system?" |
101 |
+ |
102 |
+ |
103 |
+ def calc_hash(self, file_, hash_, verbose=False): |
104 |
+ ''' |
105 |
+ Calculate the hash for "file_" |
106 |
+ |
107 |
+ @param file_: the file to generate the hash for |
108 |
+ @param hash_: the hash algorythm to use |
109 |
+ @param verbose: boolean |
110 |
+ @returns the hash result |
111 |
+ ''' |
112 |
+ _hash = self.hash_map[hash_] |
113 |
+ args = [_hash.cmd] |
114 |
+ args.extend(_hash.args) |
115 |
+ args.append(file_) |
116 |
+ source = Popen(args, stdout=PIPE) |
117 |
+ mylines = source.communicate()[0] |
118 |
+ mylines=mylines[0].split() |
119 |
+ result=mylines[0] |
120 |
+ if verbose: |
121 |
+ print _hash.id + " (%s) = %s" % (file_, result) |
122 |
+ return result |
123 |
+ |
124 |
+ |
125 |
+ def calc_hash2(self, file_, hash_type, verbose=False): |
126 |
+ ''' |
127 |
+ Calculate the hash for "file_" |
128 |
+ |
129 |
+ @param file_: the file to generate the hash for |
130 |
+ @param hash_: the hash algorythm to use |
131 |
+ @param verbose: boolean |
132 |
+ @returns the hash result |
133 |
+ ''' |
134 |
+ _hash = self.hash_map[hash_type] |
135 |
+ args = [_hash.cmd] |
136 |
+ args.extend(_hash.args) |
137 |
+ args.append(file_) |
138 |
+ #print("DEBUG: calc_hash2; args =", args) |
139 |
+ source = Popen(args, stdout=PIPE) |
140 |
+ output = source.communicate() |
141 |
+ lines = output[0].split('\n') |
142 |
+ #print("DEBUG: calc_hash2; output =", output) |
143 |
+ header = lines[0] |
144 |
+ h_f = lines[1].split() |
145 |
+ hash_result = h_f[0] |
146 |
+ short_file = os.path.split(h_f[1])[1] |
147 |
+ result = header + "\n" + hash_result + " " + short_file + "\n" |
148 |
+ if verbose: |
149 |
+ print header + " (%s) = %s" % (short_file, result) |
150 |
+ return result |
151 |
+ |
152 |
+ |
153 |
+ |
154 |
diff --git a/catalyst/main.py b/catalyst/main.py |
155 |
index 6b90989..7bcf2cb 100644 |
156 |
--- a/catalyst/main.py |
157 |
+++ b/catalyst/main.py |
158 |
@@ -22,7 +22,10 @@ from . import __version__ |
159 |
import catalyst.config |
160 |
import catalyst.util |
161 |
from catalyst.support import (required_build_targets, |
162 |
- valid_build_targets, CatalystError, hash_map, find_binary, LockInUse) |
163 |
+ valid_build_targets, CatalystError, find_binary, LockInUse) |
164 |
+ |
165 |
+from hash_utils import HashMap, HASH_DEFINITIONS |
166 |
+ |
167 |
|
168 |
|
169 |
conf_values={} |
170 |
@@ -345,40 +348,43 @@ def main(): |
171 |
# import configuration file and import our main module using those settings |
172 |
parse_config(myconfig) |
173 |
|
174 |
- # Start checking that digests are valid now that the hash_map was imported |
175 |
- # from catalyst.support |
176 |
+ # initialze our hash and contents generators |
177 |
+ hash_map = HashMap(HASH_DEFINITIONS) |
178 |
+ conf_values["hash_map"] = hash_map |
179 |
+ |
180 |
+ # Start checking that digests are valid now that hash_map is initialized |
181 |
if "digests" in conf_values: |
182 |
for i in conf_values["digests"].split(): |
183 |
- if i not in hash_map: |
184 |
+ if i not in HASH_DEFINITIONS: |
185 |
print |
186 |
print i+" is not a valid digest entry" |
187 |
print "Valid digest entries:" |
188 |
- print hash_map.keys() |
189 |
+ print HASH_DEFINITIONS.keys() |
190 |
print |
191 |
print "Catalyst aborting...." |
192 |
sys.exit(2) |
193 |
- if find_binary(hash_map[i][1]) == None: |
194 |
+ if find_binary(hash_map.hash_map[i].cmd) == None: |
195 |
print |
196 |
- print "digest="+i |
197 |
- print "\tThe "+hash_map[i][1]+\ |
198 |
+ print "digest=" + i |
199 |
+ print "\tThe " + hash_map.hash_map[i].cmd + \ |
200 |
" binary was not found. It needs to be in your system path" |
201 |
print |
202 |
print "Catalyst aborting...." |
203 |
sys.exit(2) |
204 |
if "hash_function" in conf_values: |
205 |
- if conf_values["hash_function"] not in hash_map: |
206 |
+ if conf_values["hash_function"] not in HASH_DEFINITIONS: |
207 |
print |
208 |
print conf_values["hash_function"]+\ |
209 |
" is not a valid hash_function entry" |
210 |
print "Valid hash_function entries:" |
211 |
- print hash_map.keys() |
212 |
+ print HASH_DEFINITIONS.keys() |
213 |
print |
214 |
print "Catalyst aborting...." |
215 |
sys.exit(2) |
216 |
- if find_binary(hash_map[conf_values["hash_function"]][1]) == None: |
217 |
+ if find_binary(hash_map.hash_map[conf_values["hash_function"]].cmd) == None: |
218 |
print |
219 |
print "hash_function="+conf_values["hash_function"] |
220 |
- print "\tThe "+hash_map[conf_values["hash_function"]][1]+\ |
221 |
+ print "\tThe "+hash_map.hash_map[conf_values["hash_function"]].cmd + \ |
222 |
" binary was not found. It needs to be in your system path" |
223 |
print |
224 |
print "Catalyst aborting...." |
225 |
diff --git a/catalyst/support.py b/catalyst/support.py |
226 |
index 5e7ce92..308d9c0 100644 |
227 |
--- a/catalyst/support.py |
228 |
+++ b/catalyst/support.py |
229 |
@@ -114,68 +114,6 @@ contents_map={ |
230 |
"isoinfo-f":[calc_contents,"isoinfo -f -i %(file)s"], |
231 |
} |
232 |
|
233 |
-def generate_hash(file,hash_function="crc32",verbose=False): |
234 |
- try: |
235 |
- return hash_map[hash_function][0](file,hash_map[hash_function][1],hash_map[hash_function][2],\ |
236 |
- hash_map[hash_function][3],verbose) |
237 |
- except: |
238 |
- raise CatalystError,"Error generating hash, is appropriate utility installed on your system?" |
239 |
- |
240 |
-def calc_hash(file,cmd,cmd_args,id_string="MD5",verbose=False): |
241 |
- a=os.popen(cmd+" "+cmd_args+" "+file) |
242 |
- mylines=a.readlines() |
243 |
- a.close() |
244 |
- mylines=mylines[0].split() |
245 |
- result=mylines[0] |
246 |
- if verbose: |
247 |
- print id_string+" (%s) = %s" % (file, result) |
248 |
- return result |
249 |
- |
250 |
-def calc_hash2(file,cmd,cmd_args,id_string="MD5",verbose=False): |
251 |
- a=os.popen(cmd+" "+cmd_args+" "+file) |
252 |
- header=a.readline() |
253 |
- mylines=a.readline().split() |
254 |
- hash=mylines[0] |
255 |
- short_file=os.path.split(mylines[1])[1] |
256 |
- a.close() |
257 |
- result=header+hash+" "+short_file+"\n" |
258 |
- if verbose: |
259 |
- print header+" (%s) = %s" % (short_file, result) |
260 |
- return result |
261 |
- |
262 |
-# This has map must be defined after the function calc_hash |
263 |
-# It is possible to call different functions from this but they must be defined |
264 |
-# before hash_map |
265 |
-# Key,function,cmd,cmd_args,Print string |
266 |
-hash_map={ |
267 |
- "adler32":[calc_hash2,"shash","-a ADLER32","ADLER32"],\ |
268 |
- "crc32":[calc_hash2,"shash","-a CRC32","CRC32"],\ |
269 |
- "crc32b":[calc_hash2,"shash","-a CRC32B","CRC32B"],\ |
270 |
- "gost":[calc_hash2,"shash","-a GOST","GOST"],\ |
271 |
- "haval128":[calc_hash2,"shash","-a HAVAL128","HAVAL128"],\ |
272 |
- "haval160":[calc_hash2,"shash","-a HAVAL160","HAVAL160"],\ |
273 |
- "haval192":[calc_hash2,"shash","-a HAVAL192","HAVAL192"],\ |
274 |
- "haval224":[calc_hash2,"shash","-a HAVAL224","HAVAL224"],\ |
275 |
- "haval256":[calc_hash2,"shash","-a HAVAL256","HAVAL256"],\ |
276 |
- "md2":[calc_hash2,"shash","-a MD2","MD2"],\ |
277 |
- "md4":[calc_hash2,"shash","-a MD4","MD4"],\ |
278 |
- "md5":[calc_hash2,"shash","-a MD5","MD5"],\ |
279 |
- "ripemd128":[calc_hash2,"shash","-a RIPEMD128","RIPEMD128"],\ |
280 |
- "ripemd160":[calc_hash2,"shash","-a RIPEMD160","RIPEMD160"],\ |
281 |
- "ripemd256":[calc_hash2,"shash","-a RIPEMD256","RIPEMD256"],\ |
282 |
- "ripemd320":[calc_hash2,"shash","-a RIPEMD320","RIPEMD320"],\ |
283 |
- "sha1":[calc_hash2,"shash","-a SHA1","SHA1"],\ |
284 |
- "sha224":[calc_hash2,"shash","-a SHA224","SHA224"],\ |
285 |
- "sha256":[calc_hash2,"shash","-a SHA256","SHA256"],\ |
286 |
- "sha384":[calc_hash2,"shash","-a SHA384","SHA384"],\ |
287 |
- "sha512":[calc_hash2,"shash","-a SHA512","SHA512"],\ |
288 |
- "snefru128":[calc_hash2,"shash","-a SNEFRU128","SNEFRU128"],\ |
289 |
- "snefru256":[calc_hash2,"shash","-a SNEFRU256","SNEFRU256"],\ |
290 |
- "tiger":[calc_hash2,"shash","-a TIGER","TIGER"],\ |
291 |
- "tiger128":[calc_hash2,"shash","-a TIGER128","TIGER128"],\ |
292 |
- "tiger160":[calc_hash2,"shash","-a TIGER160","TIGER160"],\ |
293 |
- "whirlpool":[calc_hash2,"shash","-a WHIRLPOOL","WHIRLPOOL"],\ |
294 |
- } |
295 |
|
296 |
def read_from_clst(file): |
297 |
line = '' |
298 |
diff --git a/catalyst/targets/generic_stage_target.py b/catalyst/targets/generic_stage_target.py |
299 |
index eaf2c1f..b6a6200 100644 |
300 |
--- a/catalyst/targets/generic_stage_target.py |
301 |
+++ b/catalyst/targets/generic_stage_target.py |
302 |
@@ -428,10 +428,11 @@ class generic_stage_target(generic_target): |
303 |
if os.path.isfile(self.settings["source_path"]): |
304 |
# XXX: Is this even necessary if the previous check passes? |
305 |
if os.path.exists(self.settings["source_path"]): |
306 |
- self.settings["source_path_hash"]=\ |
307 |
- generate_hash(self.settings["source_path"],\ |
308 |
- hash_function=self.settings["hash_function"],\ |
309 |
- verbose=False) |
310 |
+ self.settings["source_path_hash"] = \ |
311 |
+ self.settings["hash_map"].generate_hash( |
312 |
+ self.settings["source_path"], |
313 |
+ hash_ = self.settings["hash_function"], |
314 |
+ verbose = False) |
315 |
print "Source path set to "+self.settings["source_path"] |
316 |
if os.path.isdir(self.settings["source_path"]): |
317 |
print "\tIf this is not desired, remove this directory or turn off" |
318 |
@@ -457,18 +458,22 @@ class generic_stage_target(generic_target): |
319 |
self.settings["snapshot"] + ".tar.xz") |
320 |
|
321 |
if os.path.exists(self.settings["snapshot_path"]): |
322 |
- self.settings["snapshot_path_hash"]=\ |
323 |
- generate_hash(self.settings["snapshot_path"],\ |
324 |
- hash_function=self.settings["hash_function"],verbose=False) |
325 |
+ self.settings["snapshot_path_hash"] = \ |
326 |
+ self.settings["hash_map"].generate_hash( |
327 |
+ self.settings["snapshot_path"], |
328 |
+ hash_ = self.settings["hash_function"], |
329 |
+ verbose = False) |
330 |
else: |
331 |
self.settings["snapshot_path"]=normpath(self.settings["storedir"]+\ |
332 |
"/snapshots/" + self.settings["snapshot_name"] + |
333 |
self.settings["snapshot"] + ".tar.bz2") |
334 |
|
335 |
if os.path.exists(self.settings["snapshot_path"]): |
336 |
- self.settings["snapshot_path_hash"]=\ |
337 |
- generate_hash(self.settings["snapshot_path"],\ |
338 |
- hash_function=self.settings["hash_function"],verbose=False) |
339 |
+ self.settings["snapshot_path_hash"] = \ |
340 |
+ self.settings["hash_map"].generate_hash( |
341 |
+ self.settings["snapshot_path"], |
342 |
+ hash_ = self.settings["hash_function"], |
343 |
+ verbose = False) |
344 |
|
345 |
def set_snapcache_path(self): |
346 |
if "SNAPCACHE" in self.settings: |
347 |
@@ -1716,6 +1721,7 @@ class generic_stage_target(generic_target): |
348 |
if os.path.exists(file+".DIGESTS"): |
349 |
os.remove(file+".DIGESTS") |
350 |
if "digests" in self.settings: |
351 |
+ hash_map = self.settings["hash_map"] |
352 |
if os.path.exists(file): |
353 |
myf=open(file+".DIGESTS","w") |
354 |
keys={} |
355 |
@@ -1726,14 +1732,14 @@ class generic_stage_target(generic_target): |
356 |
for f in [file, file+'.CONTENTS']: |
357 |
if os.path.exists(f): |
358 |
if "all" in array: |
359 |
- for k in hash_map.keys(): |
360 |
- hash=generate_hash(f,hash_function=k,verbose=\ |
361 |
- "VERBOSE" in self.settings) |
362 |
+ for k in list(hash_map.hash_map): |
363 |
+ hash = hash_map.generate_hash(f, hash_ = k, |
364 |
+ verbose = "VERBOSE" in self.settings) |
365 |
myf.write(hash) |
366 |
else: |
367 |
for j in array: |
368 |
- hash=generate_hash(f,hash_function=j,verbose=\ |
369 |
- "VERBOSE" in self.settings) |
370 |
+ hash = hash_map.generate_hash(f, hash_ = j, |
371 |
+ verbose = "VERBOSE" in self.settings) |
372 |
myf.write(hash) |
373 |
myf.close() |
374 |
|
375 |
diff --git a/catalyst/targets/livecd_stage2_target.py b/catalyst/targets/livecd_stage2_target.py |
376 |
index 1bfd820..e784844 100644 |
377 |
--- a/catalyst/targets/livecd_stage2_target.py |
378 |
+++ b/catalyst/targets/livecd_stage2_target.py |
379 |
@@ -35,7 +35,9 @@ class livecd_stage2_target(generic_stage_target): |
380 |
def set_source_path(self): |
381 |
self.settings["source_path"]=normpath(self.settings["storedir"]+"/builds/"+self.settings["source_subpath"]+".tar.bz2") |
382 |
if os.path.isfile(self.settings["source_path"]): |
383 |
- self.settings["source_path_hash"]=generate_hash(self.settings["source_path"]) |
384 |
+ self.settings["source_path_hash"] = \ |
385 |
+ self.settings["hash_map"].generate_hash( |
386 |
+ self.settings["source_path"]) |
387 |
else: |
388 |
self.settings["source_path"]=normpath(self.settings["storedir"]+"/tmp/"+self.settings["source_subpath"]+"/") |
389 |
if not os.path.exists(self.settings["source_path"]): |
390 |
diff --git a/catalyst/targets/stage2_target.py b/catalyst/targets/stage2_target.py |
391 |
index 15acdee..6377f5d 100644 |
392 |
--- a/catalyst/targets/stage2_target.py |
393 |
+++ b/catalyst/targets/stage2_target.py |
394 |
@@ -23,8 +23,11 @@ class stage2_target(generic_stage_target): |
395 |
if os.path.isfile(self.settings["source_path"]): |
396 |
if os.path.exists(self.settings["source_path"]): |
397 |
# XXX: Is this even necessary if the previous check passes? |
398 |
- self.settings["source_path_hash"]=generate_hash(self.settings["source_path"],\ |
399 |
- hash_function=self.settings["hash_function"],verbose=False) |
400 |
+ self.settings["source_path_hash"] = \ |
401 |
+ self.settings["hash_map"].generate_hash( |
402 |
+ self.settings["source_path"],\ |
403 |
+ hash_=self.settings["hash_function"], |
404 |
+ verbose=False) |
405 |
print "Source path set to "+self.settings["source_path"] |
406 |
if os.path.isdir(self.settings["source_path"]): |
407 |
print "\tIf this is not desired, remove this directory or turn of seedcache in the options of catalyst.conf" |
408 |
-- |
409 |
1.8.5.3 |