Gentoo Archives: gentoo-commits

From: Brian Dolbec <brian.dolbec@×××××.com>
To: gentoo-commits@l.g.o
Subject: [gentoo-commits] proj/catalyst:pending commit in: catalyst/
Date: Tue, 02 Sep 2014 05:55:02
Message-Id: 1409637114.f26eff57984166c7477158745302deade6990aaf.dol-sen@gentoo
1 commit: f26eff57984166c7477158745302deade6990aaf
2 Author: Brian Dolbec <dolsen <AT> gentoo <DOT> org>
3 AuthorDate: Sun Jan 20 19:22:27 2013 +0000
4 Commit: Brian Dolbec <brian.dolbec <AT> gmail <DOT> com>
5 CommitDate: Tue Sep 2 05:51:54 2014 +0000
6 URL: http://sources.gentoo.org/gitweb/?p=proj/catalyst.git;a=commit;h=f26eff57
7
8 Some spacing, comment and indent cleanup
9
10 Conflicts:
11 catalyst/support.py
12
13 ---
14 catalyst/support.py | 405 +++++++++++++++++++++++++++-------------------------
15 1 file changed, 213 insertions(+), 192 deletions(-)
16
17 diff --git a/catalyst/support.py b/catalyst/support.py
18 index feaa645..6f66d3c 100644
19 --- a/catalyst/support.py
20 +++ b/catalyst/support.py
21 @@ -1,12 +1,19 @@
22
23 -import sys,string,os,types,re,signal,traceback,time
24 -#import md5,sha
25 +import sys
26 +import string
27 +import os
28 +import types
29 +import re
30 +import signal
31 +import traceback
32 +import time
33
34 from catalyst.defaults import verbosity, valid_config_file_values
35
36 selinux_capable = False
37 #userpriv_capable = (os.getuid() == 0)
38 #fakeroot_capable = False
39 +
40 BASH_BINARY = "/bin/bash"
41
42 # set it to 0 for the soft limit, 1 for the hard limit
43 @@ -25,35 +32,35 @@ spawned_pids = []
44
45
46 def cleanup(pids,block_exceptions=True):
47 - """function to go through and reap the list of pids passed to it"""
48 - global spawned_pids
49 - if type(pids) == int:
50 - pids = [pids]
51 - for x in pids:
52 - try:
53 - os.kill(x,signal.SIGTERM)
54 - if os.waitpid(x,os.WNOHANG)[1] == 0:
55 - # feisty bugger, still alive.
56 - os.kill(x,signal.SIGKILL)
57 - os.waitpid(x,0)
58 -
59 - except OSError, oe:
60 - if block_exceptions:
61 - pass
62 - if oe.errno not in (10,3):
63 - raise oe
64 - except SystemExit:
65 - raise
66 - except Exception:
67 - if block_exceptions:
68 - pass
69 - try: spawned_pids.remove(x)
70 - except IndexError: pass
71 -
72 -
73 -
74 -# a function to turn a string of non-printable characters into a string of
75 -# hex characters
76 + """function to go through and reap the list of pids passed to it"""
77 + global spawned_pids
78 + if type(pids) == int:
79 + pids = [pids]
80 + for x in pids:
81 + try:
82 + os.kill(x,signal.SIGTERM)
83 + if os.waitpid(x,os.WNOHANG)[1] == 0:
84 + # feisty bugger, still alive.
85 + os.kill(x,signal.SIGKILL)
86 + os.waitpid(x,0)
87 + except OSError, oe:
88 + if block_exceptions:
89 + pass
90 + if oe.errno not in (10,3):
91 + raise oe
92 + except SystemExit:
93 + raise
94 + except Exception:
95 + if block_exceptions:
96 + pass
97 + try:
98 + spawned_pids.remove(x)
99 + except IndexError:
100 + pass
101 +
102 +
103 +# a function to turn a string of non-printable characters
104 +# into a string of hex characters
105 def hexify(str):
106 hexStr = string.hexdigits
107 r = ''
108 @@ -61,7 +68,6 @@ def hexify(str):
109 i = ord(ch)
110 r = r + hexStr[(i >> 4) & 0xF] + hexStr[i & 0xF]
111 return r
112 -# hexify()
113
114
115 def read_from_clst(file):
116 @@ -77,7 +83,6 @@ def read_from_clst(file):
117 myline = myline + line
118 myf.close()
119 return myline
120 -# read_from_clst
121
122
123 def list_bashify(mylist):
124 @@ -92,6 +97,7 @@ def list_bashify(mylist):
125 mypack=string.join(mypack)
126 return mypack
127
128 +
129 def list_to_string(mylist):
130 if type(mylist)==types.StringType:
131 mypack=[mylist]
132 @@ -104,6 +110,7 @@ def list_to_string(mylist):
133 mypack=string.join(mypack)
134 return mypack
135
136 +
137 class CatalystError(Exception):
138 def __init__(self, message):
139 if message:
140 @@ -115,84 +122,83 @@ class CatalystError(Exception):
141 print "!!! catalyst: "+message
142 print
143
144 +
145 def die(msg=None):
146 warn(msg)
147 sys.exit(1)
148
149 +
150 def warn(msg):
151 print "!!! catalyst: "+msg
152
153 +
154 def find_binary(myc):
155 """look through the environmental path for an executable file named whatever myc is"""
156 - # this sucks. badly.
157 - p=os.getenv("PATH")
158 - if p == None:
159 - return None
160 - for x in p.split(":"):
161 - #if it exists, and is executable
162 - if os.path.exists("%s/%s" % (x,myc)) and os.stat("%s/%s" % (x,myc))[0] & 0x0248:
163 - return "%s/%s" % (x,myc)
164 - return None
165 + # this sucks. badly.
166 + p=os.getenv("PATH")
167 + if p == None:
168 + return None
169 + for x in p.split(":"):
170 + #if it exists, and is executable
171 + if os.path.exists("%s/%s" % (x,myc)) and os.stat("%s/%s" % (x,myc))[0] & 0x0248:
172 + return "%s/%s" % (x,myc)
173 + return None
174 +
175
176 def spawn_bash(mycommand,env={},debug=False,opt_name=None,**keywords):
177 """spawn mycommand as an arguement to bash"""
178 args=[BASH_BINARY]
179 if not opt_name:
180 - opt_name=mycommand.split()[0]
181 + opt_name=mycommand.split()[0]
182 if "BASH_ENV" not in env:
183 - env["BASH_ENV"] = "/etc/spork/is/not/valid/profile.env"
184 + env["BASH_ENV"] = "/etc/spork/is/not/valid/profile.env"
185 if debug:
186 - args.append("-x")
187 + args.append("-x")
188 args.append("-c")
189 args.append(mycommand)
190 return spawn(args,env=env,opt_name=opt_name,**keywords)
191
192 -#def spawn_get_output(mycommand,spawn_type=spawn,raw_exit_code=False,emulate_gso=True, \
193 -# collect_fds=[1],fd_pipes=None,**keywords):
194
195 def spawn_get_output(mycommand,raw_exit_code=False,emulate_gso=True, \
196 - collect_fds=[1],fd_pipes=None,**keywords):
197 - """call spawn, collecting the output to fd's specified in collect_fds list
198 - emulate_gso is a compatability hack to emulate commands.getstatusoutput's return, minus the
199 - requirement it always be a bash call (spawn_type controls the actual spawn call), and minus the
200 - 'lets let log only stdin and let stderr slide by'.
201 -
202 - emulate_gso was deprecated from the day it was added, so convert your code over.
203 - spawn_type is the passed in function to call- typically spawn_bash, spawn, spawn_sandbox, or spawn_fakeroot"""
204 - global selinux_capable
205 - pr,pw=os.pipe()
206 -
207 - #if type(spawn_type) not in [types.FunctionType, types.MethodType]:
208 - # s="spawn_type must be passed a function, not",type(spawn_type),spawn_type
209 - # raise Exception,s
210 -
211 - if fd_pipes==None:
212 - fd_pipes={}
213 - fd_pipes[0] = 0
214 -
215 - for x in collect_fds:
216 - fd_pipes[x] = pw
217 - keywords["returnpid"]=True
218 -
219 - mypid=spawn_bash(mycommand,fd_pipes=fd_pipes,**keywords)
220 - os.close(pw)
221 - if type(mypid) != types.ListType:
222 - os.close(pr)
223 - return [mypid, "%s: No such file or directory" % mycommand.split()[0]]
224 -
225 - fd=os.fdopen(pr,"r")
226 - mydata=fd.readlines()
227 - fd.close()
228 - if emulate_gso:
229 - mydata=string.join(mydata)
230 - if len(mydata) and mydata[-1] == "\n":
231 - mydata=mydata[:-1]
232 - retval=os.waitpid(mypid[0],0)[1]
233 - cleanup(mypid)
234 - if raw_exit_code:
235 - return [retval,mydata]
236 - retval=process_exit_code(retval)
237 - return [retval, mydata]
238 + collect_fds=[1],fd_pipes=None,**keywords):
239 + """call spawn, collecting the output to fd's specified in collect_fds list
240 + emulate_gso is a compatability hack to emulate commands.getstatusoutput's return, minus the
241 + requirement it always be a bash call (spawn_type controls the actual spawn call), and minus the
242 + 'lets let log only stdin and let stderr slide by'.
243 +
244 + emulate_gso was deprecated from the day it was added, so convert your code over.
245 + spawn_type is the passed in function to call- typically spawn_bash, spawn, spawn_sandbox, or spawn_fakeroot"""
246 + global selinux_capable
247 + pr,pw=os.pipe()
248 +
249 + if fd_pipes==None:
250 + fd_pipes={}
251 + fd_pipes[0] = 0
252 +
253 + for x in collect_fds:
254 + fd_pipes[x] = pw
255 + keywords["returnpid"]=True
256 +
257 + mypid=spawn_bash(mycommand,fd_pipes=fd_pipes,**keywords)
258 + os.close(pw)
259 + if type(mypid) != types.ListType:
260 + os.close(pr)
261 + return [mypid, "%s: No such file or directory" % mycommand.split()[0]]
262 +
263 + fd=os.fdopen(pr,"r")
264 + mydata=fd.readlines()
265 + fd.close()
266 + if emulate_gso:
267 + mydata=string.join(mydata)
268 + if len(mydata) and mydata[-1] == "\n":
269 + mydata=mydata[:-1]
270 + retval=os.waitpid(mypid[0],0)[1]
271 + cleanup(mypid)
272 + if raw_exit_code:
273 + return [retval,mydata]
274 + retval=process_exit_code(retval)
275 + return [retval, mydata]
276 +
277
278 # base spawn function
279 def spawn(mycommand,env={},raw_exit_code=False,opt_name=None,fd_pipes=None,returnpid=False,\
280 @@ -230,8 +236,8 @@ def spawn(mycommand,env={},raw_exit_code=False,opt_name=None,fd_pipes=None,retur
281 return None
282 myc = find_binary(myc)
283 if myc == None:
284 - return None
285 - mypid=[]
286 + return None
287 + mypid=[]
288 if logfile:
289 pr,pw=os.pipe()
290 mypid.extend(spawn(('tee','-i','-a',logfile),returnpid=True,fd_pipes={0:pr,1:1,2:2}))
291 @@ -295,77 +301,77 @@ def spawn(mycommand,env={},raw_exit_code=False,opt_name=None,fd_pipes=None,retur
292 if x not in trg_fd:
293 try:
294 os.close(x)
295 - except SystemExit, e:
296 - raise
297 - except:
298 - pass
299 -
300 - # note this order must be preserved- can't change gid/groups if you change uid first.
301 - if selinux_capable and selinux_context:
302 - import selinux
303 - selinux.setexec(selinux_context)
304 - if gid:
305 - os.setgid(gid)
306 - if groups:
307 - os.setgroups(groups)
308 - if uid:
309 - os.setuid(uid)
310 - if umask:
311 - os.umask(umask)
312 - else:
313 - os.umask(022)
314 -
315 - try:
316 - #print "execing", myc, myargs
317 - if func_call:
318 - # either use a passed in func for interpretting the results, or return if no exception.
319 - # note the passed in list, and dict are expanded.
320 - if len(mycommand) == 4:
321 - os._exit(mycommand[3](mycommand[0](*mycommand[1],**mycommand[2])))
322 - try:
323 - mycommand[0](*mycommand[1],**mycommand[2])
324 - except Exception,e:
325 - print "caught exception",e," in forked func",mycommand[0]
326 - sys.exit(0)
327 -
328 - #os.execvp(myc,myargs)
329 - os.execve(myc,myargs,env)
330 - except SystemExit, e:
331 - raise
332 - except Exception, e:
333 - if not func_call:
334 - raise str(e)+":\n "+myc+" "+string.join(myargs)
335 - print "func call failed"
336 -
337 - # If the execve fails, we need to report it, and exit
338 - # *carefully* --- report error here
339 - os._exit(1)
340 - sys.exit(1)
341 - return # should never get reached
342 -
343 - # if we were logging, kill the pipes.
344 - if logfile:
345 - os.close(pr)
346 - os.close(pw)
347 -
348 - if returnpid:
349 - return mypid
350 -
351 - # loop through pids (typically one, unless logging), either waiting on their death, or waxing them
352 - # if the main pid (mycommand) returned badly.
353 - while len(mypid):
354 - retval=os.waitpid(mypid[-1],0)[1]
355 - if retval != 0:
356 - cleanup(mypid[0:-1],block_exceptions=False)
357 - # at this point we've killed all other kid pids generated via this call.
358 - # return now.
359 - if raw_exit_code:
360 - return retval
361 - return process_exit_code(retval,throw_signals=raise_signals)
362 - else:
363 - mypid.pop(-1)
364 - cleanup(mypid)
365 - return 0
366 + except SystemExit, e:
367 + raise
368 + except:
369 + pass
370 +
371 + # note this order must be preserved- can't change gid/groups if you change uid first.
372 + if selinux_capable and selinux_context:
373 + import selinux
374 + selinux.setexec(selinux_context)
375 + if gid:
376 + os.setgid(gid)
377 + if groups:
378 + os.setgroups(groups)
379 + if uid:
380 + os.setuid(uid)
381 + if umask:
382 + os.umask(umask)
383 + else:
384 + os.umask(022)
385 +
386 + try:
387 + #print "execing", myc, myargs
388 + if func_call:
389 + # either use a passed in func for interpretting the results, or return if no exception.
390 + # note the passed in list, and dict are expanded.
391 + if len(mycommand) == 4:
392 + os._exit(mycommand[3](mycommand[0](*mycommand[1],**mycommand[2])))
393 + try:
394 + mycommand[0](*mycommand[1],**mycommand[2])
395 + except Exception,e:
396 + print "caught exception",e," in forked func",mycommand[0]
397 + sys.exit(0)
398 +
399 + os.execve(myc,myargs,env)
400 + except SystemExit, e:
401 + raise
402 + except Exception, e:
403 + if not func_call:
404 + raise str(e)+":\n "+myc+" "+string.join(myargs)
405 + print "func call failed"
406 +
407 + # If the execve fails, we need to report it, and exit
408 + # *carefully* --- report error here
409 + os._exit(1)
410 + sys.exit(1)
411 + return # should never get reached
412 +
413 + # if we were logging, kill the pipes.
414 + if logfile:
415 + os.close(pr)
416 + os.close(pw)
417 +
418 + if returnpid:
419 + return mypid
420 +
421 + # loop through pids (typically one, unless logging), either waiting on their death, or waxing them
422 + # if the main pid (mycommand) returned badly.
423 + while len(mypid):
424 + retval=os.waitpid(mypid[-1],0)[1]
425 + if retval != 0:
426 + cleanup(mypid[0:-1],block_exceptions=False)
427 + # at this point we've killed all other kid pids generated via this call.
428 + # return now.
429 + if raw_exit_code:
430 + return retval
431 + return process_exit_code(retval,throw_signals=raise_signals)
432 + else:
433 + mypid.pop(-1)
434 + cleanup(mypid)
435 + return 0
436 +
437
438 def cmd(mycmd,myexc="",env={}):
439 try:
440 @@ -376,19 +382,21 @@ def cmd(mycmd,myexc="",env={}):
441 except:
442 raise
443
444 +
445 def process_exit_code(retval,throw_signals=False):
446 - """process a waitpid returned exit code, returning exit code if it exit'd, or the
447 - signal if it died from signalling
448 - if throw_signals is on, it raises a SystemExit if the process was signaled.
449 - This is intended for usage with threads, although at the moment you can't signal individual
450 - threads in python, only the master thread, so it's a questionable option."""
451 - if (retval & 0xff)==0:
452 - return retval >> 8 # return exit code
453 - else:
454 - if throw_signals:
455 - #use systemexit, since portage is stupid about exception catching.
456 - raise SystemExit()
457 - return (retval & 0xff) << 8 # interrupted by signal
458 + """process a waitpid returned exit code, returning exit code if it exit'd, or the
459 + signal if it died from signalling
460 + if throw_signals is on, it raises a SystemExit if the process was signaled.
461 + This is intended for usage with threads, although at the moment you can't signal individual
462 + threads in python, only the master thread, so it's a questionable option."""
463 + if (retval & 0xff)==0:
464 + return retval >> 8 # return exit code
465 + else:
466 + if throw_signals:
467 + #use systemexit, since portage is stupid about exception catching.
468 + raise SystemExit()
469 + return (retval & 0xff) << 8 # interrupted by signal
470 +
471
472 def file_locate(settings,filelist,expand=1):
473 #if expand=1, non-absolute paths will be accepted and
474 @@ -398,15 +406,18 @@ def file_locate(settings,filelist,expand=1):
475 #filenames such as cdtar are optional, so we don't assume the variable is defined.
476 pass
477 else:
478 - if len(settings[myfile])==0:
479 - raise CatalystError, "File variable \""+myfile+"\" has a length of zero (not specified.)"
480 - if settings[myfile][0]=="/":
481 - if not os.path.exists(settings[myfile]):
482 - raise CatalystError, "Cannot locate specified "+myfile+": "+settings[myfile]
483 - elif expand and os.path.exists(os.getcwd()+"/"+settings[myfile]):
484 - settings[myfile]=os.getcwd()+"/"+settings[myfile]
485 - else:
486 - raise CatalystError, "Cannot locate specified "+myfile+": "+settings[myfile]+" (2nd try)"
487 + if len(settings[myfile])==0:
488 + raise CatalystError("File variable \"" + myfile +
489 + "\" has a length of zero (not specified.)", print_traceback=True)
490 + if settings[myfile][0]=="/":
491 + if not os.path.exists(settings[myfile]):
492 + raise CatalystError("Cannot locate specified " + myfile +
493 + ": "+settings[myfile], print_traceback=True)
494 + elif expand and os.path.exists(os.getcwd()+"/"+settings[myfile]):
495 + settings[myfile]=os.getcwd()+"/"+settings[myfile]
496 + else:
497 + raise CatalystError("Cannot locate specified " + myfile +
498 + ": "+settings[myfile]+" (2nd try)" +
499 """
500 Spec file format:
501
502 @@ -427,6 +438,8 @@ that the order of multiple-value items is preserved, but the order that the item
503 defined are not preserved. In other words, "foo", "bar", "oni" ordering is preserved but "item1"
504 "item2" "item3" ordering is not, as the item strings are stored in a dictionary (hash).
505 """
506 + , print_traceback=True)
507 +
508
509 def parse_makeconf(mylines):
510 mymakeconf={}
511 @@ -450,6 +463,7 @@ def parse_makeconf(mylines):
512 mymakeconf[mobj.group(1)]=clean_string
513 return mymakeconf
514
515 +
516 def read_makeconf(mymakeconffile):
517 if os.path.exists(mymakeconffile):
518 try:
519 @@ -475,10 +489,12 @@ def read_makeconf(mymakeconffile):
520 makeconf={}
521 return makeconf
522
523 +
524 def msg(mymsg,verblevel=1):
525 if verbosity>=verblevel:
526 print mymsg
527
528 +
529 def pathcompare(path1,path2):
530 # Change double slashes to slash
531 path1 = re.sub(r"//",r"/",path1)
532 @@ -491,6 +507,7 @@ def pathcompare(path1,path2):
533 return 1
534 return 0
535
536 +
537 def ismount(path):
538 "enhanced to handle bind mounts"
539 if os.path.ismount(path):
540 @@ -504,6 +521,7 @@ def ismount(path):
541 return 1
542 return 0
543
544 +
545 def addl_arg_parse(myspec,addlargs,requiredspec,validspec):
546 "helper function to help targets parse additional arguments"
547 global valid_config_file_values
548 @@ -522,6 +540,7 @@ def addl_arg_parse(myspec,addlargs,requiredspec,validspec):
549 if messages:
550 raise CatalystError, '\n\tAlso: '.join(messages)
551
552 +
553 def touch(myfile):
554 try:
555 myf=open(myfile,"w")
556 @@ -529,8 +548,9 @@ def touch(myfile):
557 except IOError:
558 raise CatalystError, "Could not touch "+myfile+"."
559
560 +
561 def countdown(secs=5, doing="Starting"):
562 - if secs:
563 + if secs:
564 print ">>> Waiting",secs,"seconds before starting..."
565 print ">>> (Control-C to abort)...\n"+doing+" in: ",
566 ticks=range(secs)
567 @@ -541,14 +561,15 @@ def countdown(secs=5, doing="Starting"):
568 time.sleep(1)
569 print
570
571 +
572 def normpath(mypath):
573 TrailingSlash=False
574 - if mypath[-1] == "/":
575 - TrailingSlash=True
576 - newpath = os.path.normpath(mypath)
577 - if len(newpath) > 1:
578 - if newpath[:2] == "//":
579 - newpath = newpath[1:]
580 + if mypath[-1] == "/":
581 + TrailingSlash=True
582 + newpath = os.path.normpath(mypath)
583 + if len(newpath) > 1:
584 + if newpath[:2] == "//":
585 + newpath = newpath[1:]
586 if TrailingSlash:
587 - newpath=newpath+'/'
588 - return newpath
589 + newpath=newpath+'/'
590 + return newpath