Gentoo Archives: gentoo-catalyst

From: Brian Dolbec <dolsen@g.o>
To: gentoo-catalyst@l.g.o
Subject: [gentoo-catalyst] [PATCH 3/5] Begin splitting up generic_stage_target into smaller code blocks.
Date: Thu, 11 Sep 2014 03:43:57
Message-Id: 1410407024-2445-4-git-send-email-dolsen@gentoo.org
In Reply to: [gentoo-catalyst] [PATCH 0/5] Next group of 5 patches by Brian Dolbec
1 This so snapshot_target does not need to import it since most of it was
2 not used or initialized properly.
3 ---
4 catalyst/defaults.py | 2 +-
5 catalyst/targets/clearbase.py | 115 +++++++++++++++++++++
6 catalyst/targets/genbase.py | 58 +++++++++++
7 catalyst/targets/generic_stage_target.py | 167 ++++---------------------------
8 catalyst/targets/generic_target.py | 15 ---
9 catalyst/targets/snapshot_target.py | 16 ++-
10 catalyst/targets/targetbase.py | 15 +++
11 7 files changed, 220 insertions(+), 168 deletions(-)
12 create mode 100644 catalyst/targets/clearbase.py
13 create mode 100644 catalyst/targets/genbase.py
14 delete mode 100644 catalyst/targets/generic_target.py
15 create mode 100644 catalyst/targets/targetbase.py
16
17 diff --git a/catalyst/defaults.py b/catalyst/defaults.py
18 index b36eff7..f6b2163 100644
19 --- a/catalyst/defaults.py
20 +++ b/catalyst/defaults.py
21 @@ -1,7 +1,7 @@
22
23
24 # these should never be touched
25 -required_build_targets = ["generic_target", "generic_stage_target"]
26 +required_build_targets = ["targetbase", "generic_stage_target"]
27
28 # new build types should be added here
29 valid_build_targets = ["stage1_target", "stage2_target", "stage3_target",
30 diff --git a/catalyst/targets/clearbase.py b/catalyst/targets/clearbase.py
31 new file mode 100644
32 index 0000000..8519acc
33 --- /dev/null
34 +++ b/catalyst/targets/clearbase.py
35 @@ -0,0 +1,115 @@
36 +
37 +import os
38 +import shutil
39 +from stat import ST_UID, ST_GID, ST_MODE
40 +
41 +
42 +from catalyst.support import cmd, countdown
43 +
44 +
45 +class ClearBase(object):
46 + """
47 + This class does all of clearing after task completion
48 + """
49 + def __init__(self, myspec):
50 + self.settings = myspec
51 +
52 +
53 +
54 + def clear_autoresume(self):
55 + """ Clean resume points since they are no longer needed """
56 + if "autoresume" in self.settings["options"]:
57 + print "Removing AutoResume Points: ..."
58 + myemp=self.settings["autoresume_path"]
59 + if os.path.isdir(myemp):
60 + if "autoresume" in self.settings["options"]:
61 + print "Emptying directory",myemp
62 + """
63 + stat the dir, delete the dir, recreate the dir and set
64 + the proper perms and ownership
65 + """
66 + mystat=os.stat(myemp)
67 + if os.uname()[0] == "FreeBSD":
68 + cmd("chflags -R noschg "+myemp,\
69 + "Could not remove immutable flag for file "\
70 + +myemp)
71 + #cmd("rm -rf "+myemp, "Could not remove existing file: "+myemp,env-self.env)
72 + shutil.rmtree(myemp)
73 + os.makedirs(myemp,0755)
74 + os.chown(myemp,mystat[ST_UID],mystat[ST_GID])
75 + os.chmod(myemp,mystat[ST_MODE])
76 +
77 +
78 + def clear_chroot(self):
79 + myemp=self.settings["chroot_path"]
80 + if os.path.isdir(myemp):
81 + print "Emptying directory",myemp
82 + """
83 + stat the dir, delete the dir, recreate the dir and set
84 + the proper perms and ownership
85 + """
86 + mystat=os.stat(myemp)
87 + #cmd("rm -rf "+myemp, "Could not remove existing file: "+myemp,env=self.env)
88 + """ There's no easy way to change flags recursively in python """
89 + if os.uname()[0] == "FreeBSD":
90 + os.system("chflags -R noschg "+myemp)
91 + shutil.rmtree(myemp)
92 + os.makedirs(myemp,0755)
93 + os.chown(myemp,mystat[ST_UID],mystat[ST_GID])
94 + os.chmod(myemp,mystat[ST_MODE])
95 +
96 +
97 + def clear_packages(self):
98 + if "pkgcache" in self.settings["options"]:
99 + print "purging the pkgcache ..."
100 +
101 + myemp=self.settings["pkgcache_path"]
102 + if os.path.isdir(myemp):
103 + print "Emptying directory",myemp
104 + """
105 + stat the dir, delete the dir, recreate the dir and set
106 + the proper perms and ownership
107 + """
108 + mystat=os.stat(myemp)
109 + #cmd("rm -rf "+myemp, "Could not remove existing file: "+myemp,env=self.env)
110 + shutil.rmtree(myemp)
111 + os.makedirs(myemp,0755)
112 + os.chown(myemp,mystat[ST_UID],mystat[ST_GID])
113 + os.chmod(myemp,mystat[ST_MODE])
114 +
115 +
116 + def clear_kerncache(self):
117 + if "kerncache" in self.settings["options"]:
118 + print "purging the kerncache ..."
119 +
120 + myemp=self.settings["kerncache_path"]
121 + if os.path.isdir(myemp):
122 + print "Emptying directory",myemp
123 + """
124 + stat the dir, delete the dir, recreate the dir and set
125 + the proper perms and ownership
126 + """
127 + mystat=os.stat(myemp)
128 + #cmd("rm -rf "+myemp, "Could not remove existing file: "+myemp,env=self.env)
129 + shutil.rmtree(myemp)
130 + os.makedirs(myemp,0755)
131 + os.chown(myemp,mystat[ST_UID],mystat[ST_GID])
132 + os.chmod(myemp,mystat[ST_MODE])
133 +
134 +
135 + def purge(self):
136 + countdown(10,"Purging Caches ...")
137 + if any(k in self.settings["options"] for k in ("purge","purgeonly","purgetmponly")):
138 + print "clearing autoresume ..."
139 + self.clear_autoresume()
140 +
141 + print "clearing chroot ..."
142 + self.clear_chroot()
143 +
144 + if "PURGETMPONLY" not in self.settings:
145 + print "clearing package cache ..."
146 + self.clear_packages()
147 +
148 + print "clearing kerncache ..."
149 + self.clear_kerncache()
150 +
151 diff --git a/catalyst/targets/genbase.py b/catalyst/targets/genbase.py
152 new file mode 100644
153 index 0000000..e818781
154 --- /dev/null
155 +++ b/catalyst/targets/genbase.py
156 @@ -0,0 +1,58 @@
157 +
158 +
159 +import os
160 +
161 +
162 +class GenBase(object):
163 + """
164 + This class does generation of the contents and digests files.
165 + """
166 + def __init__(self,myspec):
167 + self.settings = myspec
168 +
169 +
170 + def gen_contents_file(self,file):
171 + if os.path.exists(file+".CONTENTS"):
172 + os.remove(file+".CONTENTS")
173 + if "contents" in self.settings:
174 + contents_map = self.settings["contents_map"]
175 + if os.path.exists(file):
176 + myf=open(file+".CONTENTS","w")
177 + keys={}
178 + for i in self.settings["contents"].split():
179 + keys[i]=1
180 + array=keys.keys()
181 + array.sort()
182 + for j in array:
183 + contents = contents_map.generate_contents(file, j,
184 + verbose="VERBOSE" in self.settings)
185 + if contents:
186 + myf.write(contents)
187 + myf.close()
188 +
189 + def gen_digest_file(self,file):
190 + if os.path.exists(file+".DIGESTS"):
191 + os.remove(file+".DIGESTS")
192 + if "digests" in self.settings:
193 + hash_map = self.settings["hash_map"]
194 + if os.path.exists(file):
195 + myf=open(file+".DIGESTS","w")
196 + keys={}
197 + for i in self.settings["digests"].split():
198 + keys[i]=1
199 + array=keys.keys()
200 + array.sort()
201 + for f in [file, file+'.CONTENTS']:
202 + if os.path.exists(f):
203 + if "all" in array:
204 + for k in list(hash_map.hash_map):
205 + hash = hash_map.generate_hash(f,hash_=k,
206 + verbose = "VERBOSE" in self.settings)
207 + myf.write(hash)
208 + else:
209 + for j in array:
210 + hash = hash_map.generate_hash(f,hash_=j,
211 + verbose = "VERBOSE" in self.settings)
212 + myf.write(hash)
213 + myf.close()
214 +
215 diff --git a/catalyst/targets/generic_stage_target.py b/catalyst/targets/generic_stage_target.py
216 index 0b506e2..c6b8dcc 100644
217 --- a/catalyst/targets/generic_stage_target.py
218 +++ b/catalyst/targets/generic_stage_target.py
219 @@ -1,17 +1,26 @@
220 -import os,string,imp,types,shutil
221 -from catalyst.support import *
222 -from generic_target import *
223 -from stat import *
224 -from catalyst.lock import LockDir
225
226 -from catalyst.defaults import (SOURCE_MOUNT_DEFAULTS, TARGET_MOUNT_DEFAULTS,
227 - PORT_LOGDIR_CLEAN)
228 +import os
229 +import string
230 +import imp
231 +import types
232 +import shutil
233 +import sys
234 +from stat import ST_UID, ST_GID, ST_MODE
235
236 # for convienience
237 pjoin = os.path.join
238
239 +from catalyst.defaults import (SOURCE_MOUNT_DEFAULTS, TARGET_MOUNT_DEFAULTS,
240 + PORT_LOGDIR_CLEAN)
241 +from catalyst.support import (CatalystError, msg, file_locate, normpath,
242 + touch, cmd, warn, list_bashify, read_makeconf, read_from_clst, ismount)
243 +from catalyst.targets.targetbase import TargetBase
244 +from catalyst.targets.clearbase import ClearBase
245 +from catalyst.targets.genbase import GenBase
246 +from catalyst.lock import LockDir
247 +
248
249 -class generic_stage_target(generic_target):
250 +class generic_stage_target(TargetBase, ClearBase, GenBase):
251 """
252 This class does all of the chroot setup, copying of files, etc. It is
253 the driver class for pretty much everything that Catalyst does.
254 @@ -26,7 +35,9 @@ class generic_stage_target(generic_target):
255 "distcc_hosts","makeopts","pkgcache_path","kerncache_path"])
256
257 self.set_valid_build_kernel_vars(addlargs)
258 - generic_target.__init__(self,myspec,addlargs)
259 + TargetBase.__init__(self, myspec, addlargs)
260 + GenBase.__init__(self, myspec)
261 + ClearBase.__init__(self, myspec)
262
263 """
264 The semantics of subarchmap and machinemap changed a bit in 2.0.3 to
265 @@ -1616,142 +1627,4 @@ class generic_stage_target(generic_target):
266 self.unbind()
267 raise CatalystError,"build aborting due to livecd_update error."
268
269 - def clear_chroot(self):
270 - myemp=self.settings["chroot_path"]
271 - if os.path.isdir(myemp):
272 - print "Emptying directory",myemp
273 - """
274 - stat the dir, delete the dir, recreate the dir and set
275 - the proper perms and ownership
276 - """
277 - mystat=os.stat(myemp)
278 - #cmd("rm -rf "+myemp, "Could not remove existing file: "+myemp,env=self.env)
279 - """ There's no easy way to change flags recursively in python """
280 - if os.uname()[0] == "FreeBSD":
281 - os.system("chflags -R noschg "+myemp)
282 - shutil.rmtree(myemp)
283 - os.makedirs(myemp,0755)
284 - os.chown(myemp,mystat[ST_UID],mystat[ST_GID])
285 - os.chmod(myemp,mystat[ST_MODE])
286 -
287 - def clear_packages(self):
288 - if "pkgcache" in self.settings["options"]:
289 - print "purging the pkgcache ..."
290 -
291 - myemp=self.settings["pkgcache_path"]
292 - if os.path.isdir(myemp):
293 - print "Emptying directory",myemp
294 - """
295 - stat the dir, delete the dir, recreate the dir and set
296 - the proper perms and ownership
297 - """
298 - mystat=os.stat(myemp)
299 - #cmd("rm -rf "+myemp, "Could not remove existing file: "+myemp,env=self.env)
300 - shutil.rmtree(myemp)
301 - os.makedirs(myemp,0755)
302 - os.chown(myemp,mystat[ST_UID],mystat[ST_GID])
303 - os.chmod(myemp,mystat[ST_MODE])
304 -
305 - def clear_kerncache(self):
306 - if "kerncache" in self.settings["options"]:
307 - print "purging the kerncache ..."
308 -
309 - myemp=self.settings["kerncache_path"]
310 - if os.path.isdir(myemp):
311 - print "Emptying directory",myemp
312 - """
313 - stat the dir, delete the dir, recreate the dir and set
314 - the proper perms and ownership
315 - """
316 - mystat=os.stat(myemp)
317 - #cmd("rm -rf "+myemp, "Could not remove existing file: "+myemp,env=self.env)
318 - shutil.rmtree(myemp)
319 - os.makedirs(myemp,0755)
320 - os.chown(myemp,mystat[ST_UID],mystat[ST_GID])
321 - os.chmod(myemp,mystat[ST_MODE])
322 -
323 - def clear_autoresume(self):
324 - """ Clean resume points since they are no longer needed """
325 - if "autoresume" in self.settings["options"]:
326 - print "Removing AutoResume Points: ..."
327 - myemp=self.settings["autoresume_path"]
328 - if os.path.isdir(myemp):
329 - if "autoresume" in self.settings["options"]:
330 - print "Emptying directory",myemp
331 - """
332 - stat the dir, delete the dir, recreate the dir and set
333 - the proper perms and ownership
334 - """
335 - mystat=os.stat(myemp)
336 - if os.uname()[0] == "FreeBSD":
337 - cmd("chflags -R noschg "+myemp,\
338 - "Could not remove immutable flag for file "\
339 - +myemp)
340 - #cmd("rm -rf "+myemp, "Could not remove existing file: "+myemp,env-self.env)
341 - shutil.rmtree(myemp)
342 - os.makedirs(myemp,0755)
343 - os.chown(myemp,mystat[ST_UID],mystat[ST_GID])
344 - os.chmod(myemp,mystat[ST_MODE])
345 -
346 - def gen_contents_file(self,file):
347 - if os.path.exists(file+".CONTENTS"):
348 - os.remove(file+".CONTENTS")
349 - if "contents" in self.settings:
350 - contents_map = self.settings["contents_map"]
351 - if os.path.exists(file):
352 - myf=open(file+".CONTENTS","w")
353 - keys={}
354 - for i in self.settings["contents"].split():
355 - keys[i]=1
356 - array=keys.keys()
357 - array.sort()
358 - for j in array:
359 - contents = contents_map.generate_contents(file, j,
360 - verbose="VERBOSE" in self.settings)
361 - if contents:
362 - myf.write(contents)
363 - myf.close()
364 -
365 - def gen_digest_file(self,file):
366 - if os.path.exists(file+".DIGESTS"):
367 - os.remove(file+".DIGESTS")
368 - if "digests" in self.settings:
369 - hash_map = self.settings["hash_map"]
370 - if os.path.exists(file):
371 - myf=open(file+".DIGESTS","w")
372 - keys={}
373 - for i in self.settings["digests"].split():
374 - keys[i]=1
375 - array=keys.keys()
376 - array.sort()
377 - for f in [file, file+'.CONTENTS']:
378 - if os.path.exists(f):
379 - if "all" in array:
380 - for k in list(hash_map.hash_map):
381 - hash = hash_map.generate_hash(f, hash_ = k,
382 - verbose = "VERBOSE" in self.settings)
383 - myf.write(hash)
384 - else:
385 - for j in array:
386 - hash = hash_map.generate_hash(f, hash_ = j,
387 - verbose = "VERBOSE" in self.settings)
388 - myf.write(hash)
389 - myf.close()
390 -
391 - def purge(self):
392 - countdown(10,"Purging Caches ...")
393 - if any(k in self.settings["options"] for k in ("purge","purgeonly","purgetmponly")):
394 - print "clearing autoresume ..."
395 - self.clear_autoresume()
396 -
397 - print "clearing chroot ..."
398 - self.clear_chroot()
399 -
400 - if "PURGETMPONLY" not in self.settings:
401 - print "clearing package cache ..."
402 - self.clear_packages()
403 -
404 - print "clearing kerncache ..."
405 - self.clear_kerncache()
406 -
407 # vim: ts=4 sw=4 sta et sts=4 ai
408 diff --git a/catalyst/targets/generic_target.py b/catalyst/targets/generic_target.py
409 deleted file mode 100644
410 index 382f1c7..0000000
411 --- a/catalyst/targets/generic_target.py
412 +++ /dev/null
413 @@ -1,15 +0,0 @@
414 -import os
415 -
416 -from catalyst.support import *
417 -
418 -class generic_target:
419 - """
420 - The toplevel class for generic_stage_target. This is about as generic as we get.
421 - """
422 - def __init__(self,myspec,addlargs):
423 - addl_arg_parse(myspec,addlargs,self.required_values,self.valid_values)
424 - self.settings=myspec
425 - self.env = {
426 - 'PATH': '/bin:/sbin:/usr/bin:/usr/sbin',
427 - 'TERM': os.getenv('TERM', 'dumb'),
428 - }
429 diff --git a/catalyst/targets/snapshot_target.py b/catalyst/targets/snapshot_target.py
430 index 50133ec..3289bbd 100644
431 --- a/catalyst/targets/snapshot_target.py
432 +++ b/catalyst/targets/snapshot_target.py
433 @@ -3,10 +3,15 @@ Snapshot target
434 """
435
436 import os
437 -from catalyst.support import *
438 -from generic_stage_target import *
439 +import shutil
440 +from stat import ST_UID, ST_GID, ST_MODE
441
442 -class snapshot_target(generic_stage_target):
443 +
444 +from catalyst.support import normpath, cmd
445 +from catalyst.targets.targetbase import TargetBase
446 +from catalyst.targets.genbase import GenBase
447 +
448 +class snapshot_target(TargetBase, GenBase):
449 """
450 Builder class for snapshots.
451 """
452 @@ -14,8 +19,9 @@ class snapshot_target(generic_stage_target):
453 self.required_values=["version_stamp","target"]
454 self.valid_values=["version_stamp","target"]
455
456 - generic_target.__init__(self,myspec,addlargs)
457 - self.settings=myspec
458 + TargetBase.__init__(self, myspec, addlargs)
459 + GenBase.__init__(self,myspec)
460 + #self.settings=myspec
461 self.settings["target_subpath"]="portage"
462 st=self.settings["storedir"]
463 self.settings["snapshot_path"] = normpath(st + "/snapshots/"
464 diff --git a/catalyst/targets/targetbase.py b/catalyst/targets/targetbase.py
465 new file mode 100644
466 index 0000000..e0c03df
467 --- /dev/null
468 +++ b/catalyst/targets/targetbase.py
469 @@ -0,0 +1,15 @@
470 +import os
471 +
472 +from catalyst.support import *
473 +
474 +class TargetBase(object):
475 + """
476 + The toplevel class for all targets. This is about as generic as we get.
477 + """
478 + def __init__(self, myspec, addlargs):
479 + addl_arg_parse(myspec,addlargs,self.required_values,self.valid_values)
480 + self.settings=myspec
481 + self.env = {
482 + 'PATH': '/bin:/sbin:/usr/bin:/usr/sbin',
483 + 'TERM': os.getenv('TERM', 'dumb'),
484 + }
485 --
486 2.1.0