[gentoo-commits] proj/catalyst:master commit in: catalyst/base/, catalyst/, catalyst/targets/

[Brian Dolbec]

commit:     f98b415f062b624bde02d48768372c9848abe175
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Fri May 31 16:11:30 2013 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Thu Jan  1 05:58:07 2015 +0000
URL:        http://sources.gentoo.org/gitweb/?p=proj/catalyst.git;a=commit;h=f98b415f

Initial creation of fileops.py

Migrate to using ensure_dirs()

---
 catalyst/base/clearbase.py        | 10 ++++-----
 catalyst/base/stagebase.py        | 32 +++++++++++-----------------
 catalyst/fileops.py               | 45 +++++++++++++++++++++++++++++++++++++++
 catalyst/targets/grp.py           |  5 ++---
 catalyst/targets/livecd_stage1.py |  4 ++--
 catalyst/targets/livecd_stage2.py | 13 ++++-------
 catalyst/targets/netboot2.py      |  6 +++---
 catalyst/targets/snapshot.py      | 10 ++++-----
 catalyst/targets/stage1.py        | 10 ++++-----
 9 files changed, 81 insertions(+), 54 deletions(-)

diff --git a/catalyst/base/clearbase.py b/catalyst/base/clearbase.py
index 8519acc..585d4f2 100644
--- a/catalyst/base/clearbase.py
+++ b/catalyst/base/clearbase.py
@@ -5,7 +5,7 @@ from stat import ST_UID, ST_GID, ST_MODE
 
 
 from catalyst.support import cmd, countdown
-
+from catalyst.fileops import ensure_dirs
 
 class ClearBase(object):
 	"""
@@ -35,7 +35,7 @@ class ClearBase(object):
 						+myemp)
 				#cmd("rm -rf "+myemp, "Could not remove existing file: "+myemp,env-self.env)
 				shutil.rmtree(myemp)
-				os.makedirs(myemp,0755)
+				ensure_dirs(myemp, 0755)
 				os.chown(myemp,mystat[ST_UID],mystat[ST_GID])
 				os.chmod(myemp,mystat[ST_MODE])
 
@@ -54,7 +54,7 @@ class ClearBase(object):
 			if os.uname()[0] == "FreeBSD":
 				os.system("chflags -R noschg "+myemp)
 			shutil.rmtree(myemp)
-			os.makedirs(myemp,0755)
+			ensure_dirs(myemp, mode=0755)
 			os.chown(myemp,mystat[ST_UID],mystat[ST_GID])
 			os.chmod(myemp,mystat[ST_MODE])
 
@@ -73,7 +73,7 @@ class ClearBase(object):
 				mystat=os.stat(myemp)
 				#cmd("rm -rf "+myemp, "Could not remove existing file: "+myemp,env=self.env)
 				shutil.rmtree(myemp)
-				os.makedirs(myemp,0755)
+				ensure_dirs(myemp, mode=0755)
 				os.chown(myemp,mystat[ST_UID],mystat[ST_GID])
 				os.chmod(myemp,mystat[ST_MODE])
 
@@ -92,7 +92,7 @@ class ClearBase(object):
 				mystat=os.stat(myemp)
 				#cmd("rm -rf "+myemp, "Could not remove existing file: "+myemp,env=self.env)
 				shutil.rmtree(myemp)
-				os.makedirs(myemp,0755)
+				ensure_dirs(myemp, mode=0755)
 				os.chown(myemp,mystat[ST_UID],mystat[ST_GID])
 				os.chmod(myemp,mystat[ST_MODE])
 

diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py
index 3056ce2..40c7362 100644
--- a/catalyst/base/stagebase.py
+++ b/catalyst/base/stagebase.py
@@ -18,6 +18,7 @@ from catalyst.base.targetbase import TargetBase
 from catalyst.base.clearbase import ClearBase
 from catalyst.base.genbase import GenBase
 from catalyst.lock import LockDir
+from catalyst.fileops import ensure_dirs, pjoin
 
 
 class StageBase(TargetBase, ClearBase, GenBase):
@@ -339,8 +340,7 @@ class StageBase(TargetBase, ClearBase, GenBase):
 #					+self.settings["target_path"],env=self.env)
 			touch(setup_target_path_resume)
 
-			if not os.path.exists(self.settings["storedir"]+"/builds/"):
-				os.makedirs(self.settings["storedir"]+"/builds/")
+			ensure_dirs(self.settings["storedir"] + "/builds/")
 
 	def set_fsscript(self):
 		if self.settings["spec_prefix"]+"/fsscript" in self.settings:
@@ -486,8 +486,7 @@ class StageBase(TargetBase, ClearBase, GenBase):
 			))
 		if "autoresume" in self.settings["options"]:
 			print "The autoresume path is " + self.settings["autoresume_path"]
-		if not os.path.exists(self.settings["autoresume_path"]):
-			os.makedirs(self.settings["autoresume_path"],0755)
+		ensure_dirs(self.settings["autoresume_path"], mode=0755)
 
 	def set_controller_file(self):
 		self.settings["controller_file"]=normpath(self.settings["sharedir"]+\
@@ -769,19 +768,15 @@ class StageBase(TargetBase, ClearBase, GenBase):
 				self.clear_autoresume()
 				self.clear_chroot()
 
-			if not os.path.exists(self.settings["chroot_path"]):
-				os.makedirs(self.settings["chroot_path"])
+			ensure_dirs(self.settings["chroot_path"])
 
-			if not os.path.exists(self.settings["chroot_path"]+"/tmp"):
-				os.makedirs(self.settings["chroot_path"]+"/tmp",1777)
+			ensure_dirs(self.settings["chroot_path"]+"/tmp",mode=1777)
 
 			if "pkgcache" in self.settings["options"]:
-				if not os.path.exists(self.settings["pkgcache_path"]):
-					os.makedirs(self.settings["pkgcache_path"],0755)
+				ensure_dirs(self.settings["pkgcache_path"],mode=0755)
 
 			if "kerncache" in self.settings["options"]:
-				if not os.path.exists(self.settings["kerncache_path"]):
-					os.makedirs(self.settings["kerncache_path"],0755)
+				ensure_dirs(self.settings["kerncache_path"],mode=0755)
 
 			print display_msg
 			cmd(unpack_cmd,error_msg,env=self.env)
@@ -849,8 +844,7 @@ class StageBase(TargetBase, ClearBase, GenBase):
 				print cleanup_msg
 				cleanup_cmd="rm -rf "+destdir
 				cmd(cleanup_cmd,cleanup_errmsg,env=self.env)
-			if not os.path.exists(destdir):
-				os.makedirs(destdir,0755)
+			ensure_dirs(destdir,mode=0755)
 
 			print "Unpacking portage tree (This can take a long time) ..."
 			cmd(unpack_cmd,unpack_errmsg,env=self.env)
@@ -935,12 +929,11 @@ class StageBase(TargetBase, ClearBase, GenBase):
 		for x in self.mounts:
 			#print "bind(); x =", x
 			target = normpath(self.settings["chroot_path"] + self.target_mounts[x])
-			if not os.path.exists(target):
-				os.makedirs(target, 0755)
+			ensure_dirs(target, mode=0755)
 
 			if not os.path.exists(self.mountmap[x]):
 				if self.mountmap[x] not in ["tmpfs", "shmfs"]:
-					os.makedirs(self.mountmap[x], 0755)
+					ensure_dirs(self.mountmap[x], mode=0755)
 
 			src=self.mountmap[x]
 			#print "bind(); src =", src
@@ -1210,7 +1203,7 @@ class StageBase(TargetBase, ClearBase, GenBase):
 					"""
 					mystat=os.stat(myemp)
 					shutil.rmtree(myemp)
-					os.makedirs(myemp,0755)
+					ensure_dirs(myemp, mode=0755)
 					os.chown(myemp,mystat[ST_UID],mystat[ST_GID])
 					os.chmod(myemp,mystat[ST_MODE])
 			touch(empty_resume)
@@ -1266,8 +1259,7 @@ class StageBase(TargetBase, ClearBase, GenBase):
 			mypath=string.join(mypath[:-1],"/")
 
 			""" Now make sure path exists """
-			if not os.path.exists(mypath):
-				os.makedirs(mypath)
+			ensure_dirs(mypath)
 
 			print "Creating stage tarball..."
 

diff --git a/catalyst/fileops.py b/catalyst/fileops.py
new file mode 100644
index 0000000..009bf1c
--- /dev/null
+++ b/catalyst/fileops.py
@@ -0,0 +1,45 @@
+#!/usr/bin/python
+
+# Maintained in full by:
+# Catalyst Team <catalyst@g.o>
+# Release Engineering Team <releng@g.o>
+
+'''fileops.py
+
+Performs file operations such as pack/unpack,
+ensuring directories exist,... imports snakeoils osutils
+functions for use throughout catalyst.
+'''
+
+from snakeoil.osutils import (ensure_dirs as snakeoil_ensure_dirs, normpath,
+	pjoin, listdir_files)
+from catalyst.support import CatalystError
+
+
+def ensure_dirs(path, gid=-1, uid=-1, mode=0777, minimal=True,
+		failback=None, fatal=False):
+	'''Wrapper to snakeoil.osutil's ensure_dirs()
+	This additionally allows for failures to run
+	cleanup or other code and/or raise fatal errors.
+
+	:param path: directory to ensure exists on disk
+	:param gid: a valid GID to set any created directories to
+	:param uid: a valid UID to set any created directories to
+	:param mode: permissions to set any created directories to
+	:param minimal: boolean controlling whether or not the specified mode
+		must be enforced, or is the minimal permissions necessary.  For example,
+		if mode=0755, minimal=True, and a directory exists with mode 0707,
+		this will restore the missing group perms resulting in 757.
+	:param failback: function to run in the event of a failed attemp
+		to create the directory.
+	:return: True if the directory could be created/ensured to have those
+		permissions, False if not.
+	'''
+	succeeded = snakeoil_ensure_dirs(path, gid=-1, uid=-1, mode=0777, minimal=True)
+	if not succeeded:
+		if failback:
+			failback()
+		if fatal:
+			raise CatalystError(
+				"Failed to create directory: %s" % path, print_traceback=True)
+	return succeeded

diff --git a/catalyst/targets/grp.py b/catalyst/targets/grp.py
index 93d30fc..a17f9bd 100644
--- a/catalyst/targets/grp.py
+++ b/catalyst/targets/grp.py
@@ -10,7 +10,7 @@ import glob
 
 from catalyst.support import (CatalystError, normpath,
 	touch, cmd, list_bashify)
-
+from catalyst.fileops import ensure_dirs
 from catalyst.base.stagebase import StageBase
 
 
@@ -51,8 +51,7 @@ class grp(StageBase):
 			#if os.path.isdir(self.settings["target_path"]):
 				#cmd("rm -rf "+self.settings["target_path"],
 				#"Could not remove existing directory: "+self.settings["target_path"],env=self.env)
-			if not os.path.exists(self.settings["target_path"]):
-				os.makedirs(self.settings["target_path"])
+			ensure_dirs(self.settings["target_path"])
 
 			touch(self.settings["autoresume_path"]+"setup_target_path")
 

diff --git a/catalyst/targets/livecd_stage1.py b/catalyst/targets/livecd_stage1.py
index 3b19c3f..668960b 100644
--- a/catalyst/targets/livecd_stage1.py
+++ b/catalyst/targets/livecd_stage1.py
@@ -10,6 +10,7 @@ import string
 
 from catalyst.support import (normpath,
 	touch, cmd)
+from catalyst.fileops import ensure_dirs
 
 from catalyst.base.stagebase import StageBase
 
@@ -43,8 +44,7 @@ class livecd_stage1(StageBase):
 					"Could not remove existing directory: "+self.settings["target_path"],env=self.env)
 				touch(self.settings["autoresume_path"]+"setup_target_path")
 
-			if not os.path.exists(self.settings["target_path"]):
-				os.makedirs(self.settings["target_path"])
+			ensure_dirs(self.settings["target_path"])
 
 	def set_spec_prefix(self):
 		self.settings["spec_prefix"]="livecd"

diff --git a/catalyst/targets/livecd_stage2.py b/catalyst/targets/livecd_stage2.py
index b1cf6b2..94da670 100644
--- a/catalyst/targets/livecd_stage2.py
+++ b/catalyst/targets/livecd_stage2.py
@@ -7,6 +7,7 @@ import os
 
 from catalyst.support import (normpath, file_locate, CatalystError, cmd,
 	read_from_clst, touch)
+from catalyst.fileops import ensure_dirs
 from catalyst.base.stagebase import StageBase
 
 
@@ -62,8 +63,7 @@ class livecd_stage2(StageBase):
 				cmd("rm -rf "+self.settings["target_path"],
 				"Could not remove existing directory: "+self.settings["target_path"],env=self.env)
 				touch(self.settings["autoresume_path"]+"setup_target_path")
-			if not os.path.exists(self.settings["target_path"]):
-				os.makedirs(self.settings["target_path"])
+			ensure_dirs(self.settings["target_path"])
 
 	def run_local(self):
 		# what modules do we want to blacklist?
@@ -116,15 +116,10 @@ class livecd_stage2(StageBase):
 				self.clear_chroot()
 				#self.dir_setup()
 
-			if not os.path.exists(self.settings["chroot_path"]):
-				os.makedirs(self.settings["chroot_path"])
-
-			if not os.path.exists(self.settings["chroot_path"]+"/tmp"):
-				os.makedirs(self.settings["chroot_path"]+"/tmp",1777)
+			ensure_dirs(self.settings["chroot_path"]+"/tmp", mode=1777)
 
 			if "pkgcache" in self.settings["options"]:
-				if not os.path.exists(self.settings["pkgcache_path"]):
-					os.makedirs(self.settings["pkgcache_path"],0755)
+				ensure_dirs(self.settings["pkgcache_path"], mode=0755)
 
 			if not display_msg:
 				raise CatalystError("Could not find appropriate source.\n"

diff --git a/catalyst/targets/netboot2.py b/catalyst/targets/netboot2.py
index e9e2625..7edfc1f 100644
--- a/catalyst/targets/netboot2.py
+++ b/catalyst/targets/netboot2.py
@@ -10,6 +10,7 @@ from stat import ST_UID, ST_GID, ST_MODE
 
 from catalyst.support import (CatalystError, normpath,
 	touch, cmd, list_bashify)
+from catalyst.fileops import ensure_dirs
 
 from catalyst.base.stagebase import StageBase
 
@@ -62,8 +63,7 @@ class netboot2(StageBase):
 					"Could not remove existing file: "+self.settings["target_path"],env=self.env)
 				touch(self.settings["autoresume_path"]+"setup_target_path")
 
-		if not os.path.exists(self.settings["storedir"]+"/builds/"):
-			os.makedirs(self.settings["storedir"]+"/builds/")
+		ensure_dirs(self.settings["storedir"]+"/builds/")
 
 	def copy_files_to_image(self):
 		# copies specific files from the buildroot to merge_path
@@ -158,7 +158,7 @@ class netboot2(StageBase):
 					# the proper perms and ownership
 					mystat=os.stat(myemp)
 					shutil.rmtree(myemp)
-					os.makedirs(myemp,0755)
+					ensure_dirs(myemp, mode=0755)
 					os.chown(myemp,mystat[ST_UID],mystat[ST_GID])
 					os.chmod(myemp,mystat[ST_MODE])
 		touch(self.settings["autoresume_path"]+"empty")

diff --git a/catalyst/targets/snapshot.py b/catalyst/targets/snapshot.py
index 6c2396e..1887110 100644
--- a/catalyst/targets/snapshot.py
+++ b/catalyst/targets/snapshot.py
@@ -10,7 +10,7 @@ from stat import ST_UID, ST_GID, ST_MODE
 from catalyst.support import normpath, cmd
 from catalyst.base.targetbase import TargetBase
 from catalyst.base.genbase import GenBase
-
+from catalyst.fileops import ensure_dirs
 class snapshot(TargetBase, GenBase):
 	"""
 	Builder class for snapshots.
@@ -31,8 +31,7 @@ class snapshot(TargetBase, GenBase):
 
 	def setup(self):
 		x=normpath(self.settings["storedir"]+"/snapshots")
-		if not os.path.exists(x):
-			os.makedirs(x)
+		ensure_dirs(x)
 
 	def mount_safety_check(self):
 		pass
@@ -50,8 +49,7 @@ class snapshot(TargetBase, GenBase):
 			" from "+self.settings["portdir"]+"..."
 
 		mytmp=self.settings["tmp_path"]
-		if not os.path.exists(mytmp):
-			os.makedirs(mytmp)
+		ensure_dirs(mytmp)
 
 		cmd("rsync -a --delete --exclude /packages/ --exclude /distfiles/ " +
 			"--exclude /local/ --exclude CVS/ --exclude .svn --filter=H_**/files/digest-* " +
@@ -88,6 +86,6 @@ class snapshot(TargetBase, GenBase):
 			if os.uname()[0] == "FreeBSD":
 				os.system("chflags -R noschg "+myemp)
 			shutil.rmtree(myemp)
-			os.makedirs(myemp,0755)
+			ensure_dirs(myemp, mode=0755)
 			os.chown(myemp,mystat[ST_UID],mystat[ST_GID])
 			os.chmod(myemp,mystat[ST_MODE])

diff --git a/catalyst/targets/stage1.py b/catalyst/targets/stage1.py
index e72cbcc..3d3b9e9 100644
--- a/catalyst/targets/stage1.py
+++ b/catalyst/targets/stage1.py
@@ -6,6 +6,7 @@ stage1 target
 import os
 
 from catalyst.support import normpath, list_to_string
+from catalyst.fileops import ensure_dirs
 from catalyst.base.stagebase import StageBase
 
 
@@ -64,10 +65,8 @@ class stage1(StageBase):
 			# baselayout no longer creates the .keep files in proc and dev for FreeBSD as it
 			# would create them too late...we need them earlier before bind mounting filesystems
 			# since proc and dev are not writeable, so...create them here
-			if not os.path.exists(self.settings["stage_path"]+"/proc"):
-				os.makedirs(self.settings["stage_path"]+"/proc")
-			if not os.path.exists(self.settings["stage_path"]+"/dev"):
-				os.makedirs(self.settings["stage_path"]+"/dev")
+			ensure_dirs(self.settings["stage_path"]+"/proc")
+			ensure_dirs(self.settings["stage_path"]+"/dev")
 			if not os.path.isfile(self.settings["stage_path"]+"/proc/.keep"):
 				try:
 					proc_keepfile = open(self.settings["stage_path"]+"/proc/.keep","w")
@@ -87,8 +86,7 @@ class stage1(StageBase):
 
 	def set_mounts(self):
 		# stage_path/proc probably doesn't exist yet, so create it
-		if not os.path.exists(self.settings["stage_path"]+"/proc"):
-			os.makedirs(self.settings["stage_path"]+"/proc")
+		ensure_dirs(self.settings["stage_path"]+"/proc")
 
 		# alter the mount mappings to bind mount proc onto it
 		self.mounts.append("stage1root/proc")