[gentoo-catalyst] [PATCH 2/8] Initial separation and creation of contents.py

[gentoo-catalyst] [PATCH 2/8] Initial separation and creation of contents.py

[Brian Dolbec]

---
 catalyst/contents.py                     | 87 ++++++++++++++++++++++++++++++++
 catalyst/main.py                         |  8 ++-
 catalyst/support.py                      | 52 -------------------
 catalyst/targets/generic_stage_target.py |  3 +-
 4 files changed, 96 insertions(+), 54 deletions(-)
 create mode 100644 catalyst/contents.py

diff --git a/catalyst/contents.py b/catalyst/contents.py
new file mode 100644
index 0000000..79ef9a6
--- /dev/null
+++ b/catalyst/contents.py
@@ -0,0 +1,87 @@
+
+from collections import namedtuple
+from subprocess import Popen, PIPE
+
+from support import CatalystError, warn
+
+
+# use ContentsMap.fields for the value legend
+# Key:[function, cmd]
+CONTENTS_DEFINITIONS = {
+	# 'find' is disabled because it requires the source path, which is not
+	# always available
+	#"find"		:["calc_contents","find %(path)s"],
+	"tar_tv":["calc_contents","tar tvf %(file)s"],
+	"tar_tvz":["calc_contents","tar tvzf %(file)s"],
+	"tar_tvj":["calc_contents","tar -I lbzip2 -tvf %(file)s"],
+	"isoinfo_l":["calc_contents","isoinfo -l -i %(file)s"],
+	# isoinfo_f should be a last resort only
+	"isoinfo_f":["calc_contents","isoinfo -f -i %(file)s"],
+}
+
+
+class ContentsMap(object):
+	'''Class to encompass all known commands to list
+	the contents of an archive'''
+
+
+	fields = ['func', 'cmd']
+
+
+	def __init__(self, defs=None):
+		'''Class init
+
+		@param defs: dictionary of Key:[function, cmd]
+		'''
+		if defs is None:
+			defs = {}
+		#self.contents = {}
+		self.contents_map = {}
+
+		# create the archive type namedtuple classes
+		for name in list(defs):
+			#obj = self.contents[name] = namedtuple(name, self.fields)
+			obj = namedtuple(name, self.fields)
+			obj.__slots__ = ()
+			self.contents_map[name] = obj._make(defs[name])
+		del obj
+
+
+	def generate_contents(self, file_, getter="auto", verbose=False):
+		try:
+			archive = getter
+			if archive == 'auto' and file_.endswith('.iso'):
+				archive = 'isoinfo_l'
+			if (archive in ['tar_tv','auto']):
+				if file_.endswith('.tgz') or file_.endswith('.tar.gz'):
+					archive = 'tar_tvz'
+				elif file_.endswith('.tbz2') or file_.endswith('.tar.bz2'):
+					archive = 'tar_tvj'
+				elif file_.endswith('.tar'):
+					archive = 'tar_tv'
+
+			if archive == 'auto':
+				warn('File %r has unknown type for automatic detection.'
+					% (file_, ))
+				return None
+			else:
+				getter = archive
+				func = getattr(self, '_%s_' % self.contents_map[getter].func)
+				return func(file_, self.contents_map[getter].cmd, verbose)
+		except:
+			raise CatalystError,\
+				"Error generating contents, is appropriate utility " +\
+				"(%s) installed on your system?" \
+				% (self.contents_map[getter].cmd)
+
+
+	@staticmethod
+	def _calc_contents_(file_, cmd, verbose):
+		_cmd = (cmd % {'file': file_ }).split()
+		proc = Popen(_cmd, stdout=PIPE, stderr=PIPE)
+		results = proc.communicate()
+		result = "\n".join(results)
+		if verbose:
+			print result
+		return result
+
diff --git a/catalyst/main.py b/catalyst/main.py
index 7bcf2cb..4146bca 100644
--- a/catalyst/main.py
+++ b/catalyst/main.py
@@ -25,6 +25,7 @@ from catalyst.support import (required_build_targets,
 	valid_build_targets, CatalystError, find_binary, LockInUse)
 
 from hash_utils import HashMap, HASH_DEFINITIONS
+from contents import ContentsMap, CONTENTS_DEFINITIONS
 
 
 
@@ -184,7 +185,8 @@ def parse_config(myconfig):
 	if "digests" in myconf:
 		conf_values["digests"]=myconf["digests"]
 	if "contents" in myconf:
-		conf_values["contents"]=myconf["contents"]
+		# replace '-' with '_' (for compatibility with existing configs)
+		conf_values["contents"] = myconf["contents"].replace("-", '_')
 
 	if "envscript" in myconf:
 		print "Envscript support enabled."
@@ -348,6 +350,10 @@ def main():
 	# import configuration file and import our main module using those settings
 	parse_config(myconfig)
 
+	# initialize our contents generator
+	contents_map = ContentsMap(CONTENTS_DEFINITIONS)
+	conf_values["contents_map"] = contents_map
+
 	# initialze our hash and contents generators
 	hash_map = HashMap(HASH_DEFINITIONS)
 	conf_values["hash_map"] = hash_map
diff --git a/catalyst/support.py b/catalyst/support.py
index 308d9c0..e25394e 100644
--- a/catalyst/support.py
+++ b/catalyst/support.py
@@ -62,58 +62,6 @@ def hexify(str):
 	return r
 # hexify()
 
-def generate_contents(file,contents_function="auto",verbose=False):
-	try:
-		_ = contents_function
-		if _ == 'auto' and file.endswith('.iso'):
-			_ = 'isoinfo-l'
-		if (_ in ['tar-tv','auto']):
-			if file.endswith('.tgz') or file.endswith('.tar.gz'):
-				_ = 'tar-tvz'
-			elif file.endswith('.tbz2') or file.endswith('.tar.bz2'):
-				_ = 'tar-tvj'
-			elif file.endswith('.tar'):
-				_ = 'tar-tv'
-
-		if _ == 'auto':
-			warn('File %r has unknown type for automatic detection.' % (file, ))
-			return None
-		else:
-			contents_function = _
-			_ = contents_map[contents_function]
-			return _[0](file,_[1],verbose)
-	except:
-		raise CatalystError,\
-			"Error generating contents, is appropriate utility (%s) installed on your system?" \
-			% (contents_function, )
-
-def calc_contents(file,cmd,verbose):
-	args={ 'file': file }
-	cmd=cmd % dict(args)
-	a=os.popen(cmd)
-	mylines=a.readlines()
-	a.close()
-	result="".join(mylines)
-	if verbose:
-		print result
-	return result
-
-# This has map must be defined after the function calc_content
-# It is possible to call different functions from this but they must be defined
-# before hash_map
-# Key,function,cmd
-contents_map={
-	# 'find' is disabled because it requires the source path, which is not
-	# always available
-	#"find"		:[calc_contents,"find %(path)s"],
-	"tar-tv":[calc_contents,"tar tvf %(file)s"],
-	"tar-tvz":[calc_contents,"tar tvzf %(file)s"],
-	"tar-tvj":[calc_contents,"tar -I lbzip2 -tvf %(file)s"],
-	"isoinfo-l":[calc_contents,"isoinfo -l -i %(file)s"],
-	# isoinfo-f should be a last resort only
-	"isoinfo-f":[calc_contents,"isoinfo -f -i %(file)s"],
-}
-
 
 def read_from_clst(file):
 	line = ''
diff --git a/catalyst/targets/generic_stage_target.py b/catalyst/targets/generic_stage_target.py
index b6a6200..de4842c 100644
--- a/catalyst/targets/generic_stage_target.py
+++ b/catalyst/targets/generic_stage_target.py
@@ -1703,6 +1703,7 @@ class generic_stage_target(generic_target):
 		if os.path.exists(file+".CONTENTS"):
 			os.remove(file+".CONTENTS")
 		if "contents" in self.settings:
+			contents_map = self.settings["contents_map"]
 			if os.path.exists(file):
 				myf=open(file+".CONTENTS","w")
 				keys={}
@@ -1711,7 +1712,7 @@ class generic_stage_target(generic_target):
 					array=keys.keys()
 					array.sort()
 				for j in array:
-					contents=generate_contents(file,contents_function=j,\
+					contents = contents_map.generate_contents(file, j,
 						verbose="VERBOSE" in self.settings)
 					if contents:
 						myf.write(contents)
-- 
1.8.5.3

[gentoo-catalyst] [PATCH 3/8] main.py: print the output of an ImportError to help in debugging.

[Brian Dolbec]

---
 catalyst/main.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/catalyst/main.py b/catalyst/main.py
index 4146bca..bba3cba 100644
--- a/catalyst/main.py
+++ b/catalyst/main.py
@@ -227,9 +227,10 @@ def import_modules():
 				raise CatalystError,"Can't find " + x + ".py plugin in " + \
 					module_dir
 
-	except ImportError:
+	except ImportError as e:
 		print "!!! catalyst: Python modules not found in "+\
 			module_dir + "; exiting."
+		print e
 		sys.exit(1)
 
 	return targetmap
-- 
1.8.5.3

[gentoo-catalyst] [PATCH 4/8] Fix undefined variable: RLIMIT_NOFILE

[Brian Dolbec]

It was not imported from resource, it was also not used correctly.
---
 catalyst/support.py | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/catalyst/support.py b/catalyst/support.py
index e25394e..5abf614 100644
--- a/catalyst/support.py
+++ b/catalyst/support.py
@@ -6,14 +6,16 @@ selinux_capable = False
 #fakeroot_capable = False
 BASH_BINARY             = "/bin/bash"
 
+# set it to 0 for the soft limit, 1 for the hard limit
+DESIRED_RLIMIT = 0
 try:
-        import resource
-        max_fd_limit=resource.getrlimit(RLIMIT_NOFILE)
+	import resource
+	max_fd_limit=resource.getrlimit(resource.RLIMIT_NOFILE)[DESIRED_RLIMIT]
 except SystemExit, e:
-        raise
+	raise
 except:
-        # hokay, no resource module.
-        max_fd_limit=256
+	# hokay, no resource module.
+	max_fd_limit=256
 
 # pids this process knows of.
 spawned_pids = []
-- 
1.8.5.3

[gentoo-catalyst] [PATCH 5/8] Creation of a new defaults.py

[Brian Dolbec]

catalyst/support.py: Move some defaults to a new defaults file.
---
 catalyst/defaults.py | 23 +++++++++++++++++++++++
 catalyst/main.py     |  5 ++---
 catalyst/support.py  | 35 +++--------------------------------
 3 files changed, 28 insertions(+), 35 deletions(-)
 create mode 100644 catalyst/defaults.py

diff --git a/catalyst/defaults.py b/catalyst/defaults.py
new file mode 100644
index 0000000..b1dbda4
--- /dev/null
+++ b/catalyst/defaults.py
@@ -0,0 +1,23 @@
+
+
+# these should never be touched
+required_build_targets = ["generic_target", "generic_stage_target"]
+
+# new build types should be added here
+valid_build_targets = ["stage1_target", "stage2_target", "stage3_target",
+	"stage4_target", "grp_target", "livecd_stage1_target", "livecd_stage2_target",
+	"embedded_target", "tinderbox_target", "snapshot_target", "netboot_target",
+	"netboot2_target"
+	]
+
+required_config_file_values = ["storedir", "sharedir", "distdir", "portdir"]
+
+valid_config_file_values = required_config_file_values[:]
+valid_config_file_values.extend(["PKGCACHE", "KERNCACHE", "CCACHE", "DISTCC",
+	"ICECREAM", "ENVSCRIPT", "AUTORESUME", "FETCH", "CLEAR_AUTORESUME",
+	"options", "DEBUG", "VERBOSE", "PURGE", "PURGEONLY", "SNAPCACHE",
+	"snapshot_cache", "hash_function", "digests", "contents", "SEEDCACHE"
+	])
+
+verbosity = 1
+
diff --git a/catalyst/main.py b/catalyst/main.py
index bba3cba..e2ef976 100644
--- a/catalyst/main.py
+++ b/catalyst/main.py
@@ -21,9 +21,8 @@ sys.path.append(__selfpath__ + "/modules")
 from . import __version__
 import catalyst.config
 import catalyst.util
-from catalyst.support import (required_build_targets,
-	valid_build_targets, CatalystError, find_binary, LockInUse)
-
+from catalyst.support import CatalystError, find_binary, LockInUse
+from catalyst.defaults import required_build_targets, valid_build_targets
 from hash_utils import HashMap, HASH_DEFINITIONS
 from contents import ContentsMap, CONTENTS_DEFINITIONS
 
diff --git a/catalyst/support.py b/catalyst/support.py
index 5abf614..4fe4603 100644
--- a/catalyst/support.py
+++ b/catalyst/support.py
@@ -1,6 +1,9 @@
 
 import sys,string,os,types,re,signal,traceback,time
 #import md5,sha
+
+from catalyst.defaults import verbosity, valid_config_file_values
+
 selinux_capable = False
 #userpriv_capable = (os.getuid() == 0)
 #fakeroot_capable = False
@@ -80,38 +83,6 @@ def read_from_clst(file):
 	return myline
 # read_from_clst
 
-# these should never be touched
-required_build_targets=["generic_target","generic_stage_target"]
-
-# new build types should be added here
-valid_build_targets=["stage1_target","stage2_target","stage3_target","stage4_target","grp_target",
-			"livecd_stage1_target","livecd_stage2_target","embedded_target",
-			"tinderbox_target","snapshot_target","netboot_target","netboot2_target"]
-
-required_config_file_values=["storedir","sharedir","distdir","portdir"]
-valid_config_file_values=required_config_file_values[:]
-valid_config_file_values.append("PKGCACHE")
-valid_config_file_values.append("KERNCACHE")
-valid_config_file_values.append("CCACHE")
-valid_config_file_values.append("DISTCC")
-valid_config_file_values.append("ICECREAM")
-valid_config_file_values.append("ENVSCRIPT")
-valid_config_file_values.append("AUTORESUME")
-valid_config_file_values.append("FETCH")
-valid_config_file_values.append("CLEAR_AUTORESUME")
-valid_config_file_values.append("options")
-valid_config_file_values.append("DEBUG")
-valid_config_file_values.append("VERBOSE")
-valid_config_file_values.append("PURGE")
-valid_config_file_values.append("PURGEONLY")
-valid_config_file_values.append("SNAPCACHE")
-valid_config_file_values.append("snapshot_cache")
-valid_config_file_values.append("hash_function")
-valid_config_file_values.append("digests")
-valid_config_file_values.append("contents")
-valid_config_file_values.append("SEEDCACHE")
-
-verbosity=1
 
 def list_bashify(mylist):
 	if type(mylist)==types.StringType:
-- 
1.8.5.3

[gentoo-catalyst] [PATCH 6/8] Move generic_stage_targets.py constants to defaults.py

[Brian Dolbec]

Rename the source and target mounts defaults to remove
the double plural.
---
 catalyst/defaults.py                     | 27 +++++++++++++++++++++++++
 catalyst/targets/generic_stage_target.py | 34 ++++----------------------------
 2 files changed, 31 insertions(+), 30 deletions(-)

diff --git a/catalyst/defaults.py b/catalyst/defaults.py
index b1dbda4..748d1dd 100644
--- a/catalyst/defaults.py
+++ b/catalyst/defaults.py
@@ -21,3 +21,30 @@ valid_config_file_values.extend(["PKGCACHE", "KERNCACHE", "CCACHE", "DISTCC",
 
 verbosity = 1
 
+PORT_LOGDIR_CLEAN = \
+	'find "${PORT_LOGDIR}" -type f ! -name "summary.log*" -mtime +30 -delete'
+
+TARGET_MOUNT_DEFAULTS = {
+	"ccache": "/var/tmp/ccache",
+	"dev": "/dev",
+	"devpts": "/dev/pts",
+	"distdir": "/usr/portage/distfiles",
+	"icecream": "/usr/lib/icecc/bin",
+	"kerncache": "/tmp/kerncache",
+	"packagedir": "/usr/portage/packages",
+	"portdir": "/usr/portage",
+	"port_tmpdir": "/var/tmp/portage",
+	"port_logdir": "/var/log/portage",
+	"proc": "/proc",
+	"shm": "/dev/shm",
+	}
+
+SOURCE_MOUNT_DEFAULTS = {
+	"dev": "/dev",
+	"devpts": "/dev/pts",
+	"distdir": "/usr/portage/distfiles",
+	"portdir": "/usr/portage",
+	"port_tmpdir": "tmpfs",
+	"proc": "/proc",
+	"shm": "shmfs",
+	}
diff --git a/catalyst/targets/generic_stage_target.py b/catalyst/targets/generic_stage_target.py
index de4842c..05c61e8 100644
--- a/catalyst/targets/generic_stage_target.py
+++ b/catalyst/targets/generic_stage_target.py
@@ -4,34 +4,8 @@ from generic_target import *
 from stat import *
 from catalyst.lock import LockDir
 
-
-PORT_LOGDIR_CLEAN = \
-	'find "${PORT_LOGDIR}" -type f ! -name "summary.log*" -mtime +30 -delete'
-
-TARGET_MOUNTS_DEFAULTS = {
-	"ccache": "/var/tmp/ccache",
-	"dev": "/dev",
-	"devpts": "/dev/pts",
-	"distdir": "/usr/portage/distfiles",
-	"icecream": "/usr/lib/icecc/bin",
-	"kerncache": "/tmp/kerncache",
-	"packagedir": "/usr/portage/packages",
-	"portdir": "/usr/portage",
-	"port_tmpdir": "/var/tmp/portage",
-	"port_logdir": "/var/log/portage",
-	"proc": "/proc",
-	"shm": "/dev/shm",
-	}
-
-SOURCE_MOUNTS_DEFAULTS = {
-	"dev": "/dev",
-	"devpts": "/dev/pts",
-	"distdir": "/usr/portage/distfiles",
-	"portdir": "/usr/portage",
-	"port_tmpdir": "tmpfs",
-	"proc": "/proc",
-	"shm": "shmfs",
-	}
+from catalyst.defaults import (SOURCE_MOUNT_DEFAULTS, TARGET_MOUNT_DEFAULTS,
+	PORT_LOGDIR_CLEAN)
 
 # for convienience
 pjoin = os.path.join
@@ -208,11 +182,11 @@ class generic_stage_target(generic_target):
 
 		""" Setup our mount points """
 		# initialize our target mounts.
-		self.target_mounts = TARGET_MOUNTS_DEFAULTS.copy()
+		self.target_mounts = TARGET_MOUNT_DEFAULTS.copy()
 
 		self.mounts = ["proc", "dev", "portdir", "distdir", "port_tmpdir"]
 		# initialize our source mounts
-		self.mountmap = SOURCE_MOUNTS_DEFAULTS.copy()
+		self.mountmap = SOURCE_MOUNT_DEFAULTS.copy()
 		# update them from settings
 		self.mountmap["distdir"] = self.settings["distdir"]
 		if "SNAPCACHE" not in self.settings:
-- 
1.8.5.3

[gentoo-catalyst] [PATCH 7/8] Move confdefaults out of main.py

[Brian Dolbec]

---
 catalyst/defaults.py | 17 +++++++++++++++++
 catalyst/main.py     | 20 +++-----------------
 2 files changed, 20 insertions(+), 17 deletions(-)

diff --git a/catalyst/defaults.py b/catalyst/defaults.py
index 748d1dd..b83e4f5 100644
--- a/catalyst/defaults.py
+++ b/catalyst/defaults.py
@@ -21,6 +21,22 @@ valid_config_file_values.extend(["PKGCACHE", "KERNCACHE", "CCACHE", "DISTCC",
 
 verbosity = 1
 
+confdefaults={
+	"distdir": "/usr/portage/distfiles",
+	"hash_function": "crc32",
+	"icecream": "/var/cache/icecream",
+	"local_overlay": "/usr/local/portage",
+	"options": "",
+	"packagedir": "/usr/portage/packages",
+	"portdir": "/usr/portage",
+	"port_tmpdir": "/var/tmp/portage",
+	"repo_name": "portage",
+	"sharedir": "/usr/lib/catalyst",
+	"snapshot_cache": "/var/tmp/catalyst/snapshot_cache",
+	"snapshot_name": "portage-",
+	"storedir": "/var/tmp/catalyst",
+	}
+
 PORT_LOGDIR_CLEAN = \
 	'find "${PORT_LOGDIR}" -type f ! -name "summary.log*" -mtime +30 -delete'
 
@@ -48,3 +64,4 @@ SOURCE_MOUNT_DEFAULTS = {
 	"proc": "/proc",
 	"shm": "shmfs",
 	}
+
diff --git a/catalyst/main.py b/catalyst/main.py
index e2ef976..5748d31 100644
--- a/catalyst/main.py
+++ b/catalyst/main.py
@@ -22,7 +22,8 @@ from . import __version__
 import catalyst.config
 import catalyst.util
 from catalyst.support import CatalystError, find_binary, LockInUse
-from catalyst.defaults import required_build_targets, valid_build_targets
+from catalyst.defaults import (required_build_targets, valid_build_targets,
+	confdefaults)
 from hash_utils import HashMap, HASH_DEFINITIONS
 from contents import ContentsMap, CONTENTS_DEFINITIONS
 
@@ -70,21 +71,6 @@ def parse_config(myconfig):
 	myconf={}
 	config_file=""
 
-	confdefaults = {
-		"distdir": "/usr/portage/distfiles",
-		"hash_function": "crc32",
-		"icecream": "/var/cache/icecream",
-		"local_overlay": "/usr/local/portage",
-		"options": "",
-		"packagedir": "/usr/portage/packages",
-		"portdir": "/usr/portage",
-		"repo_name": "portage",
-		"sharedir": "/usr/share/catalyst",
-		"snapshot_name": "portage-",
-		"snapshot_cache": "/var/tmp/catalyst/snapshot_cache",
-		"storedir": "/var/tmp/catalyst",
-		}
-
 	# first, try the one passed (presumably from the cmdline)
 	if myconfig:
 		if os.path.exists(myconfig):
@@ -117,7 +103,7 @@ def parse_config(myconfig):
 		sys.exit(1)
 
 	# now, load up the values into conf_values so that we can use them
-	for x in confdefaults.keys():
+	for x in list(confdefaults):
 		if x in myconf:
 			print "Setting",x,"to config file value \""+myconf[x]+"\""
 			conf_values[x]=myconf[x]
-- 
1.8.5.3

[gentoo-catalyst] [PATCH 8/8] livecdfs-update: No tmpfs on /lib/firmware

[Brian Dolbec]

From: Douglas Freed <dwfreed@mtu.edu>

As of a while ago, we no longer ship a separate firmware tarball, so we
don't need a tmpfs for /lib/firmware anymore, so let's stop mounting
one.  Fixes firmware issues with current minimal install ISOs.
---
 targets/support/livecdfs-update.sh | 1 -
 1 file changed, 1 deletion(-)

diff --git a/targets/support/livecdfs-update.sh b/targets/support/livecdfs-update.sh
index 2b41f9d..b017baf 100755
--- a/targets/support/livecdfs-update.sh
+++ b/targets/support/livecdfs-update.sh
@@ -101,7 +101,6 @@ echo "####################################################" >> /etc/fstab
 
 # fstab tweaks
 echo "tmpfs	/					tmpfs	defaults	0 0" >> /etc/fstab
-echo "tmpfs	/lib/firmware			tmpfs	defaults	0 0" >> /etc/fstab
 echo "tmpfs	/usr/portage			tmpfs	defaults	0 0" >> /etc/fstab
 # If /usr/lib/X11/xkb/compiled then make it tmpfs
 if [ -d /usr/lib/X11/xkb/compiled ]
-- 
1.8.5.3