[gentoo-commits] proj/catalyst:pending commit in: etc/, catalyst/targets/, catalyst/base/, examples/, catalyst/

[Brian Dolbec]

commit:     f6c3ede7ec389b9b9d39ede27ee1165a8bcc417b
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Sun May 24 07:36:49 2015 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Tue Sep  1 05:54:11 2015 +0000
URL:        https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=f6c3ede7

apply new compress code, normalize paths

 catalyst/base/stagebase.py           | 171 +++++++++++++++++++----------------
 catalyst/defaults.py                 |   4 +
 catalyst/main.py                     |   8 ++
 catalyst/targets/grp.py              |   2 +-
 catalyst/targets/livecd_stage1.py    |   2 +-
 catalyst/targets/livecd_stage2.py    |   6 +-
 catalyst/targets/netboot.py          |   8 +-
 catalyst/targets/netboot2.py         |   2 +-
 catalyst/targets/stage2.py           |   8 +-
 etc/catalyst.conf                    |  11 ++-
 examples/generic_stage_template.spec |  32 ++++++-
 11 files changed, 159 insertions(+), 95 deletions(-)

diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py
index 6adc3b9..2e175f1 100644
--- a/catalyst/base/stagebase.py
+++ b/catalyst/base/stagebase.py
@@ -10,6 +10,8 @@ from stat import ST_UID, ST_GID, ST_MODE
 # for convienience
 pjoin = os.path.join
 
+from decomp.compress import CompressMap
+
 from catalyst.defaults import (SOURCE_MOUNT_DEFAULTS, TARGET_MOUNT_DEFAULTS,
 	PORT_LOGDIR_CLEAN)
 from catalyst.support import (CatalystError, msg, file_locate, normpath,
@@ -259,6 +261,14 @@ class StageBase(TargetBase, ClearBase, GenBase):
 			self.env["PORT_LOGDIR"] = self.settings["port_logdir"]
 			self.env["PORT_LOGDIR_CLEAN"] = PORT_LOGDIR_CLEAN
 
+		# Initialize our (de)compressor's)
+		self.decompressor = CompressMap(self.settings["decompress_definitions"],
+			env=self.env,
+			search_order=self.settings["decompressor_search_order"])
+
+		# save resources, it is not always needed
+		self.compressor = None
+
 	def override_cbuild(self):
 		if "CBUILD" in self.makeconf:
 			self.settings["CBUILD"]=self.makeconf["CBUILD"]
@@ -319,11 +329,11 @@ class StageBase(TargetBase, ClearBase, GenBase):
 					normpath(self.settings["kerncache_path"])
 		else:
 			self.settings["kerncache_path"]=normpath(self.settings["storedir"]+\
-				"/kerncache/"+self.settings["target_subpath"]+"/")
+				"/kerncache/"+self.settings["target_subpath"])
 
 	def set_target_path(self):
 		self.settings["target_path"]=normpath(self.settings["storedir"]+\
-			"/builds/"+self.settings["target_subpath"].rstrip('/')+".tar.bz2")
+			"/builds/"+self.settings["target_subpath"])
 		if "autoresume" in self.settings["options"]\
 			and self.resume.is_enabled("setup_target_path"):
 			print \
@@ -407,7 +417,7 @@ class StageBase(TargetBase, ClearBase, GenBase):
 				"/tmp/" + self.settings["source_subpath"] + "/")
 		else:
 			self.settings["source_path"]=normpath(self.settings["storedir"]+\
-				"/builds/"+self.settings["source_subpath"].rstrip('/')+".tar.bz2")
+				"/builds/"+self.settings["source_subpath"])
 			if os.path.isfile(self.settings["source_path"]):
 				# XXX: Is this even necessary if the previous check passes?
 				if os.path.exists(self.settings["source_path"]):
@@ -422,7 +432,7 @@ class StageBase(TargetBase, ClearBase, GenBase):
 			print "\tseedcache in the options of catalyst.conf the source path"
 			print "\twill then be "+\
 				normpath(self.settings["storedir"]+"/builds/"+\
-				self.settings["source_subpath"].rstrip('/')+".tar.bz2\n")
+					self.settings["source_subpath"]+"\n")
 
 	def set_dest_path(self):
 		if "root_path" in self.settings:
@@ -449,7 +459,7 @@ class StageBase(TargetBase, ClearBase, GenBase):
 		else:
 			self.settings["snapshot_path"]=normpath(self.settings["storedir"]+\
 				"/snapshots/" + self.settings["snapshot_name"] +
-				self.settings["snapshot"].rstrip('/')+".tar.bz2")
+				self.settings["snapshot"])
 
 			if os.path.exists(self.settings["snapshot_path"]):
 				self.settings["snapshot_path_hash"] = \
@@ -459,6 +469,9 @@ class StageBase(TargetBase, ClearBase, GenBase):
 						verbose = False)
 
 	def set_snapcache_path(self):
+		self.settings["snapshot_cache_path"]=\
+			normpath(pjoin(self.settings["snapshot_cache"],
+				self.settings["snapshot"]))
 		if "snapcache" in self.settings["options"]:
 			self.settings["snapshot_cache_path"] = \
 				normpath(pjoin(self.settings["snapshot_cache"],
@@ -677,47 +690,32 @@ class StageBase(TargetBase, ClearBase, GenBase):
 
 		clst_unpack_hash = self.resume.get("unpack")
 
+		unpack_info = {
+			'source': self.settings["source_path"],
+			"destination": self.settings["chroot_path"],
+			'mode': None,
+			'auto-ext': False,
+			}
+
+		display_msg="\nStarting %(mode)s from %(source)s\nto "+\
+			"%(destination)s (This may take some time) ...\n"
+
+		error_msg="'%(mode)s' extraction of %(source)s to %(destination)s failed."
+
 		if "seedcache" in self.settings["options"]:
-			if os.path.isdir(self.settings["source_path"]):
+			if os.path.isdir(unpack_info["source"]):
 				""" SEEDCACHE Is a directory, use rsync """
-				unpack_cmd="rsync -a --delete "+self.settings["source_path"]+\
-					" "+self.settings["chroot_path"]
-				display_msg="\nStarting rsync from "+\
-					self.settings["source_path"]+"\nto "+\
-					self.settings["chroot_path"]+\
-					" (This may take some time) ...\n"
-				error_msg="Rsync of "+self.settings["source_path"]+" to "+\
-					self.settings["chroot_path"]+" failed."
+				unpack_info['mode'] = "rsync"
 			else:
 				""" SEEDCACHE is a not a directory, try untar'ing """
 				print "Referenced SEEDCACHE does not appear to be a directory, trying to untar..."
-				display_msg="\nStarting tar extract from "+\
-					self.settings["source_path"]+"\nto "+\
-					self.settings["chroot_path"]+\
-						" (This may take some time) ...\n"
-				if "bz2" == self.settings["chroot_path"][-3:]:
-					unpack_cmd="tar --xattrs --xattrs-include=security.capability --xattrs-include=user.pax.flags --selinux -I lbzip2 -xpf "+self.settings["source_path"]+" -C "+\
-						self.settings["chroot_path"]
-				else:
-					unpack_cmd="tar --xattrs --xattrs-include=security.capability --xattrs-include=user.pax.flags --selinux -I lbzip2 -xpf "+self.settings["source_path"]+" -C "+\
-						self.settings["chroot_path"]
-				error_msg="Tarball extraction of "+\
-					self.settings["source_path"]+" to "+\
-					self.settings["chroot_path"]+" failed."
+				unpack_info['mode'] = self.decompressor.determine_mode(
+						self.settings["source_path"])
 		else:
 			""" No SEEDCACHE, use tar """
-			display_msg="\nStarting tar extract from "+\
-				self.settings["source_path"]+"\nto "+\
-				self.settings["chroot_path"]+\
-				" (This may take some time) ...\n"
-			if "bz2" == self.settings["chroot_path"][-3:]:
-				unpack_cmd="tar --xattrs --xattrs-include=security.capability --xattrs-include=user.pax.flags --selinux -I lbzip2 -xpf "+self.settings["source_path"]+" -C "+\
-					self.settings["chroot_path"]
-			else:
-				unpack_cmd="tar --xattrs --xattrs-include=security.capability --xattrs-include=user.pax.flags --selinux -I lbzip2 -xpf "+self.settings["source_path"]+" -C "+\
-					self.settings["chroot_path"]
-			error_msg="Tarball extraction of "+self.settings["source_path"]+\
-				" to "+self.settings["chroot_path"]+" failed."
+			unpack_info['mode'] = self.decompressor.determine_mode(
+					unpack_info["source"])
+		# endif "seedcache"
 
 		if "autoresume" in self.settings["options"]:
 			if os.path.isdir(self.settings["source_path"]) \
@@ -785,8 +783,11 @@ class StageBase(TargetBase, ClearBase, GenBase):
 			if "kerncache" in self.settings["options"]:
 				ensure_dirs(self.settings["kerncache_path"],mode=0755)
 
-			print display_msg
-			cmd(unpack_cmd,error_msg,env=self.env)
+			print display_msg %(unpack_info)
+
+			# now run the decompressor
+			if not self.decompressor.extract(unpack_info):
+				print error_msg %(unpack_info)
 
 			if "source_path_hash" in self.settings:
 				self.resume.enable("unpack",
@@ -800,16 +801,26 @@ class StageBase(TargetBase, ClearBase, GenBase):
 		unpack=True
 		snapshot_hash = self.resume.get("unpack_portage")
 
+		unpack_errmsg="Error unpacking snapshot using mode %(mode)s"
+
+		unpack_info = {
+			'source': self.settings["snapshot_path"],
+			'destination': self.settings["snapshot_cache_path"],
+			'mode': None,
+			'auto-ext': False,
+			}
+
+		target_portdir = normpath(self.settings["chroot_path"] +
+			self.settings["repo_basedir"] + "/" + self.settings["repo_name"])
+		print self.settings["chroot_path"]
+		print "unpack(), target_portdir = " + target_portdir
 		if "snapcache" in self.settings["options"]:
 			snapshot_cache_hash=\
-				read_from_clst(self.settings["snapshot_cache_path"] + "/" +
-					"catalyst-hash")
-			destdir=self.settings["snapshot_cache_path"]
-			if "bz2" == self.settings["chroot_path"][-3:]:
-				unpack_cmd="tar --xattrs --xattrs-include=security.capability --xattrs-include=user.pax.flags --selinux -I lbzip2 -xpf "+self.settings["snapshot_path"]+" -C "+destdir
-			else:
-				unpack_cmd="tar --xattrs --xattrs-include=security.capability --xattrs-include=user.pax.flags --selinux -xpf "+self.settings["snapshot_path"]+" -C "+destdir
-			unpack_errmsg="Error unpacking snapshot"
+				read_from_clst(self.settings["snapshot_cache_path"]+\
+				"catalyst-hash")
+			unpack_info['mode'] = self.decompressor.determine_mode(
+				unpack_info['source'])
+
 			cleanup_msg="Cleaning up invalid snapshot cache at \n\t"+\
 				self.settings["snapshot_cache_path"]+\
 				" (This can take a long time)..."
@@ -820,21 +831,16 @@ class StageBase(TargetBase, ClearBase, GenBase):
 				print "Valid snapshot cache, skipping unpack of portage tree..."
 				unpack=False
 		else:
-			destdir = normpath(self.settings["chroot_path"] + self.settings["portdir"])
 			cleanup_errmsg="Error removing existing snapshot directory."
 			cleanup_msg=\
 				"Cleaning up existing portage tree (This can take a long time)..."
-			if "bz2" == self.settings["chroot_path"][-3:]:
-				unpack_cmd="tar --xattrs --xattrs-include=security.capability --xattrs-include=user.pax.flags --selinux -I lbzip2 -xpf "+self.settings["snapshot_path"]+" -C "+\
-					self.settings["chroot_path"]+"/usr"
-			else:
-				unpack_cmd="tar --xattrs --xattrs-include=security.capability --xattrs-include=user.pax.flags --selinux -xpf "+self.settings["snapshot_path"]+" -C "+\
-					self.settings["chroot_path"]+"/usr"
-			unpack_errmsg="Error unpacking snapshot"
+			unpack_info['destination'] = normpath(
+				self.settings["chroot_path"] + self.settings["repo_basedir"])
+			unpack_info['mode'] = self.decompressor.determine_mode(
+				unpack_info['source'])
 
 			if "autoresume" in self.settings["options"] \
-				and os.path.exists(self.settings["chroot_path"] +
-					self.settings["portdir"]) \
+				and os.path.exists(target_portdir) \
 				and self.resume.is_enabled("unpack_portage") \
 				and self.settings["snapshot_path_hash"] == snapshot_hash:
 					print \
@@ -844,14 +850,16 @@ class StageBase(TargetBase, ClearBase, GenBase):
 		if unpack:
 			if "snapcache" in self.settings["options"]:
 				self.snapshot_lock_object.write_lock()
-			if os.path.exists(destdir):
+			if os.path.exists(target_portdir):
 				print cleanup_msg
-				cleanup_cmd="rm -rf "+destdir
+				cleanup_cmd = "rm -rf " + target_portdir
+				print "unpack() cleanup_cmd = " + cleanup_cmd
 				cmd(cleanup_cmd,cleanup_errmsg,env=self.env)
-			ensure_dirs(destdir,mode=0755)
+			ensure_dirs(target_portdir, mode=0755)
 
 			print "Unpacking portage tree (This can take a long time) ..."
-			cmd(unpack_cmd,unpack_errmsg,env=self.env)
+			if not self.decompressor.extract(unpack_info):
+				print unpack_errmsg %(unpack_info)
 
 			if "snapcache" in self.settings["options"]:
 				myf=open(self.settings["snapshot_cache_path"] +
@@ -1243,6 +1251,12 @@ class StageBase(TargetBase, ClearBase, GenBase):
 				raise CatalystError("Build failed, could not execute preclean")
 
 	def capture(self):
+		# initialize it here so it doesn't use
+		# resources if it is not needed
+		if not self.compressor:
+			self.compressor = CompressMap(self.settings["compress_definitions"],
+				env=self.env, default_mode=self.settings['compression_mode'])
+
 		if "autoresume" in self.settings["options"] \
 			and self.resume.is_enabled("capture"):
 			print "Resume point detected, skipping capture operation..."
@@ -1254,16 +1268,24 @@ class StageBase(TargetBase, ClearBase, GenBase):
 			""" Now make sure path exists """
 			ensure_dirs(mypath)
 
-			print "Creating stage tarball..."
-
-			cmd("tar --xattrs --xattrs-include=security.capability --xattrs-include=user.pax.flags --selinux -I lbzip2 -cpf "+self.settings["target_path"]+" -C "+\
-				self.settings["stage_path"]+" .",\
-				"Couldn't create stage tarball",env=self.env)
+			pack_info = self.compressor.create_infodict(
+				source=".",
+				basedir=self.settings["stage_path"],
+				filename=self.settings["target_path"],
+				mode=self.settings["compression_mode"],
+				auto_extension=True
+				)
+			target_filename = ".".join([self.settings["target_path"],
+				self.compressor.extension(pack_info['mode'])])
 
-			self.gen_contents_file(self.settings["target_path"])
-			self.gen_digest_file(self.settings["target_path"])
+			print "Creating stage tarball..."
 
-			self.resume.enable("capture")
+			if self.compressor.compress(pack_info):
+				self.gen_contents_file(target_filename)
+				self.gen_digest_file(target_filename)
+				self.resume.enable("capture")
+			else:
+				print "Couldn't create stage tarball:", target_filename
 
 	def run_local(self):
 		if "autoresume" in self.settings["options"] \
@@ -1365,12 +1387,7 @@ class StageBase(TargetBase, ClearBase, GenBase):
 		for x in self.settings["action_sequence"]:
 			print "--- Running action sequence: "+x
 			sys.stdout.flush()
-			try:
-				apply(getattr(self,x))
-			except Exception as e:
-				print "--- Exeption running action sequence:" + x
-				self.mount_safety_check()
-				raise e
+			apply(getattr(self,x))
 
 
 	def unmerge(self):

diff --git a/catalyst/defaults.py b/catalyst/defaults.py
index 975e55b..5ce2858 100644
--- a/catalyst/defaults.py
+++ b/catalyst/defaults.py
@@ -16,12 +16,15 @@ valid_config_file_values = required_config_file_values[:]
 valid_config_file_values.extend([ "distcc", "envscript",
 	"options", "DEBUG", "VERBOSE",
 	"snapshot_cache", "hash_function", "digests", "contents"
+	"compression_mode", "decompressor_search_order",
 	])
 
 verbosity = 1
 
 confdefaults={
 	"archdir": "%(PythonDir)s/arch",
+	"compression_mode": 'lbzip2',
+	"decompressor_search_order": 'lbzip2 bzip2 tar pixz xz gzip squashfs',
 	"distdir": "/usr/portage/distfiles",
 	"hash_function": "crc32",
 	"icecream": "/var/cache/icecream",
@@ -33,6 +36,7 @@ confdefaults={
 	"portdir": "/usr/portage",
 	"port_tmpdir": "/var/tmp/portage",
 	"PythonDir": "./catalyst",
+	"repo_basedir": "/usr",
 	"repo_name": "portage",
 	"sharedir": "/usr/share/catalyst",
 	"shdir": "/usr/share/catalyst/targets/",

diff --git a/catalyst/main.py b/catalyst/main.py
index 3ab4166..0ba81b9 100644
--- a/catalyst/main.py
+++ b/catalyst/main.py
@@ -13,6 +13,7 @@ import os.path
 
 __selfpath__ = os.path.abspath(os.path.dirname(__file__))
 
+from decomp.compress import COMPRESS_DEFINITIONS, DECOMPRESS_DEFINITIONS
 
 import catalyst.config
 import catalyst.util
@@ -103,6 +104,8 @@ def parse_config(myconfig):
 		if x in myconf:
 			if x == 'options':
 				conf_values[x] = set(myconf[x].split())
+			elif x in ["decompressor_search_order"]:
+				conf_values[x] = myconf[x].split()
 			else:
 				conf_values[x]=myconf[x]
 		else:
@@ -281,6 +284,11 @@ def main():
 	hash_map = HashMap(HASH_DEFINITIONS)
 	conf_values["hash_map"] = hash_map
 
+	# initialize our (de)compression definitions
+	conf_values['decompress_definitions'] = DECOMPRESS_DEFINITIONS
+	conf_values['compress_definitions'] = COMPRESS_DEFINITIONS
+	# TODO add capability to config/spec new definitions
+
 	# Start checking that digests are valid now that hash_map is initialized
 	if "digests" in conf_values:
 		for i in conf_values["digests"].split():

diff --git a/catalyst/targets/grp.py b/catalyst/targets/grp.py
index 14cffe0..5a10fff 100644
--- a/catalyst/targets/grp.py
+++ b/catalyst/targets/grp.py
@@ -42,7 +42,7 @@ class grp(StageBase):
 		StageBase.__init__(self,spec,addlargs)
 
 	def set_target_path(self):
-		self.settings["target_path"]=normpath(self.settings["storedir"]+"/builds/"+self.settings["target_subpath"]+"/")
+		self.settings["target_path"]=normpath(self.settings["storedir"]+"/builds/"+self.settings["target_subpath"])
 		if "autoresume" in self.settings["options"] \
 			and self.resume.is_enabled("setup_target_path"):
 			print "Resume point detected, skipping target path setup operation..."

diff --git a/catalyst/targets/livecd_stage1.py b/catalyst/targets/livecd_stage1.py
index 8402e7a..d5645e8 100644
--- a/catalyst/targets/livecd_stage1.py
+++ b/catalyst/targets/livecd_stage1.py
@@ -33,7 +33,7 @@ class livecd_stage1(StageBase):
 					"unbind", "clean","clear_autoresume"]
 
 	def set_target_path(self):
-		self.settings["target_path"]=normpath(self.settings["storedir"]+"/builds/"+self.settings["target_subpath"]+'/')
+		self.settings["target_path"]=normpath(self.settings["storedir"]+"/builds/"+self.settings["target_subpath"])
 		if "autoresume" in self.settings["options"] \
 			and self.resume.is_enabled("setup_target_path"):
 				print "Resume point detected, skipping target path setup operation..."

diff --git a/catalyst/targets/livecd_stage2.py b/catalyst/targets/livecd_stage2.py
index 9f0733e..e5ca074 100644
--- a/catalyst/targets/livecd_stage2.py
+++ b/catalyst/targets/livecd_stage2.py
@@ -37,13 +37,13 @@ class livecd_stage2(StageBase):
 		file_locate(self.settings, ["cdtar","controller_file"])
 
 	def set_source_path(self):
-		self.settings["source_path"]=normpath(self.settings["storedir"]+"/builds/"+self.settings["source_subpath"].rstrip('/')+".tar.bz2")
+		self.settings["source_path"]=normpath(self.settings["storedir"]+"/builds/"+self.settings["source_subpath"])
 		if os.path.isfile(self.settings["source_path"]):
 			self.settings["source_path_hash"] = \
 				self.settings["hash_map"].generate_hash(
 					self.settings["source_path"])
 		else:
-			self.settings["source_path"]=normpath(self.settings["storedir"]+"/tmp/"+self.settings["source_subpath"]+'/')
+			self.settings["source_path"]=normpath(self.settings["storedir"]+"/tmp/"+self.settings["source_subpath"])
 		if not os.path.exists(self.settings["source_path"]):
 			raise CatalystError("Source Path: " +
 				self.settings["source_path"] + " does not exist.",
@@ -53,7 +53,7 @@ class livecd_stage2(StageBase):
 		self.settings["spec_prefix"]="livecd"
 
 	def set_target_path(self):
-		self.settings["target_path"]=normpath(self.settings["storedir"]+"/builds/"+self.settings["target_subpath"]+"/")
+		self.settings["target_path"]=normpath(self.settings["storedir"]+"/builds/"+self.settings["target_subpath"])
 		if "autoresume" in self.settings["options"] \
 			and self.resume.is_enabled("setup_target_path"):
 				print "Resume point detected, skipping target path setup operation..."

diff --git a/catalyst/targets/netboot.py b/catalyst/targets/netboot.py
index de8dc85..c41ed59 100644
--- a/catalyst/targets/netboot.py
+++ b/catalyst/targets/netboot.py
@@ -95,10 +95,10 @@ class netboot(StageBase):
 
 		for x in loopy:
 			if "netboot/packages/"+x+"/files" in self.settings:
-			    if type(self.settings["netboot/packages/"+x+"/files"]) == types.ListType:
-				    myfiles.extend(self.settings["netboot/packages/"+x+"/files"])
-			    else:
-				    myfiles.append(self.settings["netboot/packages/"+x+"/files"])
+				if type(self.settings["netboot/packages/"+x+"/files"]) == types.ListType:
+					myfiles.extend(self.settings["netboot/packages/"+x+"/files"])
+				else:
+					myfiles.append(self.settings["netboot/packages/"+x+"/files"])
 
 		if "netboot/extra_files" in self.settings:
 			if type(self.settings["netboot/extra_files"]) == types.ListType:

diff --git a/catalyst/targets/netboot2.py b/catalyst/targets/netboot2.py
index b492e6a..3305a2a 100644
--- a/catalyst/targets/netboot2.py
+++ b/catalyst/targets/netboot2.py
@@ -52,7 +52,7 @@ class netboot2(StageBase):
 
 	def set_target_path(self):
 		self.settings["target_path"]=normpath(self.settings["storedir"]+"/builds/"+\
-			self.settings["target_subpath"]+"/")
+			self.settings["target_subpath"])
 		if "autoresume" in self.settings["options"] \
 			and self.resume.is_enabled("setup_target_path"):
 				print "Resume point detected, skipping target path setup operation..."

diff --git a/catalyst/targets/stage2.py b/catalyst/targets/stage2.py
index e54b5c9..40dc938 100644
--- a/catalyst/targets/stage2.py
+++ b/catalyst/targets/stage2.py
@@ -19,10 +19,10 @@ class stage2(StageBase):
 		StageBase.__init__(self,spec,addlargs)
 
 	def set_source_path(self):
-		if "seedcache" in self.settings["options"] and os.path.isdir(normpath(self.settings["storedir"]+"/tmp/"+self.settings["source_subpath"]+"/tmp/stage1root/")):
-			self.settings["source_path"]=normpath(self.settings["storedir"]+"/tmp/"+self.settings["source_subpath"]+"/tmp/stage1root/")
+		if "seedcache" in self.settings["options"] and os.path.isdir(normpath(self.settings["storedir"]+"/tmp/"+self.settings["source_subpath"]+"/tmp/stage1root")):
+			self.settings["source_path"]=normpath(self.settings["storedir"]+"/tmp/"+self.settings["source_subpath"]+"/tmp/stage1root")
 		else:
-			self.settings["source_path"]=normpath(self.settings["storedir"]+"/builds/"+self.settings["source_subpath"].rstrip('/')+".tar.bz2")
+			self.settings["source_path"]=normpath(self.settings["storedir"]+"/builds/"+self.settings["source_subpath"])
 			if os.path.isfile(self.settings["source_path"]):
 				if os.path.exists(self.settings["source_path"]):
 				# XXX: Is this even necessary if the previous check passes?
@@ -34,7 +34,7 @@ class stage2(StageBase):
 		print "Source path set to "+self.settings["source_path"]
 		if os.path.isdir(self.settings["source_path"]):
 			print "\tIf this is not desired, remove this directory or turn of seedcache in the options of catalyst.conf"
-			print "\tthe source path will then be "+normpath(self.settings["storedir"]+"/builds/"+self.settings["source_subpath"].rstrip('/')+".tar.bz2\n")
+			print "\tthe source path will then be "+normpath(self.settings["storedir"]+"/builds/"+self.settings["source_subpath"] + "\n")
 
 	# XXX: How do these override_foo() functions differ from the ones in
 	# StageBase and why aren't they in stage3_target?

diff --git a/etc/catalyst.conf b/etc/catalyst.conf
index 7f56671..aa5d815 100644
--- a/etc/catalyst.conf
+++ b/etc/catalyst.conf
@@ -72,9 +72,18 @@ hash_function="crc32"
 # (These options can be used together)
 options="autoresume bindist kerncache pkgcache seedcache snapcache"
 
-# portdir specifies the source portage tree used by the snapshot target.
+# Source portdir specifies the source portage tree used by the snapshot target.
 portdir="/usr/portage"
 
+# Target portdir setting.  It needs to be in 2 parts.
+# They will be used separately, then added together where needed.
+# eg:
+# repo_basedir="/var/lib/repos"
+# repo_name="gentoo"
+#
+repo_basedir="/usr"
+repo_name="portage"
+
 # sharedir specifies where all of the catalyst runtime executables
 # and other shared lib objects are.
 # Most users do not need to change this.

diff --git a/examples/generic_stage_template.spec b/examples/generic_stage_template.spec
index 45268ca..b43aebe 100644
--- a/examples/generic_stage_template.spec
+++ b/examples/generic_stage_template.spec
@@ -35,16 +35,42 @@ profile:
 
 # This specifies which snapshot to use for building this target.
 # example:
-# snapshot: 2006.1
+# snapshot: 2006.1.tar.bz2
 snapshot:
 
 # This specifies where the seed stage comes from for this target,  The path is
 # relative to $clst_sharedir/builds.  The rel_type is also used as a path prefix
-# for the seed.
+# for the seed.  It must be a complete filename including the extension.
 # example:
-# default/stage3-x86-2006.1
+# default/stage3-x86-2006.1.tar.bz2
 source_subpath:
 
+# The prefered compression/decompression command and algorithm to use
+# for creating the final stage tarballs.  For decompression the configured
+# method will be used if the file's extension is one of it's types listed.
+# Otherwise it will pick one that does.  Of course you must ensure to have
+# the correct package installed that supplies that command/algorithm type.
+# available options are:
+#	Name          Compressor Extension
+#	"rsync"     : None
+#	"lbzip2"    : tar.bz2
+#	"bzip2"     : tar.bz2
+#	"tar"       : tar
+#	"xz"        : tar.xz
+#	"pixz"      : tar.xz
+#	"gzip"      : tar.gz
+#	"squashfs"  : squashfs, sfs
+#
+compression_mode: lbzip2
+
+# The search order to use for determining the decompressor
+# to use on the source file.  i.e. check the extensions it is capable of
+# decompressing to find a match.  Use the list above for the modes available.
+#Note: rsync is a special case, do not include it below.
+#
+decompressor_search_order: lbzip2 bzip2 tar pixz xz gzip squashfs
+
+
 # These are the hosts used as distcc slaves when distcc is enabled in your
 # catalyst.conf.  It follows the same syntax as distcc-config --set-hosts and
 # is entirely optional.