[gentoo-portage-dev] [PATCH v2] Support different compressors for binary packages

[Manuel Rüger]

This patch allows to set the compressor for binary packages via a
BINPKG_COMPRESSION variable. BINPKG_COMPRESSION_ARGS allows to specify
command-line arguments for the chosen compressor.
---
 bin/misc-functions.sh                              |  6 ++-
 bin/quickpkg                                       | 62 ++++++++++++++++------
 man/make.conf.5                                    | 25 +++++++++
 pym/_emerge/BinpkgExtractorAsync.py                | 43 +++++++++++++--
 pym/portage/dbapi/bintree.py                       |  8 +--
 .../package/ebuild/_config/special_env_vars.py     |  2 +-
 pym/portage/package/ebuild/doebuild.py             | 34 ++++++++++--
 pym/portage/util/compression_probe.py              | 45 +++++++++++++---
 8 files changed, 186 insertions(+), 39 deletions(-)

diff --git a/bin/misc-functions.sh b/bin/misc-functions.sh
index 58755a1e1..079369313 100755
--- a/bin/misc-functions.sh
+++ b/bin/misc-functions.sh
@@ -453,7 +453,7 @@ __dyn_package() {
 	# Make sure $PWD is not ${D} so that we don't leave gmon.out files
 	# in there in case any tools were built with -pg in CFLAGS.
 
-	cd "${T}"
+	cd "${T}" || die
 
 	if [[ -n ${PKG_INSTALL_MASK} ]] ; then
 		PROOT=${T}/packaging/
@@ -478,8 +478,10 @@ __dyn_package() {
 	[ -z "${PORTAGE_BINPKG_TMPFILE}" ] && \
 		die "PORTAGE_BINPKG_TMPFILE is unset"
 	mkdir -p "${PORTAGE_BINPKG_TMPFILE%/*}" || die "mkdir failed"
+	[ -z "${PORTAGE_COMPRESSION_COMMAND}" ] && \
+        die "PORTAGE_COMPRESSION_COMMAND is unset"
 	tar $tar_options -cf - $PORTAGE_BINPKG_TAR_OPTS -C "${PROOT}" . | \
-		$PORTAGE_BZIP2_COMMAND -c > "$PORTAGE_BINPKG_TMPFILE"
+		$PORTAGE_COMPRESSION_COMMAND -c > "$PORTAGE_BINPKG_TMPFILE"
 	assert "failed to pack binary package: '$PORTAGE_BINPKG_TMPFILE'"
 	PYTHONPATH=${PORTAGE_PYTHONPATH:-${PORTAGE_PYM_PATH}} \
 		"${PORTAGE_PYTHON:-/usr/bin/python}" "$PORTAGE_BIN_PATH"/xpak-helper.py recompose \
diff --git a/bin/quickpkg b/bin/quickpkg
index 4f26ee912..750400592 100755
--- a/bin/quickpkg
+++ b/bin/quickpkg
@@ -8,6 +8,7 @@ import argparse
 import errno
 import math
 import signal
+import subprocess
 import sys
 import tarfile
 
@@ -22,11 +23,13 @@ from portage.dbapi.dep_expand import dep_expand
 from portage.dep import Atom, use_reduce
 from portage.exception import (AmbiguousPackageName, InvalidAtom, InvalidData,
 	InvalidDependString, PackageSetNotFound, PermissionDenied)
-from portage.util import ConfigProtect, ensure_dirs, shlex_split, _xattr
+from portage.util import ConfigProtect, ensure_dirs, shlex_split, varexpand, _xattr
 xattr = _xattr.xattr
 from portage.dbapi.vartree import dblink, tar_contents
 from portage.checksum import perform_md5
 from portage._sets import load_default_config, SETPREFIX
+from portage.process import find_binary
+from portage.util.compression_probe import _compressors
 
 def quickpkg_atom(options, infos, arg, eout):
 	settings = portage.settings
@@ -50,16 +53,16 @@ def quickpkg_atom(options, infos, arg, eout):
 			" ".join(e.args[0]))
 		del e
 		infos["missing"].append(arg)
-		return
+		return 1
 	except (InvalidAtom, InvalidData):
 		eout.eerror("Invalid atom: %s" % (arg,))
 		infos["missing"].append(arg)
-		return
+		return 1
 	if atom[:1] == '=' and arg[:1] != '=':
 		# dep_expand() allows missing '=' but it's really invalid
 		eout.eerror("Invalid atom: %s" % (arg,))
 		infos["missing"].append(arg)
-		return
+		return 1
 
 	matches = vardb.match(atom)
 	pkgs_for_arg = 0
@@ -108,16 +111,16 @@ def quickpkg_atom(options, infos, arg, eout):
 					in settings.features))
 				def protect(filename):
 					if not confprot.isprotected(filename):
-						return False
+						return 1
 					if include_unmodified_config:
 						file_data = contents[filename]
 						if file_data[0] == "obj":
 							orig_md5 = file_data[2].lower()
 							cur_md5 = perform_md5(filename, calc_prelink=1)
 							if orig_md5 == cur_md5:
-								return False
+								return 1
 					excluded_config_files.append(filename)
-					return True
+					return os.EX_OK
 			existing_metadata = dict(zip(fix_metadata_keys,
 				vardb.aux_get(cpv, fix_metadata_keys)))
 			category, pf = portage.catsplit(cpv)
@@ -134,12 +137,32 @@ def quickpkg_atom(options, infos, arg, eout):
 			binpkg_tmpfile = os.path.join(bintree.pkgdir,
 				cpv + ".tbz2." + str(os.getpid()))
 			ensure_dirs(os.path.dirname(binpkg_tmpfile))
-			# The tarfile module will write pax headers holding the
-			# xattrs only if PAX_FORMAT is specified here.
-			tar = tarfile.open(binpkg_tmpfile, "w:bz2",
-				format=tarfile.PAX_FORMAT if xattrs else tarfile.DEFAULT_FORMAT)
-			tar_contents(contents, root, tar, protect=protect, xattrs=xattrs)
-			tar.close()
+			binpkg_compression = settings.get("BINPKG_COMPRESSION", "bzip2")
+			try:
+				compression = _compressors[binpkg_compression]
+			except KeyError as e:
+				eout.eerror("Invalid or unsupported compression method: %s" % e.args[0])
+				return 1
+			try:
+				compression_binary = shlex_split(varexpand(compression["compress"], mydict=settings))[0]
+			except IndexError as e:
+				eout.eerror("Invalid or unsupported compression method: %s" % e.args[0])
+				return 1
+			if find_binary(compression_binary) is None:
+				missing_package = compression["package"]
+				eout.eerror("File compression unsupported %s. Missing package: %s" % (binpkg_compression, missing_package))
+				return 1
+			cmd = [varexpand(x, mydict=settings) for x in shlex_split(compression["compress"])]
+			# Filter empty elements that make Popen fail
+			cmd = [x for x in cmd if x != ""]
+			with open(binpkg_tmpfile, "wb") as fobj:
+				proc = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=fobj)
+				# The tarfile module will write pax headers holding the
+				# xattrs only if PAX_FORMAT is specified here.
+				with tarfile.open(mode="w|",format=tarfile.PAX_FORMAT if xattrs else tarfile.DEFAULT_FORMAT, fileobj=proc.stdin) as tar:
+					tar_contents(contents, root, tar, protect=protect, xattrs=xattrs)
+				proc.stdin.close()
+				proc.wait()
 			xpak.tbz2(binpkg_tmpfile).recompose_mem(xpdata)
 		finally:
 			if have_lock:
@@ -154,16 +177,20 @@ def quickpkg_atom(options, infos, arg, eout):
 			eout.eerror(str(e))
 			del e
 			eout.eerror("Failed to create package: '%s'" % binpkg_path)
+			return 1
 		else:
 			eout.eend(0)
 			infos["successes"].append((cpv, s.st_size))
 			infos["config_files_excluded"] += len(excluded_config_files)
 			for filename in excluded_config_files:
 				eout.ewarn("Excluded config: '%s'" % filename)
+			return os.EX_OK
 	if not pkgs_for_arg:
 		eout.eerror("Could not find anything " + \
 			"to match '%s'; skipping" % arg)
 		infos["missing"].append(arg)
+		return 1
+	return os.EX_OK
 
 def quickpkg_set(options, infos, arg, eout):
 	eroot = portage.settings['EROOT']
@@ -179,7 +206,7 @@ def quickpkg_set(options, infos, arg, eout):
 	if not set in sets:
 		eout.eerror("Package set not found: '%s'; skipping" % (arg,))
 		infos["missing"].append(arg)
-		return
+		return 1
 
 	try:
 		atoms = setconfig.getSetAtoms(set)
@@ -187,10 +214,11 @@ def quickpkg_set(options, infos, arg, eout):
 		eout.eerror("Failed to process package set '%s' because " % set +
 			"it contains the non-existent package set '%s'; skipping" % e)
 		infos["missing"].append(arg)
-		return
-
+		return 1
+	retval = os.EX_OK
 	for atom in atoms:
-		quickpkg_atom(options, infos, atom, eout)
+		retval |= quickpkg_atom(options, infos, atom, eout)
+	return retval
 
 
 def quickpkg_extended_atom(options, infos, atom, eout):
diff --git a/man/make.conf.5 b/man/make.conf.5
index aea189e4a..8e0d04967 100644
--- a/man/make.conf.5
+++ b/man/make.conf.5
@@ -110,6 +110,31 @@ ACCEPT_RESTRICT="*"
 ACCEPT_RESTRICT="* -bindist"
 .fi
 .TP
+\fBBINPKG_COMPRESSION\fR = \fI"compression"\fR
+This variable is used to determine the compression used for \fIbinary
+packages\fR. Supported settings and compression algorithms are: bzip2, gzip,
+lz4, lzip, lzop, xz, zstd.
+.br
+Defaults to "bzip2".
+.br
+.I Example:
+.nf
+# Set it to use lz4:
+BINPKG_COMPRESSION="lz4"
+.fi
+.TP
+\fBBINPKG_COMPRESSION_ARGS\fR = \fI"arguments for compression command"\fR
+This variable is used to add additional arguments to the compression command
+selected by \fBBINPKG_COMPRESSION\fR.
+.br
+Defaults to "".
+.br
+.I Example:
+.nf
+# Set it to use compression level 9:
+BINPKG_COMPRESSION_ARGS="-9"
+.fi
+.TP
 .B CBUILD
 This variable is passed by the \fIebuild scripts\fR to the \fIconfigure\fR
 as \fI\-\-build=${CBUILD}\fR only if it is defined.  Do not set this yourself
diff --git a/pym/_emerge/BinpkgExtractorAsync.py b/pym/_emerge/BinpkgExtractorAsync.py
index 0bf3c74c9..e85f4ecac 100644
--- a/pym/_emerge/BinpkgExtractorAsync.py
+++ b/pym/_emerge/BinpkgExtractorAsync.py
@@ -6,8 +6,15 @@ import logging
 from _emerge.SpawnProcess import SpawnProcess
 import portage
 from portage.localization import _
-from portage.util.compression_probe import (compression_probe,
-	_decompressors)
+from portage.util.compression_probe import (
+	compression_probe,
+	_compressors,
+)
+from portage.process import find_binary
+from portage.util import (
+	shlex_split,
+	varexpand,
+)
 import signal
 import subprocess
 
@@ -28,8 +35,11 @@ class BinpkgExtractorAsync(SpawnProcess):
 					tar_options.append(portage._shell_quote("--xattrs-exclude=%s" % x))
 				tar_options = " ".join(tar_options)
 
-		decomp_cmd = _decompressors.get(
-			compression_probe(self.pkg_path))
+		decomp = _compressors.get(compression_probe(self.pkg_path))
+		if decomp is not None:
+			decomp_cmd = decomp.get("decompress")
+		else:
+			decomp_cmd = None
 		if decomp_cmd is None:
 			self.scheduler.output("!!! %s\n" %
 				_("File compression header unrecognized: %s") %
@@ -39,6 +49,31 @@ class BinpkgExtractorAsync(SpawnProcess):
 			self._async_wait()
 			return
 
+		try:
+			decompression_binary = shlex_split(varexpand(decomp_cmd, mydict=self.env))[0]
+		except IndexError:
+			decompression_binary = ""
+
+		if find_binary(decompression_binary) is None:
+			# Try alternative command if it exists
+			if _compressors.get(compression_probe(self.pkg_path)).get("decompress_alt"):
+				decomp_cmd = _compressors.get(
+					compression_probe(self.pkg_path)).get("decompress_alt")
+			try:
+				decompression_binary = shlex_split(varexpand(decomp_cmd, mydict=self.env))[0]
+			except IndexError:
+				decompression_binary = ""
+
+			if find_binary(decompression_binary) is None:
+				missing_package = _compressors.get(compression_probe(self.pkg_path)).get("package")
+				self.scheduler.output("!!! %s\n" %
+					_("File compression unsupported %s.\n Command was: %s.\n Maybe missing package: %s") %
+					(self.pkg_path, varexpand(decomp_cmd, mydict=self.env), missing_package), log_path=self.logfile,
+					background=self.background, level=logging.ERROR)
+				self.returncode = 1
+				self._async_wait()
+				return
+
 		# Add -q to decomp_cmd opts, in order to avoid "trailing garbage
 		# after EOF ignored" warning messages due to xpak trailer.
 		# SIGPIPE handling (128 + SIGPIPE) should be compatible with
diff --git a/pym/portage/dbapi/bintree.py b/pym/portage/dbapi/bintree.py
index ca90ba8f9..c833968c2 100644
--- a/pym/portage/dbapi/bintree.py
+++ b/pym/portage/dbapi/bintree.py
@@ -141,7 +141,6 @@ class bindbapi(fakedbapi):
 				return [aux_cache.get(x, "") for x in wants]
 		mysplit = mycpv.split("/")
 		mylist = []
-		tbz2name = mysplit[1]+".tbz2"
 		if not self.bintree._remotepkgs or \
 			not self.bintree.isremote(mycpv):
 			try:
@@ -1448,9 +1447,10 @@ class binarytree(object):
 	@staticmethod
 	def _parse_build_id(filename):
 		build_id = -1
-		hyphen = filename.rfind("-", 0, -6)
+		suffixlen = len(".xpak")
+		hyphen = filename.rfind("-", 0, -(suffixlen + 1))
 		if hyphen != -1:
-			build_id = filename[hyphen+1:-5]
+			build_id = filename[hyphen+1:-suffixlen]
 		try:
 			build_id = long(build_id)
 		except ValueError:
@@ -1497,7 +1497,7 @@ class binarytree(object):
 		if self._remote_has_index:
 			rel_url = self._remotepkgs[instance_key].get("PATH")
 			if not rel_url:
-				rel_url = pkgname+".tbz2"
+				rel_url = pkgname + ".tbz2"
 			remote_base_uri = self._remotepkgs[instance_key]["BASE_URI"]
 			url = remote_base_uri.rstrip("/") + "/" + rel_url.lstrip("/")
 		else:
diff --git a/pym/portage/package/ebuild/_config/special_env_vars.py b/pym/portage/package/ebuild/_config/special_env_vars.py
index f7810e007..f9b29af93 100644
--- a/pym/portage/package/ebuild/_config/special_env_vars.py
+++ b/pym/portage/package/ebuild/_config/special_env_vars.py
@@ -56,7 +56,7 @@ environ_whitelist += [
 	"PORTAGE_BIN_PATH",
 	"PORTAGE_BUILDDIR", "PORTAGE_BUILD_GROUP", "PORTAGE_BUILD_USER",
 	"PORTAGE_BUNZIP2_COMMAND", "PORTAGE_BZIP2_COMMAND",
-	"PORTAGE_COLORMAP", "PORTAGE_COMPRESS",
+	"PORTAGE_COLORMAP", "PORTAGE_COMPRESS", "PORTAGE_COMPRESSION_COMMAND",
 	"PORTAGE_COMPRESS_EXCLUDE_SUFFIXES",
 	"PORTAGE_CONFIGROOT", "PORTAGE_DEBUG", "PORTAGE_DEPCACHEDIR",
 	"PORTAGE_DOHTML_UNWARNED_SKIPPED_EXTENSIONS",
diff --git a/pym/portage/package/ebuild/doebuild.py b/pym/portage/package/ebuild/doebuild.py
index e7db54bcf..3e75d1a51 100644
--- a/pym/portage/package/ebuild/doebuild.py
+++ b/pym/portage/package/ebuild/doebuild.py
@@ -68,11 +68,19 @@ from portage.exception import (DigestException, FileNotFound,
 from portage.localization import _
 from portage.output import colormap
 from portage.package.ebuild.prepare_build_dirs import prepare_build_dirs
-from portage.util import apply_recursive_permissions, \
-	apply_secpass_permissions, noiselimit, \
-	writemsg, writemsg_stdout, write_atomic
+from portage.process import find_binary
+from portage.util import ( apply_recursive_permissions,
+	apply_secpass_permissions,
+	noiselimit,
+	shlex_split,
+	varexpand,
+	writemsg,
+	writemsg_stdout,
+	write_atomic
+	)
 from portage.util.cpuinfo import get_cpu_count
 from portage.util.lafilefixer import rewrite_lafile
+from portage.util.compression_probe import _compressors
 from portage.util.socks5 import get_socks5_proxy
 from portage.versions import _pkgsplit
 from _emerge.BinpkgEnvExtractor import BinpkgEnvExtractor
@@ -518,6 +526,26 @@ def doebuild_environment(myebuild, mydo, myroot=None, settings=None,
 				mysettings["KV"] = ""
 			mysettings.backup_changes("KV")
 
+		binpkg_compression = mysettings.get("BINPKG_COMPRESSION", "bzip2")
+		try:
+			compression = _compressors[binpkg_compression]
+		except KeyError as e:
+			writemsg("Warning: Invalid or unsupported compression method: %s" % e.args[0])
+		else:
+			try:
+				compression_binary = shlex_split(varexpand(compression["compress"], mydict=settings))[0]
+			except IndexError as e:
+				writemsg("Warning: Invalid or unsupported compression method: %s" % e.args[0])
+			else:
+				if find_binary(compression_binary) is None:
+					missing_package = compression["package"]
+					writemsg("Warning: File compression unsupported %s. Missing package: %s" % (binpkg_compression, missing_package))
+				else:
+					cmd = [varexpand(x, mydict=settings) for x in shlex_split(compression["compress"])]
+					# Filter empty elements
+					cmd = [x for x in cmd if x != ""]
+					mysettings['PORTAGE_COMPRESSION_COMMAND'] = ' '.join(cmd)
+
 _doebuild_manifest_cache = None
 _doebuild_broken_ebuilds = set()
 _doebuild_broken_manifests = set()
diff --git a/pym/portage/util/compression_probe.py b/pym/portage/util/compression_probe.py
index 754621016..b15200044 100644
--- a/pym/portage/util/compression_probe.py
+++ b/pym/portage/util/compression_probe.py
@@ -11,14 +11,43 @@ if sys.hexversion >= 0x3000000:
 from portage import _encodings, _unicode_encode
 from portage.exception import FileNotFound, PermissionDenied
 
-_decompressors = {
-	"bzip2": "${PORTAGE_BUNZIP2_COMMAND:-${PORTAGE_BZIP2_COMMAND} -d}",
-	"gzip": "gzip -d",
-	"lz4": "lz4 -d",
-	"lzip": "lzip -d",
-	"lzop": "lzop -d",
-	"xz": "xz -d",
-	"zstd": "zstd -d",
+_compressors = {
+	"bzip2": {
+		"compress": "${PORTAGE_BZIP2_COMMAND} ${BINPKG_COMPRESSION_ARGS}",
+		"decompress": "${PORTAGE_BUNZIP2_COMMAND}",
+		"decompress_alt": "${PORTAGE_BZIP2_COMMAND} -d",
+		"package": "app-arch/bzip2",
+	},
+	"gzip": {
+		"compress": "gzip ${BINPKG_COMPRESSION_ARGS}",
+		"decompress": "gzip -d",
+		"package": "app-arch/gzip",
+	},
+	"lz4": {
+		"compress": "lz4 ${BINPKG_COMPRESSION_ARGS}",
+		"decompress": "lz4 -d",
+		"package": "app-arch/lz4",
+	},
+	"lzip": {
+		"compress": "lzip ${BINPKG_COMPRESSION_ARGS}",
+		"decompress": "lzip -d",
+		"package": "app-arch/lzip",
+	},
+	"lzop": {
+		"compress": "lzop ${BINPKG_COMPRESSION_ARGS}",
+		"decompress": "lzop -d",
+		"package": "app-arch/lzop",
+	},
+	"xz": {
+		"compress": "xz ${BINPKG_COMPRESSION_ARGS}",
+		"decompress": "xz -d",
+		"package": "app-arch/xz-utils",
+	},
+	"zstd": {
+		"compress": "zstd ${BINPKG_COMPRESSION_ARGS}",
+		"decompress": "zstd -d",
+		"package": "app-arch/zstd",
+	},
 }
 
 _compression_re = re.compile(b'^(' +
-- 
2.13.3