[gentoo-portage-dev] [PATCH] BinpkgExtractorAsync: xz and gzip decompression (142579)

[Zac Medico]

This adds support for using a binary package's compression header to
determine the compression type, providing forward-compatibility for
xz and gzip decompression. The file name extension is disregared, so
that it will be possible to use a compression-independent file naming
scheme in the future (see bug #150031 for discussion about proposed
file naming schemes).

Currently, only decompression is supported. It's useful to provide
forward-compatibility now, so that binhost clients will be prepared
to handle future binhost servers that use xz or gzip compression.

X-Gentoo-Bug: 142579
X-Gentoo-Bug-URL: https://bugs.gentoo.org/show_bug.cgi?id=142579
---
 pym/_emerge/BinpkgExtractorAsync.py   | 25 ++++++++++---
 pym/portage/util/compression_probe.py | 68 +++++++++++++++++++++++++++++++++++
 2 files changed, 89 insertions(+), 4 deletions(-)
 create mode 100644 pym/portage/util/compression_probe.py

diff --git a/pym/_emerge/BinpkgExtractorAsync.py b/pym/_emerge/BinpkgExtractorAsync.py
index be74c2f..6aaa448 100644
--- a/pym/_emerge/BinpkgExtractorAsync.py
+++ b/pym/_emerge/BinpkgExtractorAsync.py
@@ -1,8 +1,13 @@
 # Copyright 1999-2013 Gentoo Foundation
 # Distributed under the terms of the GNU General Public License v2
 
+import logging
+
 from _emerge.SpawnProcess import SpawnProcess
 import portage
+from portage.localization import _
+from portage.util.compression_probe import (compression_probe,
+	_decompressors)
 import signal
 import subprocess
 
@@ -20,19 +25,31 @@ class BinpkgExtractorAsync(SpawnProcess):
 			if b"--xattrs" in output:
 				tar_options = "--xattrs"
 
-		# Add -q to bzip2 opts, in order to avoid "trailing garbage after
-		# EOF ignored" warning messages due to xpak trailer.
+		decomp_cmd = _decompressors.get(
+			compression_probe(self.pkg_path))
+		if decomp_cmd is None:
+			self.scheduler.output("!!! %s\n" %
+				_("File compression header unrecognized: %s") %
+				self.pkg_path, log_path=self.logfile,
+				background=self.background, level=logging.ERROR)
+			self.returncode = 1
+			self._async_wait()
+			return
+
+		# Add -q to decomp_cmd opts, in order to avoid "trailing garbage
+		# after EOF ignored" warning messages due to xpak trailer.
 		# SIGPIPE handling (128 + SIGPIPE) should be compatible with
 		# assert_sigpipe_ok() that's used by the ebuild unpack() helper.
 		self.args = [self._shell_binary, "-c",
-			("${PORTAGE_BUNZIP2_COMMAND:-${PORTAGE_BZIP2_COMMAND} -d} -cq -- %s | tar -xp %s -C %s -f - ; " + \
+			("%s -cq -- %s | tar -xp %s -C %s -f - ; " + \
 			"p=(${PIPESTATUS[@]}) ; " + \
 			"if [[ ${p[0]} != 0 && ${p[0]} != %d ]] ; then " % (128 + signal.SIGPIPE) + \
 			"echo bzip2 failed with status ${p[0]} ; exit ${p[0]} ; fi ; " + \
 			"if [ ${p[1]} != 0 ] ; then " + \
 			"echo tar failed with status ${p[1]} ; exit ${p[1]} ; fi ; " + \
 			"exit 0 ;") % \
-			(portage._shell_quote(self.pkg_path),
+			(decomp_cmd,
+			portage._shell_quote(self.pkg_path),
 			tar_options,
 			portage._shell_quote(self.image_dir))]
 
diff --git a/pym/portage/util/compression_probe.py b/pym/portage/util/compression_probe.py
new file mode 100644
index 0000000..1dc3547
--- /dev/null
+++ b/pym/portage/util/compression_probe.py
@@ -0,0 +1,68 @@
+# Copyright 2015 Gentoo Foundation
+# Distributed under the terms of the GNU General Public License v2
+
+import errno
+import re
+import sys
+
+if sys.hexversion >= 0x3000000:
+	basestring = str
+
+from portage import _encodings, _unicode_encode
+from portage.exception import FileNotFound, PermissionDenied
+
+_decompressors = {
+	"bzip2": "${PORTAGE_BUNZIP2_COMMAND:-${PORTAGE_BZIP2_COMMAND} -d}",
+	"gzip": "gzip -d",
+	"xz": "xz -d",
+}
+
+_compression_re = re.compile(b'^(' +
+	b'(?P<bzip2>\x42\x5a\x68\x39)|' +
+	b'(?P<gzip>\x1f\x8b)|' +
+	b'(?P<xz>\xfd\x37\x7a\x58\x5a\x00))')
+
+def compression_probe(f):
+	"""
+	Identify the compression type of a file. Returns one of the
+	following identifier strings:
+
+		bzip2
+		gzip
+		xz
+
+	@param f: a file path, or file-like object
+	@type f: str or file
+	@return: a string identifying the compression type, or None if the
+		compression type is unrecognized
+	@rtype str or None
+	"""
+
+	open_file = isinstance(f, basestring)
+	if open_file:
+		try:
+			f = open(_unicode_encode(f,
+				encoding=_encodings['fs'], errors='strict'), mode='rb')
+		except IOError as e:
+			if e.errno == PermissionDenied.errno:
+				raise PermissionDenied(f)
+			elif e.errno in (errno.ENOENT, errno.ESTALE):
+				raise FileNotFound(f)
+			else:
+				raise
+
+	try:
+		return _compression_probe_file(f)
+	finally:
+		if open_file:
+			f.close()
+
+def _compression_probe_file(f):
+
+	m = _compression_re.match(f.read(6))
+	if m is not None:
+		for k, v in m.groupdict().items():
+			if v is not None:
+				return k
+
+	return None
-- 
2.0.5