Re: [gentoo-portage-dev] [PATCH] BinpkgExtractorAsync: xz and gzip decompression (142579)

[Brian Dolbec]

On Fri, 16 Jan 2015 00:30:31 -0800
Zac Medico <zmedico@g.o> wrote:

> This adds support for using a binary package's compression header to
> determine the compression type, providing forward-compatibility for
> xz and gzip decompression. The file name extension is disregared, so
> that it will be possible to use a compression-independent file naming
> scheme in the future (see bug #150031 for discussion about proposed
> file naming schemes).
>
> Currently, only decompression is supported. It's useful to provide
> forward-compatibility now, so that binhost clients will be prepared
> to handle future binhost servers that use xz or gzip compression.
>
> X-Gentoo-Bug: 142579
> X-Gentoo-Bug-URL: https://bugs.gentoo.org/show_bug.cgi?id=142579
> ---


oooh, now that is sooooooo much better :D

Merge approved!


(P.S.  too pretty to erase the code below ;) 



>  pym/_emerge/BinpkgExtractorAsync.py   | 25 ++++++++++---
>  pym/portage/util/compression_probe.py | 68
> +++++++++++++++++++++++++++++++++++ 2 files changed, 89
> insertions(+), 4 deletions(-) create mode 100644
> pym/portage/util/compression_probe.py
>
> diff --git a/pym/_emerge/BinpkgExtractorAsync.py
> b/pym/_emerge/BinpkgExtractorAsync.py index be74c2f..6aaa448 100644
> --- a/pym/_emerge/BinpkgExtractorAsync.py
> +++ b/pym/_emerge/BinpkgExtractorAsync.py
> @@ -1,8 +1,13 @@
>  # Copyright 1999-2013 Gentoo Foundation
>  # Distributed under the terms of the GNU General Public License v2
>
> +import logging
> +
>  from _emerge.SpawnProcess import SpawnProcess
>  import portage
> +from portage.localization import _
> +from portage.util.compression_probe import (compression_probe,
> +	_decompressors)
>  import signal
>  import subprocess
>
> @@ -20,19 +25,31 @@ class BinpkgExtractorAsync(SpawnProcess):
>  			if b"--xattrs" in output:
>  				tar_options = "--xattrs"
>
> -		# Add -q to bzip2 opts, in order to avoid "trailing
> garbage after
> -		# EOF ignored" warning messages due to xpak trailer.
> +		decomp_cmd = _decompressors.get(
> +			compression_probe(self.pkg_path))
> +		if decomp_cmd is None:
> +			self.scheduler.output("!!! %s\n" %
> +				_("File compression header
> unrecognized: %s") %
> +				self.pkg_path, log_path=self.logfile,
> +				background=self.background,
> level=logging.ERROR)
> +			self.returncode = 1
> +			self._async_wait()
> +			return
> +
> +		# Add -q to decomp_cmd opts, in order to avoid
> "trailing garbage
> +		# after EOF ignored" warning messages due to xpak
> trailer. # SIGPIPE handling (128 + SIGPIPE) should be compatible with
>  		# assert_sigpipe_ok() that's used by the ebuild
> unpack() helper. self.args = [self._shell_binary, "-c",
> -
> ("${PORTAGE_BUNZIP2_COMMAND:-${PORTAGE_BZIP2_COMMAND} -d} -cq -- %s |
> tar -xp %s -C %s -f - ; " + \
> +			("%s -cq -- %s | tar -xp %s -C %s -f - ; " +
> \ "p=(${PIPESTATUS[@]}) ; " + \
>  			"if [[ ${p[0]} != 0 && ${p[0]} != %d ]] ;
> then " % (128 + signal.SIGPIPE) + \ "echo bzip2 failed with status
> ${p[0]} ; exit ${p[0]} ; fi ; " + \ "if [ ${p[1]} != 0 ] ; then " + \
>  			"echo tar failed with status ${p[1]} ; exit
> ${p[1]} ; fi ; " + \ "exit 0 ;") % \
> -			(portage._shell_quote(self.pkg_path),
> +			(decomp_cmd,
> +			portage._shell_quote(self.pkg_path),
>  			tar_options,
>  			portage._shell_quote(self.image_dir))]
>
> diff --git a/pym/portage/util/compression_probe.py
> b/pym/portage/util/compression_probe.py new file mode 100644
> index 0000000..1dc3547
> --- /dev/null
> +++ b/pym/portage/util/compression_probe.py
> @@ -0,0 +1,68 @@
> +# Copyright 2015 Gentoo Foundation
> +# Distributed under the terms of the GNU General Public License v2
> +
> +import errno
> +import re
> +import sys
> +
> +if sys.hexversion >= 0x3000000:
> +	basestring = str
> +
> +from portage import _encodings, _unicode_encode
> +from portage.exception import FileNotFound, PermissionDenied
> +
> +_decompressors = {
> +	"bzip2":
> "${PORTAGE_BUNZIP2_COMMAND:-${PORTAGE_BZIP2_COMMAND} -d}",
> +	"gzip": "gzip -d",
> +	"xz": "xz -d",
> +}
> +
> +_compression_re = re.compile(b'^(' +
> +	b'(?P<bzip2>\x42\x5a\x68\x39)|' +
> +	b'(?P<gzip>\x1f\x8b)|' +
> +	b'(?P<xz>\xfd\x37\x7a\x58\x5a\x00))')
> +
> +def compression_probe(f):
> +	"""
> +	Identify the compression type of a file. Returns one of the
> +	following identifier strings:
> +
> +		bzip2
> +		gzip
> +		xz
> +
> +	@param f: a file path, or file-like object
> +	@type f: str or file
> +	@return: a string identifying the compression type, or None
> if the
> +		compression type is unrecognized
> +	@rtype str or None
> +	"""
> +
> +	open_file = isinstance(f, basestring)
> +	if open_file:
> +		try:
> +			f = open(_unicode_encode(f,
> +				encoding=_encodings['fs'],
> errors='strict'), mode='rb')
> +		except IOError as e:
> +			if e.errno == PermissionDenied.errno:
> +				raise PermissionDenied(f)
> +			elif e.errno in (errno.ENOENT, errno.ESTALE):
> +				raise FileNotFound(f)
> +			else:
> +				raise
> +
> +	try:
> +		return _compression_probe_file(f)
> +	finally:
> +		if open_file:
> +			f.close()
> +
> +def _compression_probe_file(f):
> +
> +	m = _compression_re.match(f.read(6))
> +	if m is not None:
> +		for k, v in m.groupdict().items():
> +			if v is not None:
> +				return k
> +
> +	return None



-- 
Brian Dolbec <dolsen>