[gentoo-commits] proj/portage:master commit in: bin/ebuild-helpers/, bin/, lib/portage/tests/bin/

[Michał Górny]

commit:     aa0a94198794f8ee8eedad26251b02f7b01dce17
Author:     Michał Górny <mgorny <AT> gentoo <DOT> org>
AuthorDate: Sat Mar 24 11:47:02 2018 +0000
Commit:     Michał Górny <mgorny <AT> gentoo <DOT> org>
CommitDate: Thu Sep 20 18:48:29 2018 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=aa0a9419

ecompress: Replace with implementation from portage[mgorny]

Replace the old ecompress/ecompressdir implementation with the one used
in portage[mgorny].  This is a squashed version of a long series of
commits that gutted off parts of old logic, introduced the new code,
fixed ongoing bugs and finally restored the missing features.  Given
the scale of changes, it is easier to review it as completely new code.

The duplicate model of ecompress/ecompressdir (both not exactly par
on features) has been replaced by a single helper that compresses
everything uniformly.  Instead of complex path-wise processing,
a mark-file logic is used -- the same as in estrip.  Additionally,
the functions are moved out of ebuild path to clearly indicate they are
internal API (they were not used in any ebuilds).

The new implementation is par on features with the old one.  It supports
exclusion suffix lists, automatic decompression of already-compressed
files (with eqawarn-ing about its unreliability), lower bound for
compressed file size, hardlink breaking and symlink fixing.  In other
words, there should be no regression upon replacing it.

Signed-off-by: Michał Górny <mgorny <AT> gentoo.org>
Reviewed-by: Zac Medico <zmedico <AT> gentoo.org>

 bin/ebuild-helpers/ecompress       | 161 --------------------------
 bin/ebuild-helpers/ecompressdir    | 226 -------------------------------------
 bin/ecompress                      | 147 ++++++++++++++++++++++++
 bin/ecompress-file                 |  68 +++++++++++
 bin/misc-functions.sh              | 127 +--------------------
 lib/portage/tests/bin/setup_env.py |   2 +-
 6 files changed, 221 insertions(+), 510 deletions(-)

diff --git a/bin/ebuild-helpers/ecompress b/bin/ebuild-helpers/ecompress
deleted file mode 100755
index 50ee81129..000000000
--- a/bin/ebuild-helpers/ecompress
+++ /dev/null
@@ -1,161 +0,0 @@
-#!/bin/bash
-# Copyright 1999-2010 Gentoo Foundation
-# Distributed under the terms of the GNU General Public License v2
-
-source "${PORTAGE_BIN_PATH}"/isolated-functions.sh || exit 1
-
-if [[ -z $1 ]] ; then
-	__helpers_die "${0##*/}: at least one argument needed"
-	exit 1
-fi
-
-# setup compression stuff
-PORTAGE_COMPRESS=${PORTAGE_COMPRESS-bzip2}
-[[ -z ${PORTAGE_COMPRESS} ]] && exit 0
-
-if [[ ${PORTAGE_COMPRESS_FLAGS+set} != "set" ]] ; then
-	case ${PORTAGE_COMPRESS} in
-		bzip2|gzip)  PORTAGE_COMPRESS_FLAGS="-9";;
-	esac
-fi
-
-# decompress_args(suffix, binary)
-#	- suffix: the compression suffix to work with
-#	- binary: the program to execute that'll compress/decompress
-# new_args: global array used to return revised arguments
-decompress_args() {
-	local suffix=$1 binary=$2
-	shift 2
-
-	# Initialize the global new_args array.
-	new_args=()
-	declare -a decompress_args=()
-	local x i=0 decompress_count=0
-	for x in "$@" ; do
-		if [[ ${x%$suffix} = $x ]] ; then
-			new_args[$i]=$x
-		else
-			new_args[$i]=${x%$suffix}
-			decompress_args[$decompress_count]=$x
-			((decompress_count++))
-		fi
-		((i++))
-	done
-
-	if [ $decompress_count -gt 0 ] ; then
-		${binary} "${decompress_args[@]}"
-		if [ $? -ne 0 ] ; then
-			# Apparently decompression failed for one or more files, so
-			# drop those since we don't want to compress them twice.
-			new_args=()
-			local x i=0
-			for x in "$@" ; do
-				if [[ ${x%$suffix} = $x ]] ; then
-					new_args[$i]=$x
-					((i++))
-				elif [[ -f ${x%$suffix} ]] ; then
-					new_args[$i]=${x%$suffix}
-					((i++))
-				else
-					# Apparently decompression failed for this one, so drop
-					# it since we don't want to compress it twice.
-					true
-				fi
-			done
-		fi
-	fi
-}
-
-case $1 in
-	--suffix)
-		[[ -n $2 ]] && __vecho "${0##*/}: --suffix takes no additional arguments" 1>&2
-
-		if [[ ! -e ${T}/.ecompress.suffix ]] ; then
-			set -e
-			tmpdir="${T}"/.ecompress$$.${RANDOM}
-			mkdir "${tmpdir}"
-			cd "${tmpdir}"
-			# we have to fill the file enough so that there is something
-			# to compress as some programs will refuse to do compression
-			# if it cannot actually compress the file
-			echo {0..1000} > compressme
-			${PORTAGE_COMPRESS} ${PORTAGE_COMPRESS_FLAGS} compressme > /dev/null
-			# If PORTAGE_COMPRESS_FLAGS contains -k then we need to avoid
-			# having our glob match the uncompressed file here.
-			suffix=$(echo compressme.*)
-			[[ -z $suffix || "$suffix" == "compressme.*" ]] && \
-				suffix=$(echo compressme*)
-			suffix=${suffix#compressme}
-			cd /
-			rm -rf "${tmpdir}"
-			echo "${suffix}" > "${T}/.ecompress.suffix"
-		fi
-		cat "${T}/.ecompress.suffix"
-		;;
-	--bin)
-		[[ -n $2 ]] && __vecho "${0##*/}: --bin takes no additional arguments" 1>&2
-
-		echo "${PORTAGE_COMPRESS} ${PORTAGE_COMPRESS_FLAGS}"
-		;;
-	--queue)
-		shift
-		ret=0
-		for x in "${@/%/.ecompress.file}" ; do
-			>> "$x"
-			((ret|=$?))
-		done
-		[[ $ret -ne 0 ]] && __helpers_die "${0##*/} failed"
-		exit $ret
-		;;
-	--dequeue)
-		[[ -n $2 ]] && __vecho "${0##*/}: --dequeue takes no additional arguments" 1>&2
-		find "${D}" -name '*.ecompress.file' -print0 \
-			| sed -e 's:\.ecompress\.file::g' \
-			| ${XARGS} -0 ecompress
-		find "${D}" -name '*.ecompress.file' -print0 | ${XARGS} -0 rm -f
-		;;
-	--*)
-		__helpers_die "${0##*/}: unknown arguments '$*'"
-		exit 1
-		;;
-	*)
-		# Since dodoc calls ecompress on files that are already compressed,
-		# perform decompression here (similar to ecompressdir behavior).
-		decompress_args ".Z" "gunzip -f" "$@"
-		set -- "${new_args[@]}"
-		decompress_args ".gz" "gunzip -f" "$@"
-		set -- "${new_args[@]}"
-		decompress_args ".bz2" "bunzip2 -f" "$@"
-		set -- "${new_args[@]}"
-
-		mask_ext_re=""
-		set -f
-		for x in $PORTAGE_COMPRESS_EXCLUDE_SUFFIXES ; do
-			mask_ext_re+="|$x"
-		done
-		set +f
-		mask_ext_re="^(${mask_ext_re:1})\$"
-		declare -a filtered_args=()
-		i=0
-		for x in "$@" ; do
-			[[ ${x##*.} =~ $mask_ext_re ]] && continue
-			[[ -s ${x} ]] || continue
-			filtered_args[$i]=$x
-			((i++))
-		done
-		[ $i -eq 0 ] && exit 0
-		set -- "${filtered_args[@]}"
-
-		# If a compressed version of the file already exists, simply
-		# delete it so that the compressor doesn't whine (bzip2 will
-		# complain and skip, gzip will prompt for input)
-		suffix=$(ecompress --suffix)
-		[[ -n ${suffix} ]] && echo -n "${@/%/${suffix}$'\001'}" | \
-			tr '\001' '\000' | ${XARGS} -0 rm -f
-		# Finally, let's actually do some real work
-		"${PORTAGE_COMPRESS}" ${PORTAGE_COMPRESS_FLAGS} "$@"
-		ret=$?
-		[[ $ret -ne 0 ]] && __helpers_die "${0##*/} failed"
-		exit $ret
-		;;
-esac

diff --git a/bin/ebuild-helpers/ecompressdir b/bin/ebuild-helpers/ecompressdir
deleted file mode 100755
index dacb857be..000000000
--- a/bin/ebuild-helpers/ecompressdir
+++ /dev/null
@@ -1,226 +0,0 @@
-#!/bin/bash
-# Copyright 1999-2018 Gentoo Foundation
-# Distributed under the terms of the GNU General Public License v2
-
-source "${PORTAGE_BIN_PATH}"/helper-functions.sh || exit 1
-
-if [[ -z $1 ]] ; then
-	__helpers_die "${0##*/}: at least one argument needed"
-	exit 1
-fi
-
-if ! ___eapi_has_prefix_variables; then
-	ED=${D} EPREFIX=
-fi
-
-SIZE_LIMIT=''
-while [[ $# -gt 0 ]] ; do
-	case $1 in
-	--ignore)
-		shift
-		for skip in "$@" ; do
-			[[ -d ${ED%/}/${skip#/} || -f ${ED%/}/${skip#/} ]] \
-				&& >> "${ED%/}/${skip#/}.ecompress.skip"
-		done
-		exit 0
-		;;
-	--queue)
-		shift
-		set -- "${@#/}"
-		set -- "${@/%/.ecompress.dir}"
-		set -- "${@/#/${ED%/}/}"
-		ret=0
-		for x in "$@" ; do
-			# Stash the limit in the .dir file so we can reload it later.
-			printf "${SIZE_LIMIT}" > "${x}"
-			((ret|=$?))
-		done
-		[[ $ret -ne 0 ]] && __helpers_die "${0##*/} failed"
-		exit $ret
-		;;
-	--dequeue)
-		[[ -n $2 ]] && __vecho "${0##*/}: --dequeue takes no additional arguments" 1>&2
-		find "${ED}" -name '*.ecompress.dir' -print0 \
-			| sed -e 's:\.ecompress\.dir::g' -e "s:${ED%/}::g" \
-			| ${XARGS} -0 ecompressdir
-		find "${ED}" -name '*.ecompress.skip' -print0 | ${XARGS} -0 rm -f
-		exit 0
-		;;
-	--limit)
-		SIZE_LIMIT=$2
-		shift
-		;;
-	--*)
-		__helpers_die "${0##*/}: unknown arguments '$*'"
-		exit 1
-		;;
-	*)
-		break
-		;;
-	esac
-	shift
-done
-
-# figure out the new suffix
-suffix=$(ecompress --suffix)
-
-# funk_up_dir(action, suffix, binary, [size_limit])
-#	- action: compress or decompress
-#	- suffix: the compression suffix to work with
-#	- binary: the program to execute that'll compress/decompress
-#   - size_limit: if compressing, skip files smaller than this
-# The directory we act on is implied in the ${dir} variable
-funk_up_dir() {
-	local act=$1 suffix=$2 binary=$3 size_limit=$4
-
-	local negate=""
-	[[ ${act} == "compress" ]] && negate="!"
-
-	local ret=0
-	# first we act on all the files
-	local args=(
-		-type f
-		${negate} -iname "*${suffix}"
-	)
-	[[ -n ${size_limit} ]] && args+=( -size "+${size_limit}c" )
-	find "${dir}" "${args[@]}" -print0 | ${XARGS} -0 ${binary}
-	((ret|=$?))
-
-	# Repeat until nothing changes, in order to handle multiple
-	# levels of indirection (see bug #470916).
-	local -i indirection=0
-	while true ; do
-	local something_changed=
-	while read -r -d $'\0' brokenlink ; do
-		[[ -e ${brokenlink} ]] && continue
-		olddest=$(readlink "${brokenlink}")
-		# Ignore temporarily broken symlinks due to
-		# _relocate_skip_dirs (bug #399595), and handle
-		# absolute symlinks to files that aren't merged
-		# yet (bug #405327).
-		if [[ ${olddest} == /* ]] ; then
-			[ -e "${D%/}${olddest}" ] && continue
-			skip_dir_dest=${T}/ecompress-skip/${olddest#${EPREFIX}}
-		else
-			skip_dir_dest=${T}/ecompress-skip/${actual_dir#${ED%/}}/${brokenlink%/*}/${olddest}
-		fi
-		[[ -e ${skip_dir_dest} ]] && continue
-		if [[ ${act} == "compress" ]] ; then
-			newdest=${olddest}${suffix}
-		else
-			[[ ${olddest} == *${suffix} ]] || continue
-			newdest=${olddest%${suffix}}
-		fi
-		if [[ "${newdest}" == /* ]] ; then
-			[[ -f "${D%/}${newdest}" ]] || continue
-		else
-			[[ -f "${dir}/${brokenlink%/*}/${newdest}" ]] || continue
-		fi
-		something_changed=${brokenlink}
-		rm -f "${brokenlink}"
-		[[ ${act} == "compress" ]] \
-			&& ln -snf "${newdest}" "${brokenlink}${suffix}" \
-			|| ln -snf "${newdest}" "${brokenlink%${suffix}}"
-		((ret|=$?))
-	done < <(find "${dir}" -type l -print0)
-	[[ -n ${something_changed} ]] || break
-	(( indirection++ ))
-	if (( indirection >= 100 )) ; then
-		# Protect against possibility of a bug triggering an endless loop.
-		eerror "ecompressdir: too many levels of indirection for" \
-			"'${actual_dir#${ED%/}}/${something_changed#./}'"
-		break
-	fi
-	done
-	return ${ret}
-}
-
-# _relocate_skip_dirs(srctree, dsttree)
-# Move all files and directories we want to skip running compression
-# on from srctree to dsttree.
-_relocate_skip_dirs() {
-	local srctree="${1%/}" dsttree="${2%/}"
-
-	[[ -d ${srctree} ]] || return 0
-
-	find "${srctree}" -name '*.ecompress.skip' -print0 | \
-	while read -r -d $'\0' src ; do
-		src=${src%.ecompress.skip}
-		dst="${dsttree}${src#${srctree}}"
-		parent=${dst%/*}
-		mkdir -p "${parent}"
-		mv "${src}" "${dst}"
-		mv "${src}.ecompress.skip" "${dst}.ecompress.skip"
-	done
-}
-hide_skip_dirs()    { _relocate_skip_dirs "${ED}" "${T}"/ecompress-skip/ ; }
-restore_skip_dirs() { _relocate_skip_dirs "${T}"/ecompress-skip/ "${ED}" ; }
-
-ret=0
-
-rm -rf "${T}"/ecompress-skip
-
-decompressors=(
-	".Z"    "gunzip -f"
-	".gz"   "gunzip -f"
-	".bz2"  "bunzip2 -f"
-	".xz"   "unxz -f"
-	".lzma" "unxz -f"
-)
-
-__multijob_init
-
-for dir in "$@" ; do
-	dir=${dir#/}
-	dir="${ED%/}/${dir#/}"
-	if [[ ! -d ${dir} ]] ; then
-		__vecho "${0##*/}: /${dir#${ED%/}} does not exist!"
-		continue
-	fi
-	cd "${dir}"
-	actual_dir=${dir}
-	dir=. # use relative path to avoid 'Argument list too long' errors
-
-	# hide all the stuff we want to skip
-	hide_skip_dirs "${dir}"
-
-	# since we've been requested to compress the whole dir,
-	# delete any individual queued requests
-	size_limit=${SIZE_LIMIT:-$(<"${actual_dir}.ecompress.dir")}
-	rm -f "${actual_dir}.ecompress.dir"
-	find "${dir}" -type f -name '*.ecompress.file' -print0 | ${XARGS} -0 rm -f
-
-	# not uncommon for packages to compress doc files themselves
-	for (( i = 0; i < ${#decompressors[@]}; i += 2 )) ; do
-		# It's faster to parallelize at this stage than to try to
-		# parallelize the compressors.  This is because the find|xargs
-		# ends up launching less compressors overall, so the overhead
-		# of forking children ends up dominating.
-		(
-		__multijob_child_init
-		funk_up_dir "decompress" "${decompressors[i]}" "${decompressors[i+1]}"
-		) &
-		__multijob_post_fork
-		: $(( ret |= $? ))
-	done
-
-	__multijob_finish
-	: $(( ret |= $? ))
-
-	# forcibly break all hard links as some compressors whine about it
-	find "${dir}" -type f -links +1 -exec env file="{}" sh -c \
-		'cp -p "${file}" "${file}.ecompress.break" ; mv -f "${file}.ecompress.break" "${file}"' \;
-
-	# now lets do our work
-	if [[ -n ${suffix} ]] ; then
-		__vecho "${0##*/}: $(ecompress --bin) /${actual_dir#${ED%/}/}"
-		funk_up_dir "compress" "${suffix}" "ecompress" "${size_limit}"
-		: $(( ret |= $? ))
-	fi
-
-	# finally, restore the skipped stuff
-	restore_skip_dirs
-done
-
-[[ $ret -ne 0 ]] && __helpers_die "${0##*/} failed"
-exit ${ret}

diff --git a/bin/ecompress b/bin/ecompress
new file mode 100755
index 000000000..075f4225e
--- /dev/null
+++ b/bin/ecompress
@@ -0,0 +1,147 @@
+#!/bin/bash
+# Copyright 1999-2018 Gentoo Foundation
+# Distributed under the terms of the GNU General Public License v2
+
+source "${PORTAGE_BIN_PATH}"/isolated-functions.sh || exit 1
+
+if [[ -z $1 ]] ; then
+	__helpers_die "${0##*/}: at least one argument needed"
+	exit 1
+fi
+
+if ! ___eapi_has_prefix_variables; then
+	ED=${D} EPREFIX=
+fi
+
+while [[ $# -gt 0 ]] ; do
+	case $1 in
+	--ignore)
+		shift
+
+		skip_dirs=()
+		for skip; do
+			if [[ -d ${ED%/}/${skip#/} ]]; then
+				skip_dirs+=( "${ED%/}/${skip#/}" )
+			else
+				rm -f "${ED%/}/${skip#/}.ecompress" || die
+			fi
+		done
+
+		if [[ ${#skip_dirs[@]} -gt 0 ]]; then
+			find "${skip_dirs[@]}" -name '*.ecompress' -delete || die
+		fi
+
+		exit 0
+		;;
+	--queue)
+		shift
+
+		find_args=()
+		for path; do
+			if [[ -e ${ED%/}/${path#/} ]]; then
+				find_args+=( "${ED%/}/${path#/}" )
+			fi
+		done
+
+		if [[ ${#find_args[@]} -gt 0 ]]; then
+			find_args+=( -type f )
+			[[ -n ${PORTAGE_DOCOMPRESS_SIZE_LIMIT} ]] &&
+				find_args+=( -size "+${PORTAGE_DOCOMPRESS_SIZE_LIMIT}c" )
+
+			while IFS= read -d '' -r path; do
+				>> "${path}.ecompress" || die
+			done < <(find "${find_args[@]}" -print0 || die)
+		fi
+
+		exit 0
+		;;
+	--dequeue)
+		[[ -n ${2} ]] && die "${0##*/}: --dequeue takes no additional arguments"
+		break
+		;;
+	*)
+		die "${0##*/}: unknown arguments '$*'"
+		exit 1
+		;;
+	esac
+	shift
+done
+
+# setup compression stuff
+PORTAGE_COMPRESS=${PORTAGE_COMPRESS-bzip2}
+[[ -z ${PORTAGE_COMPRESS} ]] && exit 0
+
+if [[ ${PORTAGE_COMPRESS_FLAGS+set} != "set" ]] ; then
+	case ${PORTAGE_COMPRESS} in
+		bzip2|gzip)  PORTAGE_COMPRESS_FLAGS="-9";;
+	esac
+fi
+
+guess_suffix() {
+	set -e
+	tmpdir="${T}"/.ecompress$$.${RANDOM}
+	mkdir "${tmpdir}"
+	cd "${tmpdir}"
+	# we have to fill the file enough so that there is something
+	# to compress as some programs will refuse to do compression
+	# if it cannot actually compress the file
+	echo {0..1000} > compressme
+	${PORTAGE_COMPRESS} ${PORTAGE_COMPRESS_FLAGS} compressme > /dev/null
+	# If PORTAGE_COMPRESS_FLAGS contains -k then we need to avoid
+	# having our glob match the uncompressed file here.
+	suffix=$(echo compressme.*)
+	[[ -z $suffix || "$suffix" == "compressme.*" ]] && \
+		suffix=$(echo compressme*)
+	suffix=${suffix#compressme}
+	cd /
+	rm -rf "${tmpdir}"
+	echo "${suffix}"
+}
+
+# figure out the new suffix
+export PORTAGE_COMPRESS_SUFFIX=$(guess_suffix) || die
+
+fix_symlinks() {
+	# Repeat until nothing changes, in order to handle multiple
+	# levels of indirection (see bug #470916).
+	local -i indirection=0
+	while true ; do
+	local something_changed=
+	while read -r -d $'\0' brokenlink ; do
+		[[ -e ${brokenlink} ]] && continue
+		olddest=$(readlink "${brokenlink}")
+		newdest=${olddest}${PORTAGE_COMPRESS_SUFFIX}
+		if [[ "${newdest}" == /* ]] ; then
+			[[ -f "${D%/}${newdest}" ]] || continue
+		else
+			[[ -f "${brokenlink%/*}/${newdest}" ]] || continue
+		fi
+		something_changed=${brokenlink}
+		rm -f "${brokenlink}" &&
+		ln -snf "${newdest}" "${brokenlink}${PORTAGE_COMPRESS_SUFFIX}"
+		((ret|=$?))
+	done < <(find "${ED}" -type l -print0 || die)
+
+	[[ -n ${something_changed} ]] || break
+	(( indirection++ ))
+	if (( indirection >= 100 )) ; then
+		# Protect against possibility of a bug triggering an endless loop.
+		eerror "ecompress: too many levels of indirection for" \
+			"'${something_changed#${ED%/}}'"
+		break
+	fi
+	done
+
+	return ${ret}
+}
+
+export PORTAGE_COMPRESS PORTAGE_COMPRESS_FLAGS
+find "${ED}" -name '*.ecompress' -delete \
+	-exec "${PORTAGE_BIN_PATH}"/ecompress-file {} +
+ret=${?}
+
+fix_symlinks
+: $(( ret |= ${?} ))
+[[ $ret -ne 0 ]] && __helpers_die "${0##*/} failed"
+
+exit ${ret}

diff --git a/bin/ecompress-file b/bin/ecompress-file
new file mode 100755
index 000000000..18269c91b
--- /dev/null
+++ b/bin/ecompress-file
@@ -0,0 +1,68 @@
+#!/bin/bash
+# Copyright 1999-2018 Gentoo Foundation
+# Distributed under the terms of the GNU General Public License v2
+
+source "${PORTAGE_BIN_PATH}"/isolated-functions.sh || exit 1
+
+compress_file() {
+	mask_ext_re=""
+	set -f
+	local x
+	for x in $PORTAGE_COMPRESS_EXCLUDE_SUFFIXES ; do
+		mask_ext_re+="|$x"
+	done
+	set +f
+	mask_ext_re="^(${mask_ext_re:1})\$"
+	local filtered_args=()
+	local had_precompressed=
+	for x in "$@" ; do
+		[[ ${x##*.} =~ $mask_ext_re ]] && continue
+		[[ -s ${x} ]] || continue
+
+		# handle precompressed files
+		case ${x} in
+			*.gz|*.Z)
+				gunzip -f "${x}" || __helpers_die "gunzip failed"
+				x=${x%.*}
+				had_precompressed=1;;
+			*.bz2)
+				bunzip2 -f "${x}" || __helpers_die "bunzip2 failed"
+				x=${x%.bz2}
+				had_precompressed=1;;
+			*.lzma|*.xz)
+				unxz -f "${x}" || __helpers_die "unxz failed"
+				x=${x%.*}
+				had_precompressed=1;;
+		esac
+
+		filtered_args+=( "$x" )
+	done
+	[[ ${#filtered_args[@]} -eq 0 ]] && return 0
+	set -- "${filtered_args[@]}"
+
+	if [[ ${had_precompressed} ]]; then
+		eqawarn "One or more compressed files were found in docompress-ed directories."
+		eqawarn "Please fix the ebuild not to install compressed files (manpages,"
+		eqawarn "documentation) when automatic compression is used."
+	fi
+
+	# If a compressed version of the file already exists, simply
+	# delete it so that the compressor doesn't whine (bzip2 will
+	# complain and skip, gzip will prompt for input)
+	[[ -n ${PORTAGE_COMPRESS_SUFFIX} ]] && echo -n "${@/%/${PORTAGE_COMPRESS_SUFFIX}$'\001'}" | \
+		tr '\001' '\000' | ${XARGS} -0 rm -f
+
+	# forcibly break all hard links as some compressors whine about it
+	while IFS= read -d '' -r x ; do
+		cp -p "${x}" "${x}.ecompress.break" || die
+		mv -f "${x}.ecompress.break" "${x}" || die
+	done < <(find "${@}" -type f -links +1 -print0)
+
+	# Finally, let's actually do some real work
+	"${PORTAGE_COMPRESS}" ${PORTAGE_COMPRESS_FLAGS} "$@"
+	ret=$?
+	[[ $ret -ne 0 ]] && __helpers_die "${0##*/} failed"
+	return ${ret}
+}
+
+compress_file "${@%.ecompress}"

diff --git a/bin/misc-functions.sh b/bin/misc-functions.sh
index ed66e90ca..8c88f8f35 100755
--- a/bin/misc-functions.sh
+++ b/bin/misc-functions.sh
@@ -43,125 +43,6 @@ install_symlink_html_docs() {
 	fi
 }
 
-# replacement for "readlink -f" or "realpath"
-READLINK_F_WORKS=""
-canonicalize() {
-	if [[ -z ${READLINK_F_WORKS} ]] ; then
-		if [[ $(readlink -f -- /../ 2>/dev/null) == "/" ]] ; then
-			READLINK_F_WORKS=true
-		else
-			READLINK_F_WORKS=false
-		fi
-	fi
-	if ${READLINK_F_WORKS} ; then
-		readlink -f -- "$@"
-		return
-	fi
-
-	local f=$1 b n=10 wd=$(pwd)
-	while (( n-- > 0 )); do
-		while [[ ${f: -1} = / && ${#f} -gt 1 ]]; do
-			f=${f%/}
-		done
-		b=${f##*/}
-		cd "${f%"${b}"}" 2>/dev/null || break
-		if [[ ! -L ${b} ]]; then
-			f=$(pwd -P)
-			echo "${f%/}/${b}"
-			cd "${wd}"
-			return 0
-		fi
-		f=$(readlink "${b}")
-	done
-	cd "${wd}"
-	return 1
-}
-
-prepcompress() {
-	local -a include exclude incl_d incl_f
-	local f g i real_f real_d
-	if ! ___eapi_has_prefix_variables; then
-		local ED=${D}
-	fi
-
-	# Canonicalize path names and check for their existence.
-	real_d=$(canonicalize "${ED}")
-	for (( i = 0; i < ${#PORTAGE_DOCOMPRESS[@]}; i++ )); do
-		real_f=$(canonicalize "${ED%/}/${PORTAGE_DOCOMPRESS[i]#/}")
-		f=${real_f#"${real_d}"}
-		if [[ ${real_f} != "${f}" ]] && [[ -d ${real_f} || -f ${real_f} ]]
-		then
-			include[${#include[@]}]=${f:-/}
-		elif [[ ${i} -ge 3 ]]; then
-			ewarn "prepcompress:" \
-				"ignoring nonexistent path '${PORTAGE_DOCOMPRESS[i]}'"
-		fi
-	done
-	for (( i = 0; i < ${#PORTAGE_DOCOMPRESS_SKIP[@]}; i++ )); do
-		real_f=$(canonicalize "${ED%/}/${PORTAGE_DOCOMPRESS_SKIP[i]#/}")
-		f=${real_f#"${real_d}"}
-		if [[ ${real_f} != "${f}" ]] && [[ -d ${real_f} || -f ${real_f} ]]
-		then
-			exclude[${#exclude[@]}]=${f:-/}
-		elif [[ ${i} -ge 1 ]]; then
-			ewarn "prepcompress:" \
-				"ignoring nonexistent path '${PORTAGE_DOCOMPRESS_SKIP[i]}'"
-		fi
-	done
-
-	# Remove redundant entries from lists.
-	# For the include list, remove any entries that are:
-	# a) contained in a directory in the include or exclude lists, or
-	# b) identical with an entry in the exclude list.
-	for (( i = ${#include[@]} - 1; i >= 0; i-- )); do
-		f=${include[i]}
-		for g in "${include[@]}"; do
-			if [[ ${f} == "${g%/}"/* ]]; then
-				unset include[i]
-				continue 2
-			fi
-		done
-		for g in "${exclude[@]}"; do
-			if [[ ${f} = "${g}" || ${f} == "${g%/}"/* ]]; then
-				unset include[i]
-				continue 2
-			fi
-		done
-	done
-	# For the exclude list, remove any entries that are:
-	# a) contained in a directory in the exclude list, or
-	# b) _not_ contained in a directory in the include list.
-	for (( i = ${#exclude[@]} - 1; i >= 0; i-- )); do
-		f=${exclude[i]}
-		for g in "${exclude[@]}"; do
-			if [[ ${f} == "${g%/}"/* ]]; then
-				unset exclude[i]
-				continue 2
-			fi
-		done
-		for g in "${include[@]}"; do
-			[[ ${f} == "${g%/}"/* ]] && continue 2
-		done
-		unset exclude[i]
-	done
-
-	# Split the include list into directories and files
-	for f in "${include[@]}"; do
-		if [[ -d ${ED%/}/${f#/} ]]; then
-			incl_d[${#incl_d[@]}]=${f}
-		else
-			incl_f[${#incl_f[@]}]=${f}
-		fi
-	done
-
-	# Queue up for compression.
-	# ecompress{,dir} doesn't like to be called with empty argument lists.
-	[[ ${#incl_d[@]} -gt 0 ]] && ecompressdir --limit ${PORTAGE_DOCOMPRESS_SIZE_LIMIT:-0} --queue "${incl_d[@]}"
-	[[ ${#incl_f[@]} -gt 0 ]] && ecompress --queue "${incl_f[@]/#/${ED%/}}"
-	[[ ${#exclude[@]} -gt 0 ]] && ecompressdir --ignore "${exclude[@]}"
-	return 0
-}
-
 __prepall() {
 	if has chflags $FEATURES ; then
 		# Save all the file flags for restoration at the end of prepall.
@@ -173,6 +54,11 @@ __prepall() {
 
 	[[ -d ${ED%/}/usr/share/info ]] && prepinfo
 
+	# Apply compression.
+	"${PORTAGE_BIN_PATH}"/ecompress --queue "${PORTAGE_DOCOMPRESS[@]}"
+	"${PORTAGE_BIN_PATH}"/ecompress --ignore "${PORTAGE_DOCOMPRESS_SKIP[@]}"
+	"${PORTAGE_BIN_PATH}"/ecompress --dequeue
+
 	___eapi_has_dostrip || prepallstrip
 
 	if has chflags $FEATURES ; then
@@ -239,9 +125,6 @@ install_qa_check() {
 
 	export STRIP_MASK
 	__prepall
-	prepcompress
-	ecompressdir --dequeue
-	ecompress --dequeue
 
 	if ___eapi_has_dostrip; then
 		"${PORTAGE_BIN_PATH}"/estrip --queue "${PORTAGE_DOSTRIP[@]}"

diff --git a/lib/portage/tests/bin/setup_env.py b/lib/portage/tests/bin/setup_env.py
index 9cc26df08..33b167989 100644
--- a/lib/portage/tests/bin/setup_env.py
+++ b/lib/portage/tests/bin/setup_env.py
@@ -82,6 +82,6 @@ for bin in os.listdir(os.path.join(bindir, 'ebuild-helpers')):
 	if bin.startswith('do') or \
 	   bin.startswith('new') or \
 	   bin.startswith('prep') or \
-	   bin in ('ecompress', 'ecompressdir', 'fowners', 'fperms'):
+	   bin in ('fowners', 'fperms'):
 		globals()[bin] = create_portage_wrapper(
 			os.path.join(bindir, 'ebuild-helpers', bin))