Re: [gentoo-portage-dev] [PATCH] prepstrip: rewrite in python

[Arfrever Frehtes Taifersar Arahesis]

2013-10-16 23:34 Mike Frysinger napisał(a):
> This rewrites the prestrip logic from scratch in python.

Some things are incompatible with Python 3, resulting at least in ImportErrors and a NameError.
See other comments below.

> The bash code was getting way too hairy, and the addition of parallel stripping was
> (still) racy.
>
> The upside is this has unittests :).
>
> The downside is that this adds a dep on pyelftools in portage.  But I
> think that's fine as pax-utils has been pulling that in for some time.
> ---
>  bin/ebuild-helpers/prepstrip                       | 387 +-----------
>  pym/portage/bin/__init__.py                        |   0
>  pym/portage/bin/prepstrip.py                       | 682 +++++++++++++++++++++
>  pym/portage/tests/bin/test_prepstrip.py            | 253 ++++++++
>  pym/portage/tests/bin/testdir/.gitignore           |   2 +
>  .../bin/testdir/work/somepackage-1.2.3/Makefile    |  65 ++
>  .../testdir/work/somepackage-1.2.3/o/.gitignore    |   1 +
>  .../bin/testdir/work/somepackage-1.2.3/src/main.c  |   5 +
>  pym/portage/util/parallel.py                       | 598 ++++++++++++++++++
>  9 files changed, 1617 insertions(+), 376 deletions(-)
>  create mode 100644 pym/portage/bin/__init__.py
>  create mode 100644 pym/portage/bin/prepstrip.py
>  create mode 100644 pym/portage/tests/bin/test_prepstrip.py
>  create mode 100644 pym/portage/tests/bin/testdir/.gitignore
>  create mode 100644 pym/portage/tests/bin/testdir/work/somepackage-1.2.3/Makefile
>  create mode 100644 pym/portage/tests/bin/testdir/work/somepackage-1.2.3/o/.gitignore
>  create mode 100644 pym/portage/tests/bin/testdir/work/somepackage-1.2.3/src/main.c
>  create mode 100644 pym/portage/util/parallel.py
>
> diff --git a/bin/ebuild-helpers/prepstrip b/bin/ebuild-helpers/prepstrip
> index 9b2d47c..c8df5a5 100755
> --- a/bin/ebuild-helpers/prepstrip
> +++ b/bin/ebuild-helpers/prepstrip
> @@ -1,386 +1,21 @@
> -#!/bin/bash
> +#!/usr/bin/python
>  # Copyright 1999-2013 Gentoo Foundation
>  # Distributed under the terms of the GNU General Public License v2
>
> -PORTAGE_PYM_PATH=${PORTAGE_PYM_PATH:-/usr/lib/portage/pym}
> -source "${PORTAGE_BIN_PATH:-/usr/lib/portage/bin}"/helper-functions.sh
> +"""Helper for stripping installed programs.
>
> -# avoid multiple calls to `has`.  this creates things like:
> -#   FEATURES_foo=false
> -# if "foo" is not in $FEATURES
> -tf() { "$@" && echo true || echo false ; }
> -exp_tf() {
> -	local flag var=$1
> -	shift
> -	for flag in "$@" ; do
> -		eval ${var}_${flag}=$(tf has ${flag} ${!var})
> -	done
> -}
> -exp_tf FEATURES compressdebug installsources nostrip splitdebug xattr
> -exp_tf RESTRICT binchecks installsources splitdebug strip
> +This handles all the fun things in addition to stripping like splitdebug,
> +installsources, etc...
>
> -if ! ___eapi_has_prefix_variables; then
> -	EPREFIX= ED=${D}
> -fi
> +If no paths are specified, then $D is searched.
> +"""
>
> -banner=false
> -SKIP_STRIP=false
> -if ${RESTRICT_strip} || ${FEATURES_nostrip} ; then
> -	SKIP_STRIP=true
> -	banner=true
> -	${FEATURES_installsources} || exit 0
> -fi
> +from __future__ import print_function
>
> -PRESERVE_XATTR=false
> -if [[ ${KERNEL} == linux ]] && ${FEATURES_xattr} ; then
> -	PRESERVE_XATTR=true
> -	if type -P getfattr >/dev/null && type -P setfattr >/dev/null ; then
> -		dump_xattrs() {
> -			getfattr -d --absolute-names "$1"
> -		}
> -		restore_xattrs() {
> -			setfattr --restore=-
> -		}
> -	else
> -		dump_xattrs() {
> -			PYTHONPATH=${PORTAGE_PYTHONPATH:-${PORTAGE_PYM_PATH}} \
> -			"${PORTAGE_PYTHON:-/usr/bin/python}" \
> -			"${PORTAGE_BIN_PATH}/xattr-helper.py" --dump < <(echo -n "$1")
> -		}
> -		restore_xattrs() {
> -			PYTHONPATH=${PORTAGE_PYTHONPATH:-${PORTAGE_PYM_PATH}} \
> -			"${PORTAGE_PYTHON:-/usr/bin/python}" \
> -			"${PORTAGE_BIN_PATH}/xattr-helper.py" --restore
> -		}
> -	fi
> -fi
> +import sys
>
> -# look up the tools we might be using
> -for t in STRIP:strip OBJCOPY:objcopy READELF:readelf ; do
> -	v=${t%:*} # STRIP
> -	t=${t#*:} # strip
> -	eval ${v}=\"${!v:-${CHOST}-${t}}\"
> -	type -P -- ${!v} >/dev/null || eval ${v}=${t}
> -done
> +from portage.bin import prepstrip
>
> -# Figure out what tool set we're using to strip stuff
> -unset SAFE_STRIP_FLAGS DEF_STRIP_FLAGS SPLIT_STRIP_FLAGS
> -case $(${STRIP} --version 2>/dev/null) in
> -*elfutils*) # dev-libs/elfutils
> -	# elfutils default behavior is always safe, so don't need to specify
> -	# any flags at all
> -	SAFE_STRIP_FLAGS=""
> -	DEF_STRIP_FLAGS="--remove-comment"
> -	SPLIT_STRIP_FLAGS="-f"
> -	;;
> -*GNU*) # sys-devel/binutils
> -	# We'll leave out -R .note for now until we can check out the relevance
> -	# of the section when it has the ALLOC flag set on it ...
> -	SAFE_STRIP_FLAGS="--strip-unneeded"
> -	DEF_STRIP_FLAGS="-R .comment -R .GCC.command.line -R .note.gnu.gold-version"
> -	SPLIT_STRIP_FLAGS=
> -	;;
> -esac
> -: ${PORTAGE_STRIP_FLAGS=${SAFE_STRIP_FLAGS} ${DEF_STRIP_FLAGS}}
>
> -prepstrip_sources_dir=${EPREFIX}/usr/src/debug/${CATEGORY}/${PF}
> -
> -type -P debugedit >/dev/null && debugedit_found=true || debugedit_found=false
> -debugedit_warned=false
> -
> -__multijob_init
> -
> -# Setup $T filesystem layout that we care about.
> -tmpdir="${T}/prepstrip"
> -rm -rf "${tmpdir}"
> -mkdir -p "${tmpdir}"/{inodes,splitdebug,sources}
> -
> -# Usage: save_elf_sources <elf>
> -save_elf_sources() {
> -	${FEATURES_installsources} || return 0
> -	${RESTRICT_installsources} && return 0
> -	if ! ${debugedit_found} ; then
> -		if ! ${debugedit_warned} ; then
> -			debugedit_warned=true
> -			ewarn "FEATURES=installsources is enabled but the debugedit binary could not"
> -			ewarn "be found. This feature will not work unless debugedit is installed!"
> -		fi
> -		return 0
> -	fi
> -
> -	local x=$1
> -
> -	# since we're editing the ELF here, we should recompute the build-id
> -	# (the -i flag below).  save that output so we don't need to recompute
> -	# it later on in the save_elf_debug step.
> -	buildid=$(debugedit -i \
> -		-b "${WORKDIR}" \
> -		-d "${prepstrip_sources_dir}" \
> -		-l "${tmpdir}/sources/${x##*/}.${BASHPID}" \
> -		"${x}")
> -}
> -
> -# Usage: save_elf_debug <elf> [splitdebug file]
> -save_elf_debug() {
> -	${FEATURES_splitdebug} || return 0
> -	${RESTRICT_splitdebug} && return 0
> -
> -	# NOTE: Debug files must be installed in
> -	# ${EPREFIX}/usr/lib/debug/${EPREFIX} (note that ${EPREFIX} occurs
> -	# twice in this path) in order for gdb's debug-file-directory
> -	# lookup to work correctly.
> -	local x=$1
> -	local inode_debug=$2
> -	local splitdebug=$3
> -	local y=${ED}usr/lib/debug/${x:${#D}}.debug
> -
> -	# dont save debug info twice
> -	[[ ${x} == *".debug" ]] && return 0
> -
> -	mkdir -p "${y%/*}"
> -
> -	if [ -f "${inode_debug}" ] ; then
> -		ln "${inode_debug}" "${y}" || die "ln failed unexpectedly"
> -	else
> -		if [[ -n ${splitdebug} ]] ; then
> -			mv "${splitdebug}" "${y}"
> -		else
> -			local objcopy_flags="--only-keep-debug"
> -			${FEATURES_compressdebug} && objcopy_flags+=" --compress-debug-sections"
> -			${OBJCOPY} ${objcopy_flags} "${x}" "${y}"
> -			${OBJCOPY} --add-gnu-debuglink="${y}" "${x}"
> -		fi
> -		# Only do the following if the debug file was
> -		# successfully created (see bug #446774).
> -		if [ $? -eq 0 ] ; then
> -			local args="a-x,o-w"
> -			[[ -g ${x} || -u ${x} ]] && args+=",go-r"
> -			chmod ${args} "${y}"
> -			ln "${y}" "${inode_debug}" || die "ln failed unexpectedly"
> -		fi
> -	fi
> -
> -	# if we don't already have build-id from debugedit, look it up
> -	if [[ -z ${buildid} ]] ; then
> -		# convert the readelf output to something useful
> -		buildid=$(${READELF} -n "${x}" 2>/dev/null | awk '/Build ID:/{ print $NF; exit }')
> -	fi
> -	if [[ -n ${buildid} ]] ; then
> -		local buildid_dir="${ED}usr/lib/debug/.build-id/${buildid:0:2}"
> -		local buildid_file="${buildid_dir}/${buildid:2}"
> -		mkdir -p "${buildid_dir}"
> -		[ -L "${buildid_file}".debug ] || ln -s "../../${x:${#D}}.debug" "${buildid_file}.debug"
> -		[ -L "${buildid_file}" ] || ln -s "/${x:${#D}}" "${buildid_file}"
> -	fi
> -}
> -
> -# Usage: process_elf <elf>
> -process_elf() {
> -	local x=$1 inode_link=$2 strip_flags=${*:3}
> -	local already_stripped lockfile xt_data
> -
> -	__vecho "   ${x:${#ED}}"
> -
> -	# If two processes try to debugedit or strip the same hardlink at the
> -	# same time, it may corrupt files or cause loss of splitdebug info.
> -	# So, use a lockfile to prevent interference (easily observed with
> -	# dev-vcs/git which creates ~111 hardlinks to one file in
> -	# /usr/libexec/git-core).
> -	lockfile=${inode_link}_lockfile
> -	if ! ln "${inode_link}" "${lockfile}" 2>/dev/null ; then
> -		while [[ -f ${lockfile} ]] ; do
> -			sleep 1
> -		done
> -		unset lockfile
> -	fi
> -
> -	[ -f "${inode_link}_stripped" ] && already_stripped=true || already_stripped=false
> -
> -	if ! ${already_stripped} ; then
> -		if ${PRESERVE_XATTR} ; then
> -			xt_data=$(dump_xattrs "${x}")
> -		fi
> -		save_elf_sources "${x}"
> -	fi
> -
> -	if ${strip_this} ; then
> -
> -		# see if we can split & strip at the same time
> -		if [[ -n ${SPLIT_STRIP_FLAGS} ]] ; then
> -			local shortname="${x##*/}.debug"
> -			local splitdebug="${tmpdir}/splitdebug/${shortname}.${BASHPID}"
> -			${already_stripped} || \
> -			${STRIP} ${strip_flags} \
> -				-f "${splitdebug}" \
> -				-F "${shortname}" \
> -				"${x}"
> -			save_elf_debug "${x}" "${inode_link}_debug" "${splitdebug}"
> -		else
> -			save_elf_debug "${x}" "${inode_link}_debug"
> -			${already_stripped} || \
> -			${STRIP} ${strip_flags} "${x}"
> -		fi
> -	fi
> -
> -	if ${already_stripped} ; then
> -		rm -f "${x}" || die "rm failed unexpectedly"
> -		ln "${inode_link}_stripped" "${x}" || die "ln failed unexpectedly"
> -	else
> -		ln "${x}" "${inode_link}_stripped" || die "ln failed unexpectedly"
> -		if [[ ${xt_data} ]] ; then
> -			restore_xattrs <<< "${xt_data}"
> -		fi
> -	fi
> -
> -	[[ -n ${lockfile} ]] && rm -f "${lockfile}"
> -}
> -
> -# The existance of the section .symtab tells us that a binary is stripped.
> -# We want to log already stripped binaries, as this may be a QA violation.
> -# They prevent us from getting the splitdebug data.
> -if ! ${RESTRICT_binchecks} && ! ${RESTRICT_strip} ; then
> -	# We need to do the non-stripped scan serially first before we turn around
> -	# and start stripping the files ourselves.  The log parsing can be done in
> -	# parallel though.
> -	log=${tmpdir}/scanelf-already-stripped.log
> -	scanelf -yqRBF '#k%F' -k '!.symtab' "$@" | sed -e "s#^${ED}##" > "${log}"
> -	(
> -	__multijob_child_init
> -	qa_var="QA_PRESTRIPPED_${ARCH/-/_}"
> -	[[ -n ${!qa_var} ]] && QA_PRESTRIPPED="${!qa_var}"
> -	if [[ -n ${QA_PRESTRIPPED} && -s ${log} && \
> -		${QA_STRICT_PRESTRIPPED-unset} = unset ]] ; then
> -		shopts=$-
> -		set -o noglob
> -		for x in ${QA_PRESTRIPPED} ; do
> -			sed -e "s#^${x#/}\$##" -i "${log}"
> -		done
> -		set +o noglob
> -		set -${shopts}
> -	fi
> -	sed -e "/^\$/d" -e "s#^#/#" -i "${log}"
> -	if [[ -s ${log} ]] ; then
> -		__vecho -e "\n"
> -		eqawarn "QA Notice: Pre-stripped files found:"
> -		eqawarn "$(<"${log}")"
> -	else
> -		rm -f "${log}"
> -	fi
> -	) &
> -	__multijob_post_fork
> -fi
> -
> -# Since strip creates a new inode, we need to know the initial set of
> -# inodes in advance, so that we can avoid interference due to trying
> -# to strip the same (hardlinked) file multiple times in parallel.
> -# See bug #421099.
> -if  [[ ${USERLAND} == BSD ]] ; then
> -	get_inode_number() { stat -f '%i' "$1"; }
> -else
> -	get_inode_number() { stat -c '%i' "$1"; }
> -fi
> -cd "${tmpdir}/inodes" || die "cd failed unexpectedly"
> -while read -r x ; do
> -	inode_link=$(get_inode_number "${x}") || die "stat failed unexpectedly"
> -	echo "${x}" >> "${inode_link}" || die "echo failed unexpectedly"
> -done < <(
> -	# Use sort -u to eliminate duplicates for bug #445336.
> -	(
> -		scanelf -yqRBF '#k%F' -k '.symtab' "$@"
> -		find "$@" -type f ! -type l -name '*.a'
> -	) | LC_ALL=C sort -u
> -)
> -
> -# Now we look for unstripped binaries.
> -for inode_link in $(shopt -s nullglob; echo *) ; do
> -while read -r x
> -do
> -
> -	if ! ${banner} ; then
> -		__vecho "strip: ${STRIP} ${PORTAGE_STRIP_FLAGS}"
> -		banner=true
> -	fi
> -
> -	(
> -	__multijob_child_init
> -	f=$(file "${x}") || exit 0
> -	[[ -z ${f} ]] && exit 0
> -
> -	if ! ${SKIP_STRIP} ; then
> -		# The noglob funk is to support STRIP_MASK="/*/booga" and to keep
> -		#  the for loop from expanding the globs.
> -		# The eval echo is to support STRIP_MASK="/*/{booga,bar}" sex.
> -		set -o noglob
> -		strip_this=true
> -		for m in $(eval echo ${STRIP_MASK}) ; do
> -			[[ /${x#${ED}} == ${m} ]] && strip_this=false && break
> -		done
> -		set +o noglob
> -	else
> -		strip_this=false
> -	fi
> -
> -	# In Prefix we are usually an unprivileged user, so we can't strip
> -	# unwritable objects.  Make them temporarily writable for the
> -	# stripping.
> -	was_not_writable=false
> -	if [[ ! -w ${x} ]] ; then
> -		was_not_writable=true
> -		chmod u+w "${x}"
> -	fi
> -
> -	# only split debug info for final linked objects
> -	# or kernel modules as debuginfo for intermediatary
> -	# files (think crt*.o from gcc/glibc) is useless and
> -	# actually causes problems.  install sources for all
> -	# elf types though cause that stuff is good.
> -
> -	buildid=
> -	if [[ ${f} == *"current ar archive"* ]] ; then
> -		__vecho "   ${x:${#ED}}"
> -		if ${strip_this} ; then
> -			# hmm, can we split debug/sources for .a ?
> -			${STRIP} -g "${x}"
> -		fi
> -	elif [[ ${f} == *"SB executable"* || ${f} == *"SB shared object"* ]] ; then
> -		process_elf "${x}" "${inode_link}" ${PORTAGE_STRIP_FLAGS}
> -	elif [[ ${f} == *"SB relocatable"* ]] ; then
> -		process_elf "${x}" "${inode_link}" ${SAFE_STRIP_FLAGS}
> -	fi
> -
> -	if ${was_not_writable} ; then
> -		chmod u-w "${x}"
> -	fi
> -	) &
> -	__multijob_post_fork
> -
> -done < "${inode_link}"
> -done
> -
> -# With a bit more work, we could run the rsync processes below in
> -# parallel, but not sure that'd be an overall improvement.
> -__multijob_finish
> -
> -cd "${tmpdir}"/sources/ && cat * > "${tmpdir}/debug.sources" 2>/dev/null
> -if [[ -s ${tmpdir}/debug.sources ]] && \
> -   ${FEATURES_installsources} && \
> -   ! ${RESTRICT_installsources} && \
> -   ${debugedit_found}
> -then
> -	__vecho "installsources: rsyncing source files"
> -	[[ -d ${D}${prepstrip_sources_dir} ]] || mkdir -p "${D}${prepstrip_sources_dir}"
> -	grep -zv '/<[^/>]*>$' "${tmpdir}"/debug.sources | \
> -		(cd "${WORKDIR}"; LANG=C sort -z -u | \
> -		rsync -tL0 --chmod=ugo-st,a+r,go-w,Da+x,Fa-x --files-from=- "${WORKDIR}/" "${D}${prepstrip_sources_dir}/" )
> -
> -	# Preserve directory structure.
> -	# Needed after running save_elf_sources.
> -	# https://bugzilla.redhat.com/show_bug.cgi?id=444310
> -	while read -r -d $'\0' emptydir
> -	do
> -		>> "${emptydir}"/.keepdir
> -	done < <(find "${D}${prepstrip_sources_dir}/" -type d -empty -print0)
> -fi
> -
> -cd "${T}"
> -rm -rf "${tmpdir}"
> +if __name__ == '__main__':
> +	sys.exit(prepstrip.main(sys.argv[1:]))
> diff --git a/pym/portage/bin/__init__.py b/pym/portage/bin/__init__.py
> new file mode 100644
> index 0000000..e69de29
> diff --git a/pym/portage/bin/prepstrip.py b/pym/portage/bin/prepstrip.py
> new file mode 100644
> index 0000000..0f6eb8d
> --- /dev/null
> +++ b/pym/portage/bin/prepstrip.py
> @@ -0,0 +1,682 @@
> +#!/usr/bin/python
> +# Copyright 1999-2013 Gentoo Foundation
> +# Distributed under the terms of the GNU General Public License v2
> +
> +"""Helper code for stripping installed programs.
> +
> +This handles all the fun things in addition to stripping like splitdebug,
> +installsources, etc...
> +"""
> +
> +from __future__ import print_function
> +
> +import contextlib
> +import errno
> +import fcntl
> +import fnmatch
> +import multiprocessing
> +import re
> +import shutil
> +import stat
> +import subprocess
> +import sys
> +import tempfile
> +
> +from elftools.elf.elffile import ELFFile
> +from elftools.common import exceptions
> +
> +from portage import os, _shell_quote
> +from portage.elog.messages import eqawarn, ewarn
> +from portage.process import find_binary
> +from portage.util import parallel, shlex
> +from portage.util._argparse import ArgumentParser
> +from portage.util.movefile import _copyxattr
> +from portage.util._xattr import preserve_xattrs, xattr
> +
> +
> +class Paths(object):
> +	"""Object to hold (and cache) various paths"""
> +
> +	_PATH_VARS = ('D', 'ED', 'EPREFIX', 'T', 'WORKDIR')
> +	_OTHER_VARS = ('CATEGORY', 'PN', 'PF')
> +	_VARS = _PATH_VARS + _OTHER_VARS
> +	D = ''
> +	ED = ''
> +	EPREFIX = ''
> +	CATEGORY = ''
> +	PN = ''
> +	PF = ''
> +	T = ''
> +	WORKDIR = ''
> +	SOURCES_DIR = ''
> +	DEBUG_DIR = ''
> +	BUILDID_DIR = ''
> +
> +	@classmethod
> +	def cache(cls, environ=os.environ):
> +		for var in cls._VARS:
> +			val = environ.get(var, '')
> +			if var in cls._PATH_VARS:
> +				val = val.rstrip('/')
> +			setattr(cls, var, val)
> +		if cls.D and not cls.ED:
> +			cls.ED = cls.D
> +		if not cls.T:
> +			cls.T = tempfile.gettempdir()
> +
> +		cls.SOURCES_DIR = os.path.normpath(
> +			'%s/usr/src/debug/%s/%s' % (cls.EPREFIX, cls.CATEGORY, cls.PF))
> +		# NOTE: Debug files must be installed in
> +		# ${EPREFIX}/usr/lib/debug/${EPREFIX} (note that ${EPREFIX} occurs
> +		# twice in this path) in order for gdb's debug-file-directory
> +		# lookup to work correctly.
> +		cls.DEBUG_DIR = os.path.normpath('%s/usr/lib/debug' % (cls.EPREFIX))
> +		cls.BUILDID_DIR = '%s/.build-id' % (cls.DEBUG_DIR)
> +
> +
> +class Features(object):
> +	"""Object to hold (and cache) FEATURES availability
> +
> +	Once we've been cached, you can simply do:
> +		if Features.strip:
> +			... do strip stuff ...
> +	"""
> +
> +	# Some features are always enabled even if not explicitly so.
> +	IMPLICIT_FEATURES = frozenset((
> +		'binchecks',
> +		'strip',
> +	))
> +
> +	# These are the features we deal with in this file.
> +	FEATURES = frozenset(IMPLICIT_FEATURES | set((
> +		'compressdebug',
> +		'installsources',
> +		'splitdebug',
> +		'xattr',
> +	)))
> +
> +
> +	@classmethod
> +	def cache(cls, features=None, environ=os.environ):
> +		"""Cache |features| tests to avoid processing at runtime"""
> +		if features is None:
> +			features = cls.FEATURES
> +
> +		# Portage should have done the incremental thing for us already
> +		# so we don't have to handle it ourselves (like 'foo -foo foo').
> +		env_features = environ.get('FEATURES', '').split()
> +		env_restrict = environ.get('RESTRICT', '').split()
> +		env_features += list(cls.IMPLICIT_FEATURES)
> +
> +		for f in features:
> +			setattr(cls, f, f in env_features and f not in env_restrict)
> +
> +			# Backwards compat support for "nostrip" and such.
> +			if 'no' + f in env_features:
> +				setattr(cls, f, False)
> +
> +	@classmethod
> +	def reset(cls):
> +		for f in cls.FEATURES:
> +			delattr(cls, f)
> +
> +	@classmethod
> +	def __str__(cls):
> +		return ' '.join('%s=%s' % (f, getattr(cls, f)) for f in cls.FEATURES)
> +
> +
> +class Tools(object):
> +	"""Object to hold (and cache) toolchain tools that we'll need
> +
> +	We also need to deal with things like env vars and compiler prefixes."""
> +
> +	TOOLS = frozenset((
> +		'debugedit',
> +		'objcopy',
> +		'strip',
> +	))
> +
> +	_strip_flags = {}
> +	_strip_type = None
> +
> +	@staticmethod
> +	def find_toolchain_tool(tool, environ):
> +		"""Given a toolchain |tool|, look it up via env vars
> +
> +		e.g. We'll get "strip", so check for ${STRIP} and ${CHOST}-strip
> +		before falling back to "strip".
> +		"""
> +		# Look for $STRIP first.
> +		etool = environ.get(tool.upper())
> +		if etool:
> +			path = find_binary(etool)
> +			if path:
> +				return path
> +
> +		# Look for $CHOST-strip next.
> +		chost = environ.get('CHOST')
> +		if chost:
> +			path = find_binary('%s-%s' % (chost, tool))
> +			if path:
> +				return path
> +
> +		# Screw it, you just get `strip` now.
> +		return tool
> +
> +	@classmethod
> +	def cache(cls, environ=os.environ):
> +		for t in cls.TOOLS:
> +			setattr(cls, t, cls.find_toolchain_tool(t, environ))
> +
> +		cls._cache_strip(environ=environ)
> +		try:
> +			cls.run('debugedit', '--help', stdout=open(os.devnull, 'w'))

ResourceWarning

> +		except OSError as e:

Unused variable e

> +			Features.installsources = None
> +
> +	@classmethod
> +	def strip_type(cls):
> +		if not cls._strip_type:
> +			verinfo = subprocess.check_output([cls.strip, '--version'],
> +			                                  stderr=subprocess.STDOUT)
> +			verinfo = verinfo.split('\n', 1)[0]
> +			cls._strip_type = 'elfutils' if 'elfutils' in verinfo else 'GNU'
> +
> +		return cls._strip_type
> +
> +	@classmethod
> +	def _cache_strip(cls, environ):
> +		"""Handle various strip flags/behavior"""
> +		if cls.strip_type() == 'elfutils':
> +			cls._strip_flags = {
> +				'safe': '',
> +				'default': '--remove-comment',
> +			}
> +		elif cls.strip_type() == 'GNU':
> +			cls._strip_flags = {
> +				'safe': '--strip-unneeded',
> +				'default': '-R .comment -R .GCC.command.line -R .note.gnu.gold-version',
> +			}
> +		cls._strip_flags['debug'] = '-g'
> +		cls._strip_flags['portage'] = environ.get(
> +			'PORTAGE_STRIP_FLAGS', '%s %s' % (cls._strip_flags.get('safe', ''),
> +			                                  cls._strip_flags.get('default', '')))
> +
> +		for k, v in cls._strip_flags.iteritems():
> +			cls._strip_flags[k] = tuple(cls._strip_flags[k].split())
> +
> +	@classmethod
> +	def strip_flags(cls, strip_class):
> +		return cls._strip_flags[strip_class]
> +
> +	@classmethod
> +	def run(cls, tool, *args, **kwargs):
> +		cmd = [getattr(cls, tool)] + list(args)
> +		proc = subprocess.Popen(cmd, **kwargs)
> +		proc.wait()
> +		if proc.returncode:
> +			raise subprocess.CalledProcessError('command failed (ret=%i): %s' % (
> +				proc.returncode, ' '.join(map(repr, cmd))))

A generator expression would look better than map().

> +		return proc
> +
> +	@classmethod
> +	def run_strip(cls, strip_class, *args):
> +		# If stripping is disabled, then there's nothing for us to do.
> +		if Features.strip:
> +			return cls.run('strip', *(cls.strip_flags(strip_class) + args))
> +
> +	@classmethod
> +	def __str__(cls):
> +		return ' '.join('%s=%s' % (t, getattr(cls, t)) for t in cls.TOOLS)
> +
> +
> +class Qa(object):
> +	"""Object to hold (and cache) QA settings"""
> +
> +	QA_VARS = frozenset((
> +		'prestripped',
> +	))
> +
> +	_stripmask = []
> +	_qa_vars = {}
> +	_qa_re = {}
> +
> +	@classmethod
> +	def cache(cls, environ=os.environ):
> +		# Support an arch-specific override QA_XXX_${ARCH} for the QA_XXX var.
> +		# It's a regex, so precompile it here so we can just execute later on.
> +		arch = environ.get('ARCH')
> +		for v in cls.QA_VARS:
> +			val = None
> +			if arch:
> +				val = environ.get('QA_%s_%s' % (v.upper, arch))
> +			if val is None:
> +				val = environ.get('QA_%s' % (v.upper,))
> +			if val is None:
> +				val = ''
> +			val = val.split()
> +			cls._qa_vars[v] = val
> +			cls._qa_re[v] = re.compile(r'^(%s)$' % '|'.join(val))
> +
> +		# STRIP_MASK supports bash brace expansion (well, it supports all types
> +		# of expansion, but we only bother with brace).  People can also use
> +		# globs, but we let fnmatch handle that.  Paths should be anchored to /.
> +		brace_re = re.compile(r'([^{]*){([^}]*,[^}]*)}(.*)')
> +		stripmask = environ.get('STRIP_MASK', '')
> +
> +		def expand(expr):
> +			# This isn't terribly intelligent, but since the usage in the tree
> +			# is low (and limited to one or two expansions), we don't have to
> +			# worry about the pathological cases.
> +			m = brace_re.match(expr)
> +			if m:
> +				for x in m.group(2).split(','):
> +					expand(m.group(1) + x + m.group(3))
> +			else:
> +				cls._stripmask.append(expr)
> +
> +		for mask in stripmask.split():
> +			expand(mask)
> +
> +	@classmethod
> +	def val(cls, name):
> +		return cls._qa_vars.get(name, '')
> +
> +	@classmethod
> +	def regex(cls, name):
> +		return cls._qa_re.get(name)
> +
> +	@classmethod
> +	def stripmask(cls, path):
> +		for mask in cls._stripmask:
> +			if fnmatch.fnmatchcase(path, mask):
> +				return True
> +		return False
> +
> +
> +def CheckStripped(q_prestripped, elf, path):
> +	"""Warn about ELF files already stripped
> +
> +	The existance of the section .symtab tells us that a binary is stripped.

s/existance/existence/

> +	We want to log already stripped binaries, as this may be a QA violation.
> +	They prevent us from getting the splitdebug data.
> +	"""
> +	for section in elf.iter_sections():
> +		if section.name == '.symtab':
> +			return False
> +	else:
> +		# No .symtab!
> +		if q_prestripped:
> +			regex = Qa.regex('prestripped')
> +			if not regex.match(path):
> +				q_prestripped.put(path)
> +		return True
> +
> +
> +@××××××××××.contextmanager
> +def PreserveFileMetadata(path, mode=0600):
> +	"""Temporarily make |path| readable/writable if need be
> +
> +	In Prefix we are usually an unprivileged user, so we can't strip
> +	unwritable objects.  Make them temporarily writable for the
> +	stripping.
> +	"""
> +	st = os.stat(path)
> +	usable = ((st.st_mode & mode) == mode)
> +	if not usable:
> +		os.chmod(path, st.st_mode | mode)
> +	try:
> +		with preserve_xattrs(path):
> +			yield st.st_mode
> +	finally:
> +		if not usable:
> +			os.chmod(path, st.st_mode)
> +
> +
> +def MkdirP(path):
> +	"""Create |path|, but don't fail if it exists"""
> +	try:
> +		os.makedirs(path)
> +	except EnvironmentError as e:
> +		# We might be doing this in parallel, so don't complain
> +		# if another thread created the dir for us.
> +		if e.errno != errno.EEXIST or not os.path.isdir(path):
> +			raise
> +
> +
> +def Relink(src, dsts):
> +	"""Link all the |dsts| paths to |src|"""
> +	for dst in dsts:
> +		os.unlink(dst)
> +		os.link(src, dst)
> +
> +
> +def GetBuildId(elf):
> +	"""Extract the build id from |elf|"""
> +	# TODO: Should add PT_NOTE parsing.
> +	for section in elf.iter_sections():
> +		if section.name == '.note.gnu.build-id':
> +			return ''.join('%02x' % ord(x) for x in section.data()[16:])
> +
> +
> +def InstallSourcesProc():
> +	"""Launch a process for copying source files to the right place"""
> +	if not Features.installsources:
> +		return
> +
> +	d_sources_dir = '%s%s' % (Paths.D, Paths.SOURCES_DIR)
> +	MkdirP(d_sources_dir)
> +	proc = subprocess.Popen([
> +		'rsync', '-tL0', '--chmod=ugo-st,a+r,go-w,Da+x,Fa-x',
> +		'--files-from=-', Paths.WORKDIR, '%s/' % (d_sources_dir),
> +	], cwd=Paths.WORKDIR, stdin=subprocess.PIPE)
> +	setattr(proc, 'stdin_lock', multiprocessing.Lock())
> +	return proc
> +
> +
> +def RewriteElfSources(proc_installsources, path):
> +	"""Save the sources for this file"""
> +	if not Features.installsources:
> +		return
> +
> +	# Since we're editing the ELF here, we should recompute the build-id
> +	# (the -i flag below).  Save that output so we don't need to recompute
> +	# it later on in the save_elf_debug step.
> +	with tempfile.NamedTemporaryFile(dir=Paths.T) as tmpfile:
> +		proc = Tools.run('debugedit',
> +			'-i',
> +			'-b', Paths.WORKDIR,
> +			'-d', Paths.SOURCES_DIR,
> +			'-l', tmpfile.name,
> +			path, stdout=subprocess.PIPE)
> +		with open(tmpfile.name) as f:
> +			proc_installsources.stdin_lock.acquire()
> +			proc_installsources.stdin.write(f.read())
> +			proc_installsources.stdin_lock.release()
> +		return proc.stdout.read().strip()
> +
> +
> +def SaveElfDebug(elf, path, linked_paths, mode, buildid=None, splitdebug=None):
> +	"""Split off the debugging info for this file"""
> +	if not Features.splitdebug:
> +		return
> +
> +	# Don't save debug info twice.
> +	if os.path.splitext(path)[1] == '.debug':
> +		return
> +
> +	def _paths(p):
> +		root_path = p[len(Paths.D):]
> +		root_debug_path = '%s.debug' % (root_path)
> +		d_debug_path = '%s%s%s' % (Paths.ED, Paths.DEBUG_DIR, root_debug_path)
> +		MkdirP(os.path.dirname(d_debug_path))
> +		return (root_path, root_debug_path, d_debug_path)
> +
> +	root_path, root_debug_path, d_debug_path = _paths(path)
> +
> +	# Set up the .debug file in /usr/lib/debug/.
> +	if splitdebug:
> +		os.rename(splitdebug, d_debug_path)
> +	else:
> +		# Split out the .debug file.
> +		flags = ['--only-keep-debug']
> +		if Features.compressdebug:
> +			flags += ['--compress-debug-sections']
> +		flags += [path, d_debug_path]
> +		Tools.run('objcopy', *flags)
> +
> +		# Now link the ELF to the .debug file.  Strip out the
> +		# old section name in case there was one (otherwise
> +		# objcopy will crap itself).
> +		flags = [
> +			'--remove-section', '.gnu_debuglink',
> +			'--add-gnu-debuglink', d_debug_path,
> +			path,
> +		]
> +		Tools.run('objcopy', *flags)
> +
> +	# If we don't already have build-id from debugedit, look it up
> +	if not buildid:
> +		buildid = GetBuildId(elf)
> +	if buildid:
> +		buildid_dir = '%s%s/%s' % (Paths.ED, Paths.BUILDID_DIR, buildid[0:2])
> +		buildid_file = '%s/%s' % (buildid_dir, buildid[2:])
> +		MkdirP(buildid_dir)
> +		os.symlink('../../%s' % (root_debug_path.lstrip('/')),
> +		                         '%s.debug' % (buildid_file))
> +		os.symlink(root_debug_path, buildid_file)
> +
> +	# Create links for all the .debug files.
> +	for dst_path in linked_paths:
> +		_, _, dst_d_debug_path = _paths(dst_path)
> +		os.link(d_debug_path, dst_d_debug_path)
> +	# Make sure the .debug file has same perms as the original.
> +	os.chmod(d_debug_path, mode)
> +
> +
> +def _StripFile(q_stripped, proc_installsources, prestripped, elf, path,
> +               strip_class, linked_paths, mode):
> +	"""Do the actual stripping/splitdebug/etc..."""
> +	buildid = RewriteElfSources(proc_installsources, path)
> +
> +	if not prestripped:
> +		if Features.strip:
> +			q_stripped.put((path, ''))
> +		else:
> +			q_stripped.put((path, 'not stripping due to FEATURES=nostrip'))
> +
> +	# We don't copy xattrs from the source file to the splitdebug.
> +	# This is most likely what we want since the debug file is not
> +	# executable ...
> +
> +	# See if we can split & strip at the same time.
> +	if Tools.strip_type() == 'elfutils':
> +		splitdebug = tempfile.NamedTemporaryFile(dir=Paths.T)
> +		shortname = '%s.debug' % (os.path.basename(path),)
> +		args = [
> +			'-f', splitdebug.name,
> +			'-F', shortname,
> +			path,
> +		]
> +		if not prestripped:
> +			Tools.run_strip(strip_class, *args)
> +		SaveElfDebug(elf, path, linked_paths, mode, buildid=buildid, splitdebug=splitdebug.name)
> +
> +	else: # GNU
> +		SaveElfDebug(elf, path, linked_paths, mode, buildid=buildid)
> +		if not prestripped:
> +			Tools.run_strip(strip_class, path)
> +
> +
> +def StripFile(q_stripped, q_prestripped, proc_installsources, path, linked_paths):
> +	"""Strip |path|"""
> +	with PreserveFileMetadata(path) as mode:
> +		if path.endswith('.a'):
> +			# hmm, can we split debug/sources for .a ?
> +			q_stripped.put((path, ''))
> +			if Features.strip:
> +				Tools.run_strip(path, 'debug')
> +				Relink(path, linked_paths)
> +			return
> +
> +		with open(path, 'rb+') as f:
> +			# Make sure this open fd doesn't bleed into children (`strip`).
> +			fcntl.fcntl(f, fcntl.F_SETFD,
> +				fcntl.fcntl(f, fcntl.F_GETFD) | fcntl.FD_CLOEXEC)
> +			# Grab a lock on this file so we can handle hardlinks #421099.
> +			#fcntl.lockf(f, fcntl.LOCK_EX)
> +
> +			try:
> +				elf = ELFFile(f)
> +			except exceptions.ELFError:
> +				# Skip non-ELF files.
> +				return
> +
> +			# If it's already stripped, there's nothing for us to do.
> +			# Check it here before FEATURES for the QA aspect.
> +			prestripped = CheckStripped(q_prestripped, elf, path)
> +
> +			# First see if this thing has been masked.
> +			if Qa.stripmask(path):
> +				# Should pass down for sources saving ...
> +				q_stripped.put((path, 'skipped due to $STRIP_MASK'))
> +				for p in linked_paths:
> +					q_stripped.put((p, 'skipped due to hardlink to %s' % path))
> +				return
> +
> +			e_type = elf.header.e_type
> +			if e_type == 'ET_EXEC' or e_type == 'ET_DYN':

if e_type in ('ET_EXEC', 'ET_DYN'):

> +				strip_class = 'portage'
> +			elif e_type == 'ET_REL':
> +				strip_class = 'safe'
> +			else:
> +				strip_class = None
> +				q_stripped.put((path, 'unknown ELF type %s' % e_type))
> +
> +			if strip_class:
> +				_StripFile(q_stripped, proc_installsources, prestripped, elf,
> +				           path, strip_class, linked_paths, mode)
> +				Relink(path, linked_paths)
> +
> +
> +def ProcessFile(queue, hardlinks, path, ignore_symlink=True):
> +	"""Queue |path| for stripping"""
> +	# Now queue the file immediately if it has no hardlinks, else
> +	# delay it in the hardlinks dict for later processing.
> +	st = os.lstat(path)
> +	if not stat.S_ISLNK(st.st_mode) or not ignore_symlink:
> +		if st.st_nlink > 1:
> +			hardlinks.setdefault(st.st_ino, [])
> +			hardlinks[st.st_ino].append(path)
> +		else:
> +			queue.put((path, []))
> +
> +
> +def ProcessDir(queue, hardlinks, path):
> +	"""Queue all files found in |path| for stripping
> +
> +	Recursively descend into |path| and locate all files for stripping
> +	(ignoring symlinks and such).
> +	"""
> +	for root, _, files in os.walk(path, topdown=False):
> +		for f in files:
> +			ProcessFile(queue, hardlinks, os.path.join(root, f))
> +
> +
> +def ProcessPaths(queue, hardlinks, paths):
> +	"""Queue all files found in |paths| for stripping
> +
> +	This accepts both directories (which will be walked) and files.
> +	Symlinks to files are processed at this point.
> +	"""
> +	for p in paths:
> +		if os.path.isdir(p):
> +			ProcessDir(queue, hardlinks, p)
> +		else:
> +			ProcessFile(queue, hardlinks, p, ignore_symlink=False)
> +
> +
> +def Prepstrip(paths, jobs=None, out=None):
> +	"""Do the stripping on |paths| in parallel"""
> +	q_stripped = multiprocessing.Queue()
> +	q_prestripped = None
> +	if Features.binchecks:
> +		q_prestripped = multiprocessing.Queue()
> +
> +	proc_installsources = InstallSourcesProc()
> +
> +	# Now do the actual stripping.
> +	with parallel.BackgroundTaskRunner(StripFile, q_stripped, q_prestripped,
> +	                                   proc_installsources, processes=jobs) as queue:
> +		# First queue up all files that are not hardlinks and strip them
> +		# in the background.  Hardlinks will be processed specially.
> +		hardlinks = {}
> +		ProcessPaths(queue, hardlinks, paths)
> +
> +		# Since strip creates a new inode, we need to know the initial set of
> +		# inodes in advance, so that we can avoid interference due to trying
> +		# to strip the same (hardlinked) file multiple times in parallel.
> +		# See bug #421099.
> +		for paths in hardlinks.itervalues():
> +			queue.put((paths[0], paths[1:]))
> +
> +	# Print out the summary.
> +	stripped = []
> +	align = 0
> +	while not q_stripped.empty():
> +		path, reason = q_stripped.get()
> +		path = path[len(Paths.D) + 1:]
> +		align = max([align, len(path)])
> +		stripped.append((path, reason))
> +	if stripped:
> +		stripped.sort(key=lambda x: x[0])
> +		flags = ' '.join(_shell_quote(x) for x in Tools.strip_flags('portage'))
> +		print('%s: %s' % (Tools.strip, flags), file=out)
> +		for path, reason in stripped:
> +			if not reason:
> +				print('   %s' % path, file=out)
> +			else:
> +				print('   %-*s  # %s' % (align, path, reason), file=out)
> +
> +	prestripped = []
> +	if q_prestripped:
> +		while not q_prestripped.empty():
> +			prestripped.append(q_prestripped.get())
> +		prestripped.sort()
> +	if prestripped:
> +		eqawarn('QA Notice: Pre-stripped files found:', out=out)
> +		for p in prestripped:
> +			eqawarn(p, out=out)
> +
> +	if Features.installsources is None:
> +		ewarn('FEATURES=installsources is enabled but the debugedit binary could not', out=out)
> +		ewarn('be found. This feature will not work unless debugedit is installed!', out=out)
> +	elif Features.installsources:
> +		# Preserve directory structure.
> +		# Needed after running save_elf_sources.
> +		# https://bugzilla.redhat.com/show_bug.cgi?id=444310
> +		for root, dirs, files in os.walk('%s%s' % (Paths.D, Paths.SOURCES_DIR)):
> +			if not files and not dirs:
> +				open(os.path.join(root, '.keepdir'), 'w').close()
> +
> +		proc_installsources.stdin.close()
> +		proc_installsources.wait()
> +
> +
> +def main(argv, environ=os.environ, out=None):
> +	parser = ArgumentParser(description=__doc__)
> +	parser.add_argument('paths', nargs='*')
> +	parser.add_argument('-j', '--jobs', default=None, type=int,
> +		help='Number of jobs to run in parallel '
> +		     '(default: -j flag in $MAKEOPTS, else 1)')
> +	parser.add_argument('--clean-debugdir', default=False, action='store_true',
> +		help='Delete /usr/lib/debug first (useful for testing)')
> +	opts = parser.parse_args(argv)
> +
> +	Paths.cache(environ=environ)
> +	Features.cache(environ=environ)
> +	Tools.cache(environ=environ)
> +	Qa.cache(environ=environ)
> +
> +	if not opts.paths:
> +		opts.paths = [Paths.ED]
> +	if not opts.paths:
> +		parser.error('need some paths to strip')
> +
> +	if opts.jobs is None:
> +		# XXX: Use a common func for this.
> +		for flag in environ.get('MAKEOPTS', '').split():
> +			if flag.startswith('-j'):
> +				opts.jobs = int(flag[2:].strip())
> +				break
> +		else:
> +			opts.jobs = 1
> +
> +	if opts.clean_debugdir:
> +		for d in (Paths.SOURCES_DIR, Paths.DEBUG_DIR, Paths.BUILDID_DIR):
> +			shutil.rmtree('%s%s' % (Paths.ED, d), ignore_errors=True)
> +
> +	Prepstrip(opts.paths, jobs=opts.jobs, out=out)
> +
> +	return os.EX_OK
> diff --git a/pym/portage/tests/bin/test_prepstrip.py b/pym/portage/tests/bin/test_prepstrip.py
> new file mode 100644
> index 0000000..0bdff62
> --- /dev/null
> +++ b/pym/portage/tests/bin/test_prepstrip.py
> @@ -0,0 +1,253 @@
> +# test_prepstrip.py -- Portage Unit Testing Functionality
> +# Copyright 2007-2013 Gentoo Foundation
> +# Distributed under the terms of the GNU General Public License v2
> +
> +from copy import deepcopy
> +import cStringIO
> +import glob
> +import inspect
> +import shutil
> +import sys
> +
> +from portage import os
> +from portage.bin import prepstrip
> +from portage.process import find_binary
> +from portage.tests.bin.setup_env import BinTestCase, dobin, exists_in_D
> +from portage.tests import TestCase
> +
> +
> +class PrepStrip(BinTestCase):
> +	"""Simple/directed tests of the interface (as seen by ebuilds)"""
> +
> +	def testPrepStrip(self):
> +		self.init()
> +		try:
> +			dobin("/bin/bash")
> +			exists_in_D("/usr/bin/bash")
> +		finally:
> +			self.cleanup()
> +
> +
> +class PrepStripFull(TestCase):
> +	"""Full integration tests of the interface (as seen by ebuilds)"""
> +
> +	CATEGORY = 'cat'
> +	PN = 'somepackage'
> +	PV = '1.2.3'
> +	P = '%s-%s' % (PN, PV)
> +	PF = P
> +
> +	TESTDIR = os.path.realpath(__file__ + '/../testdir')
> +	WORKDIR = os.path.join(TESTDIR, 'work')
> +	S = os.path.join(WORKDIR, P)
> +	T = ''
> +	D = ''
> +
> +	# We'll join this to D during setup.
> +	DEBUG_DIR = 'usr/lib/debug'
> +	SOURCES_DIR = 'usr/src/debug'
> +
> +	def _setUp(self):
> +		"""Install the files to a test-specific root"""
> +		name = inspect.stack()[1][3]
> +		for v, d in (('D', 'image'), ('T', 'temp')):
> +			d = os.path.join(self.TESTDIR, '%s.%s' % (d, name))
> +			setattr(self, v, d)
> +			shutil.rmtree(d, ignore_errors=True)
> +			os.makedirs(d)
> +		for v in ('DEBUG_DIR', 'SOURCES_DIR'):
> +			setattr(self, v, os.path.join(self.D, getattr(self, v)))
> +		self._make('install', 'DESTDIR=%s' % self.D)
> +
> +	def _make(self, *args):
> +		"""Run make!"""
> +		cmd = (
> +			os.environ.get('MAKE', 'make'),
> +			'-s', '-C', self.S,
> +		) + args
> +		os.system(' '.join(cmd))
> +
> +	def _prepstrip(self, args, features='', restrict=''):
> +		"""Run prepstrip"""
> +		environ = {
> +			'MAKEOPTS': '-j1',
> +
> +			'CATEGORY': self.CATEGORY,
> +			'PN': self.PN,
> +			'PV': self.PV,
> +			'P': self.P,
> +			'PF': self.PF,
> +
> +			'WORKDIR': self.WORKDIR,
> +			'S': self.S,
> +			'D': self.D,
> +			'T': self.T,
> +
> +			'FEATURES': features,
> +			'RESTRICT': restrict,
> +		}
> +		output = cStringIO.StringIO()
> +		prepstrip.main(args, environ=environ, out=output)
> +		return output
> +
> +	def _sizes(self):
> +		d = os.path.join(self.D, 'bin')
> +		return [os.path.getsize(os.path.join(d, x)) for x in os.listdir(d)]
> +
> +	@staticmethod
> +	def _inode(path):
> +		"""Return the inode number for |path|"""
> +		return os.stat(path).st_ino
> +
> +	def _assertHardlinks(self, debugdir=False):
> +		"""Make sure hardlinks are still hardlinks"""
> +		inodes = set()
> +		dinodes = set()
> +		for sfx in ('', '-1', '-2'):
> +			p = os.path.join(self.D, 'bin', 'debug-hardlinked%s' % sfx)
> +			inodes.add(self._inode(p))
> +			if debugdir:
> +				p = os.path.join(self.DEBUG_DIR, 'bin', 'debug-hardlinked%s.debug' % sfx)
> +				dinodes.add(self._inode(p))
> +		self.assertEqual(len(inodes), 1)
> +		if debugdir:
> +			self.assertEqual(len(dinodes), 1)
> +
> +	def testStripSimple(self):
> +		"""Only strip objects"""
> +		self._setUp()
> +		before = self._sizes()
> +		output = self._prepstrip([])
> +		after = self._sizes()
> +		# Verify things were stripped by checking the file size.
> +		self.assertNotEqual(before, after)
> +		# We didn't split debug, so the dir should not exist.
> +		self.assertNotExists(self.DEBUG_DIR)
> +		# Make sure hardlinks didn't get messed up.
> +		self._assertHardlinks()
> +		# Verify QA pre-stripped check kicks in.
> +		self.assertIn('QA Notice: Pre-stripped', output.getvalue())
> +
> +	def testNoStrip(self):
> +		"""Verify FEATURES=nostrip behavior"""
> +		self._setUp()
> +		before = self._sizes()
> +		self._prepstrip([], features='nostrip')
> +		after = self._sizes()
> +		# Verify nothing was stripped by checking the file size.
> +		self.assertEqual(before, after)
> +		# Make sure hardlinks didn't get messed up.
> +		self._assertHardlinks()
> +
> +	def testNoBinChecks(self):
> +		"""Verify RESTRICT=binchecks behavior"""
> +		self._setUp()
> +		output = self._prepstrip([], restrict='binchecks')
> +		# Verify QA pre-stripped checks were skipped.
> +		self.assertNotIn('QA Notice: Pre-stripped', output.getvalue())
> +		# Make sure hardlinks didn't get messed up.
> +		self._assertHardlinks()
> +
> +	def testSplitdebug(self):
> +		"""Strip objects and check splitdebug"""
> +		self._setUp()
> +		self._prepstrip([], features='splitdebug')
> +		# Verify things got split.
> +		self.assertExists(os.path.join(self.DEBUG_DIR, 'bin', 'debug-unreadable.debug'))
> +		self.assertExists(os.path.join(self.DEBUG_DIR, '.build-id'))
> +		# Make sure hardlinks didn't get messed up.
> +		self._assertHardlinks(debugdir=True)
> +
> +	def testInstallSources(self):
> +		"""Strip objects and check sources"""
> +		self._setUp()
> +		self._prepstrip([], features='installsources')
> +		# We didn't split debug, so the dir should not exist.
> +		self.assertNotExists(self.DEBUG_DIR)
> +		# Verify sources got copied.
> +		self.assertExists(os.path.join(
> +			self.SOURCES_DIR, self.CATEGORY, self.PF, self.PF, 'src', 'main.c'))
> +		# Make sure hardlinks didn't get messed up.
> +		self._assertHardlinks()
> +
> +
> +class PrepStripApiFeatures(TestCase):
> +	"""Unittests for FEATURES logic"""
> +
> +	def _cache(self, features, env_features, env_restrict):
> +		features.cache(environ={
> +			'FEATURES': ' '.join(env_features),
> +			'RESTRICT': ' '.join(env_restrict),
> +		})
> +
> +	def testDefault(self):
> +		"""Verify default Features works"""
> +		features = deepcopy(prepstrip.Features)
> +		self._cache(features, [], [])
> +		self.assertTrue(features.binchecks)
> +		self.assertFalse(features.compressdebug)
> +		self.assertTrue(features.strip)
> +
> +	def testRestrict(self):
> +		"""Check RESTRICT handling"""
> +		features = deepcopy(prepstrip.Features)
> +
> +		self._cache(features, [], [])
> +		self.assertFalse(features.xattr)
> +		features.reset()
> +
> +		self._cache(features, ['xattr'], [])
> +		self.assertTrue(features.xattr)
> +		features.reset()
> +
> +		self._cache(features, ['xattr'], ['xattr'])
> +		self.assertFalse(features.xattr)
> +
> +	def testNegatives(self):
> +		"""Check handling of nostrip"""
> +		features = deepcopy(prepstrip.Features)
> +
> +		self._cache(features, ['strip'], [''])
> +		self.assertTrue(features.strip)
> +		features.reset()
> +
> +		self._cache(features, ['strip'], ['strip'])
> +		self.assertFalse(features.strip)
> +		features.reset()
> +
> +		self._cache(features, ['nostrip'], [''])
> +		self.assertFalse(features.strip)
> +
> +
> +class PrepStripApiTools(TestCase):
> +	"""Unittests for helper tool logic"""
> +
> +	def testDefault(self):
> +		"""Verify basic sanity"""
> +		tools = deepcopy(prepstrip.Tools)
> +		tools.cache(environ={})
> +		self.assertEqual(tools.strip, 'strip')
> +
> +	def testChost(self):
> +		"""Check looking up by CHOST prefix"""
> +		tools = deepcopy(prepstrip.Tools)
> +		objcopy = glob.glob('/usr/bin/*-objcopy')
> +		if not objcopy:
> +			# Maybe we should mock this stuff out.
> +			return
> +		objcopy = objcopy[0]
> +		tools.cache(environ={'CHOST': objcopy[:-8]})
> +		self.assertEqual(tools.objcopy, objcopy)
> +
> +	def testEnv(self):
> +		"""Check overriding by specific env var names"""
> +		tools = deepcopy(prepstrip.Tools)
> +		tools.cache(environ={'STRIP': 'true'})
> +		true = find_binary('true')
> +		self.assertEqual(tools.strip, true)
> +
> +	def testMissing(self):
> +		"""Check we get a sane value when user gives us crap"""
> +		tools = deepcopy(prepstrip.Tools)
> +		tools.cache(environ={'DEBUGEDIT': 'asldk19sdfj*!@af'})
> +		self.assertEqual(tools.debugedit, 'debugedit')
> diff --git a/pym/portage/tests/bin/testdir/.gitignore b/pym/portage/tests/bin/testdir/.gitignore
> new file mode 100644
> index 0000000..31dbb9d
> --- /dev/null
> +++ b/pym/portage/tests/bin/testdir/.gitignore
> @@ -0,0 +1,2 @@
> +image*/
> +temp*/
> diff --git a/pym/portage/tests/bin/testdir/work/somepackage-1.2.3/Makefile b/pym/portage/tests/bin/testdir/work/somepackage-1.2.3/Makefile
> new file mode 100644
> index 0000000..3e73f61
> --- /dev/null
> +++ b/pym/portage/tests/bin/testdir/work/somepackage-1.2.3/Makefile
> @@ -0,0 +1,65 @@
> +src = src/main.c
> +L = $(CC) $(CFLAGS) $(CPPFLAGS) $(LDFLAGS) -o $@ $<
> +
> +C_NO_DBG  = -g0
> +C_DBG     = -g -ggdb
> +C_STRIP   = -s
> +L_NO_BILD = -Wl,--build-id=none
> +L_BILD    = -Wl,--build-id
> +
> +_PROGS = \
> +	debug-buildid \
> +	debug-no-buildid \
> +	no-debug-buildid \
> +	no-debug-no-buildid \
> +	strip-buildid \
> +	strip-no-buildid \
> +	debug-hardlinked \
> +	debug-hardlinked-1 \
> +	debug-hardlinked-2 \
> +	debug-softlinked \
> +	debug-softlinked-1 \
> +	debug-softlinked-2 \
> +	debug-unreadable
> +PROGS = $(addprefix o/,$(_PROGS))
> +
> +all: $(PROGS)
> +clean:; rm -f o/*
> +%: $(src); $(L)
> +
> +o/debug-buildid:        CFLAGS += $(C_DBG)
> +o/debug-buildid:       LDFLAGS += $(L_BILD)
> +o/debug-no-buildid:     CFLAGS += $(C_DBG)
> +o/debug-no-buildid:    LDFLAGS += $(L_NO_BILD)
> +o/no-debug-buildid:     CFLAGS += $(C_NO_DBG)
> +o/no-debug-buildid:    LDFLAGS += $(L_BILD)
> +o/no-debug-no-buildid:  CFLAGS += $(C_NO_DBG)
> +o/no-debug-no-buildid: LDFLAGS += $(L_NO_BILD)
> +o/strip-buildid:        CFLAGS += $(C_STRIP)
> +o/strip-buildid:       LDFLAGS += $(L_BILD)
> +o/strip-no-buildid:     CFLAGS += $(C_STRIP)
> +o/strip-no-buildid:    LDFLAGS += $(L_NO_BILD)
> +
> +o/debug-hardlinked: CFLAGS += $(C_DBG)
> +o/debug-hardlinked-1: o/debug-hardlinked; ln -f $< $@
> +o/debug-hardlinked-2: o/debug-hardlinked; ln -f $< $@
> +o/debug-softlinked: CFLAGS += $(C_DBG)
> +o/debug-softlinked-1: o/debug-softlinked; ln -sf $(<F) $@
> +o/debug-softlinked-2: o/debug-softlinked; ln -sf $(<F) $@
> +
> +o/debug-unreadable: CFLAGS += $(C_DBG)
> +#debug-unreadable: $(src)
> +#	$(L)
> +#	chmod 000 $@
> +
> +#gnulink-debug-no-buildid
> +#--add-gnu-debuglink=path-to-file
> +
> +DESTDIR = $(PWD)/../../image
> +install: $(PROGS)
> +	rm -rf $(DESTDIR)
> +	mkdir -p $(DESTDIR)/bin
> +	rsync -aH $(PROGS) $(DESTDIR)/bin/
> +	chmod 000 $(DESTDIR)/bin/debug-unreadable
> +
> +.PHONY: all clean install
> diff --git a/pym/portage/tests/bin/testdir/work/somepackage-1.2.3/o/.gitignore b/pym/portage/tests/bin/testdir/work/somepackage-1.2.3/o/.gitignore
> new file mode 100644
> index 0000000..72e8ffc
> --- /dev/null
> +++ b/pym/portage/tests/bin/testdir/work/somepackage-1.2.3/o/.gitignore
> @@ -0,0 +1 @@
> +*
> diff --git a/pym/portage/tests/bin/testdir/work/somepackage-1.2.3/src/main.c b/pym/portage/tests/bin/testdir/work/somepackage-1.2.3/src/main.c
> new file mode 100644
> index 0000000..9989d20
> --- /dev/null
> +++ b/pym/portage/tests/bin/testdir/work/somepackage-1.2.3/src/main.c
> @@ -0,0 +1,5 @@
> +#include <stdio.h>
> +int main() {
> +	puts("hi");
> +	return 0;
> +}
> diff --git a/pym/portage/util/parallel.py b/pym/portage/util/parallel.py
> new file mode 100644
> index 0000000..068f0ae
> --- /dev/null
> +++ b/pym/portage/util/parallel.py
> @@ -0,0 +1,598 @@
> +# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
> +# Use of this source code is governed by a BSD-style license that can be
> +# found in the LICENSE file.
> +
> +"""Module for running cbuildbot stages in the background."""
> +
> +import collections
> +import contextlib
> +import errno
> +import functools
> +import logging
> +import multiprocessing
> +import os
> +import Queue
> +import signal
> +import sys
> +import tempfile
> +import time
> +import traceback
> +
> +
> +_BUFSIZE = 1024
> +
> +logger = logging.getLogger(__name__)
> +
> +
> +class BackgroundFailure(Exception):
> +  """A failure happened in the background"""
> +
> +
> +class _BackgroundTask(multiprocessing.Process):
> +  """Run a task in the background.
> +
> +  This task may be the 'Run' function from a buildbot stage or just a plain
> +  function. It will be run in the background. Output from this task is saved
> +  to a temporary file and is printed when the 'Wait' function is called.
> +  """
> +
> +  # The time we give Python to startup and exit.
> +  STARTUP_TIMEOUT = 60 * 5
> +  EXIT_TIMEOUT = 60 * 10
> +
> +  # The time we allow processes to be silent. This must be greater than the
> +  # hw_test_timeout set in cbuildbot_config.py, and less than the timeout set
> +  # by buildbot itself (typically, 150 minutes.)
> +  SILENT_TIMEOUT = 60 * 145
> +
> +  # The amount by which we reduce the SILENT_TIMEOUT every time we launch
> +  # a subprocess. This helps ensure that children get a chance to enforce the
> +  # SILENT_TIMEOUT prior to the parents enforcing it.
> +  SILENT_TIMEOUT_STEP = 30
> +  MINIMUM_SILENT_TIMEOUT = 60 * 135
> +
> +  # The time before terminating or killing a task.
> +  SIGTERM_TIMEOUT = 30
> +  SIGKILL_TIMEOUT = 60
> +
> +  # Interval we check for updates from print statements.
> +  PRINT_INTERVAL = 1
> +
> +  def __init__(self, task, semaphore=None, task_args=None, task_kwargs=None):
> +    """Create a new _BackgroundTask object.
> +
> +    If semaphore is supplied, it will be acquired for the duration of the
> +    steps that are run in the background. This can be used to limit the
> +    number of simultaneous parallel tasks.
> +
> +    Args:
> +      task: The task (a functor) to run in the background.
> +      semaphore: The lock to hold while |task| runs.
> +      task_args: A list of args to pass to the |task|.
> +      task_kwargs: A dict of optional args to pass to the |task|.
> +    """
> +    multiprocessing.Process.__init__(self)
> +    self._task = task
> +    self._queue = multiprocessing.Queue()
> +    self._semaphore = semaphore
> +    self._started = multiprocessing.Event()
> +    self._killing = multiprocessing.Event()
> +    self._output = None
> +    self._parent_pid = None
> +    self._task_args = task_args if task_args else ()
> +    self._task_kwargs = task_kwargs if task_kwargs else {}
> +
> +  def _WaitForStartup(self):
> +    # TODO(davidjames): Use python-2.7 syntax to simplify this.
> +    self._started.wait(self.STARTUP_TIMEOUT)
> +    msg = 'Process failed to start in %d seconds' % self.STARTUP_TIMEOUT
> +    assert self._started.is_set(), msg
> +
> +  def Kill(self, sig, log_level):
> +    """Kill process with signal, ignoring if the process is dead.
> +
> +    Args:
> +      sig: Signal to send.
> +      log_level: The log level of log messages.
> +    """
> +    self._killing.set()
> +    self._WaitForStartup()
> +    if logger.isEnabledFor(log_level):
> +      logger.log(log_level, 'Killing %r (sig=%r)', self.pid, sig)
> +
> +    try:
> +      os.kill(self.pid, sig)
> +    except OSError as ex:
> +      if ex.errno != errno.ESRCH:
> +        raise
> +
> +  def Cleanup(self, silent=False):
> +    """Wait for a process to exit."""
> +    if os.getpid() != self._parent_pid or self._output is None:
> +      return
> +    try:
> +      # Print output from subprocess.
> +      if not silent and logger.isEnabledFor(logging.DEBUG):
> +        with open(self._output.name, 'r') as f:
> +          for line in f:
> +            logging.debug(line.rstrip('\n'))
> +    finally:
> +      # Clean up our temporary file.
> +      os.unlink(self._output.name)
> +      self._output.close()
> +      self._output = None
> +
> +  def Wait(self):
> +    """Wait for the task to complete.
> +
> +    Output from the task is printed as it runs.
> +
> +    If an exception occurs, return a string containing the traceback.
> +    """
> +    try:
> +      # Flush stdout and stderr to be sure no output is interleaved.
> +      sys.stdout.flush()
> +      sys.stderr.flush()
> +
> +      # File position pointers are shared across processes, so we must open
> +      # our own file descriptor to ensure output is not lost.
> +      self._WaitForStartup()
> +      silent_death_time = time.time() + self.SILENT_TIMEOUT
> +      results = []
> +      with open(self._output.name, 'r') as output:
> +        pos = 0
> +        running, exited_cleanly, msg, error = (True, False, None, None)
> +        while running:
> +          # Check whether the process is still alive.
> +          running = self.is_alive()
> +
> +          try:
> +            error = self._queue.get(True, self.PRINT_INTERVAL)[0]
> +            running = False
> +            exited_cleanly = True
> +          except Queue.Empty:
> +            pass
> +
> +          if not running:
> +            # Wait for the process to actually exit. If the child doesn't exit
> +            # in a timely fashion, kill it.
> +            self.join(self.EXIT_TIMEOUT)
> +            if self.exitcode is None:
> +              msg = '%r hung for %r seconds' % (self, self.EXIT_TIMEOUT)
> +              self._KillChildren([self])
> +            elif not exited_cleanly:
> +              msg = ('%r exited unexpectedly with code %s' %
> +                     (self, self.EXIT_TIMEOUT))
> +          # Read output from process.
> +          output.seek(pos)
> +          buf = output.read(_BUFSIZE)
> +
> +          if len(buf) > 0:
> +            silent_death_time = time.time() + self.SILENT_TIMEOUT
> +          elif running and time.time() > silent_death_time:
> +            msg = ('No output from %r for %r seconds' %
> +                   (self, self.SILENT_TIMEOUT))
> +            self._KillChildren([self])
> +
> +            # Read remaining output from the process.
> +            output.seek(pos)
> +            buf = output.read(_BUFSIZE)
> +            running = False
> +
> +          # Print output so far.
> +          while len(buf) > 0:
> +            sys.stdout.write(buf)
> +            pos += len(buf)
> +            if len(buf) < _BUFSIZE:
> +              break
> +            buf = output.read(_BUFSIZE)
> +
> +          # Print error messages if anything exceptional occurred.
> +          if msg:
> +            error = '\n'.join(x for x in (error, msg) if x)
> +            logger.warning(error)
> +            traceback.print_stack()
> +
> +          sys.stdout.flush()
> +          sys.stderr.flush()
> +
> +    finally:
> +      self.Cleanup(silent=True)
> +
> +    # If a traceback occurred, return it.
> +    return error
> +
> +  def start(self):
> +    """Invoke multiprocessing.Process.start after flushing output/err."""
> +    if self.SILENT_TIMEOUT < self.MINIMUM_SILENT_TIMEOUT:
> +      raise AssertionError('Maximum recursion depth exceeded in %r' % self)
> +
> +    sys.stdout.flush()
> +    sys.stderr.flush()
> +    self._output = tempfile.NamedTemporaryFile(delete=False, bufsize=0,
> +                                               prefix='chromite-parallel-')
> +    self._parent_pid = os.getpid()
> +    return multiprocessing.Process.start(self)
> +
> +  def run(self):
> +    """Run the list of steps."""
> +    if self._semaphore is not None:
> +      self._semaphore.acquire()
> +
> +    error = 'Unexpected exception in %r' % self
> +    pid = os.getpid()
> +    try:
> +      error = self._Run()
> +    finally:
> +      if not self._killing.is_set() and os.getpid() == pid:
> +        self._queue.put((error,))
> +        if self._semaphore is not None:
> +          self._semaphore.release()
> +
> +  def _Run(self):
> +    """Internal method for running the list of steps."""
> +
> +    # The default handler for SIGINT sometimes forgets to actually raise the
> +    # exception (and we can reproduce this using unit tests), so we define a
> +    # custom one instead.
> +    def kill_us(_sig_num, _frame):
> +      raise KeyboardInterrupt('SIGINT received')
> +    signal.signal(signal.SIGINT, kill_us)
> +
> +    sys.stdout.flush()
> +    sys.stderr.flush()
> +    # Send all output to a named temporary file.
> +    with open(self._output.name, 'w', 0) as output:
> +      # Back up sys.std{err,out}. These aren't used, but we keep a copy so
> +      # that they aren't garbage collected. We intentionally don't restore
> +      # the old stdout and stderr at the end, because we want shutdown errors
> +      # to also be sent to the same log file.
> +      _orig_stdout, _orig_stderr = sys.stdout, sys.stderr
> +
> +      # Replace std{out,err} with unbuffered file objects.
> +      os.dup2(output.fileno(), sys.__stdout__.fileno())
> +      os.dup2(output.fileno(), sys.__stderr__.fileno())
> +      sys.stdout = os.fdopen(sys.__stdout__.fileno(), 'w', 0)
> +      sys.stderr = os.fdopen(sys.__stderr__.fileno(), 'w', 0)
> +
> +      error = None
> +      try:
> +        self._started.set()
> +
> +        # Reduce the silent timeout by the prescribed amount.
> +        cls = self.__class__
> +        cls.SILENT_TIMEOUT -= cls.SILENT_TIMEOUT_STEP
> +
> +        # Actually launch the task.
> +        self._task(*self._task_args, **self._task_kwargs)
> +      except BaseException as ex:

Unused variable ex

> +        error = traceback.format_exc()
> +        if self._killing.is_set():
> +          traceback.print_exc()
> +      finally:
> +        sys.stdout.flush()
> +        sys.stderr.flush()
> +
> +    return error
> +
> +  @classmethod
> +  def _KillChildren(cls, bg_tasks, log_level=logging.WARNING):
> +    """Kill a deque of background tasks.
> +
> +    This is needed to prevent hangs in the case where child processes refuse
> +    to exit.
> +
> +    Arguments:
> +      bg_tasks: A list filled with _BackgroundTask objects.
> +      log_level: The log level of log messages.
> +    """
> +    logger.log(log_level, 'Killing tasks: %r', bg_tasks)
> +    signals = ((signal.SIGINT, cls.SIGTERM_TIMEOUT),
> +               (signal.SIGTERM, cls.SIGKILL_TIMEOUT),
> +               (signal.SIGKILL, None))
> +    for sig, timeout in signals:
> +      # Send signal to all tasks.
> +      for task in bg_tasks:
> +        task.Kill(sig, log_level)
> +
> +      # Wait for all tasks to exit, if requested.
> +      if timeout is None:
> +        for task in bg_tasks:
> +          task.join()
> +          task.Cleanup()
> +        break
> +
> +      # Wait until timeout expires.
> +      end_time = time.time() + timeout
> +      while bg_tasks:
> +        time_left = end_time - time.time()
> +        if time_left <= 0:
> +          break
> +        task = bg_tasks[-1]
> +        task.join(time_left)
> +        if task.exitcode is not None:
> +          task.Cleanup()
> +          bg_tasks.pop()
> +
> +  @classmethod
> +  @contextlib.contextmanager
> +  def ParallelTasks(cls, steps, max_parallel=None, halt_on_error=False):
> +    """Run a list of functions in parallel.
> +
> +    This function launches the provided functions in the background, yields,
> +    and then waits for the functions to exit.
> +
> +    The output from the functions is saved to a temporary file and printed as if
> +    they were run in sequence.
> +
> +    If exceptions occur in the steps, we join together the tracebacks and print
> +    them after all parallel tasks have finished running. Further, a
> +    BackgroundFailure is raised with full stack traces of all exceptions.
> +
> +    Args:
> +      steps: A list of functions to run.
> +      max_parallel: The maximum number of simultaneous tasks to run in parallel.
> +        By default, run all tasks in parallel.
> +      halt_on_error: After the first exception occurs, halt any running steps,
> +        and squelch any further output, including any exceptions that might
> +        occur.
> +    """
> +
> +    semaphore = None
> +    if max_parallel is not None:
> +      semaphore = multiprocessing.Semaphore(max_parallel)
> +
> +    # First, start all the steps.
> +    bg_tasks = collections.deque()
> +    for step in steps:
> +      task = cls(step, semaphore=semaphore)
> +      task.start()
> +      bg_tasks.append(task)
> +
> +    try:
> +      yield
> +    finally:
> +      # Wait for each step to complete.
> +      tracebacks = []
> +      while bg_tasks:
> +        task = bg_tasks.popleft()
> +        error = task.Wait()
> +        if error is not None:
> +          tracebacks.append(error)
> +          if halt_on_error:
> +            break
> +
> +      # If there are still tasks left, kill them.
> +      if bg_tasks:
> +        cls._KillChildren(bg_tasks, log_level=logging.DEBUG)
> +
> +      # Propagate any exceptions.
> +      if tracebacks:
> +        raise BackgroundFailure('\n' + ''.join(tracebacks))
> +
> +  @staticmethod
> +  def TaskRunner(queue, task, onexit=None, task_args=None, task_kwargs=None):
> +    """Run task(*input) for each input in the queue.
> +
> +    Returns when it encounters an _AllTasksComplete object on the queue.
> +    If exceptions occur, save them off and re-raise them as a
> +    BackgroundFailure once we've finished processing the items in the queue.
> +
> +    Args:
> +      queue: A queue of tasks to run. Add tasks to this queue, and they will
> +        be run.
> +      task: Function to run on each queued input.
> +      onexit: Function to run after all inputs are processed.
> +      task_args: A list of args to pass to the |task|.
> +      task_kwargs: A dict of optional args to pass to the |task|.
> +    """
> +    if task_args is None:
> +      task_args = []
> +    elif not isinstance(task_args, list):
> +      task_args = list(task_args)
> +    if task_kwargs is None:
> +      task_kwargs = {}
> +
> +    tracebacks = []
> +    while True:
> +      # Wait for a new item to show up on the queue. This is a blocking wait,
> +      # so if there's nothing to do, we just sit here.
> +      x = queue.get()
> +      if isinstance(x, _AllTasksComplete):
> +        # All tasks are complete, so we should exit.
> +        break
> +      elif not isinstance(x, list):
> +        x = task_args + list(x)
> +      else:
> +        x = task_args + x
> +
> +      # If no tasks failed yet, process the remaining tasks.
> +      if not tracebacks:
> +        try:
> +          task(*x, **task_kwargs)
> +        except BaseException:
> +          tracebacks.append(traceback.format_exc())
> +
> +    # Run exit handlers.
> +    if onexit:
> +      onexit()
> +
> +    # Propagate any exceptions.
> +    if tracebacks:
> +      raise BackgroundFailure('\n' + ''.join(tracebacks))
> +
> +
> +def RunParallelSteps(steps, max_parallel=None, halt_on_error=False,
> +                     return_values=False):
> +  """Run a list of functions in parallel.
> +
> +  This function blocks until all steps are completed.
> +
> +  The output from the functions is saved to a temporary file and printed as if
> +  they were run in sequence.
> +
> +  If exceptions occur in the steps, we join together the tracebacks and print
> +  them after all parallel tasks have finished running. Further, a
> +  BackgroundFailure is raised with full stack traces of all exceptions.
> +
> +  Args:
> +    steps: A list of functions to run.
> +    max_parallel: The maximum number of simultaneous tasks to run in parallel.
> +      By default, run all tasks in parallel.
> +    halt_on_error: After the first exception occurs, halt any running steps,
> +      and squelch any further output, including any exceptions that might occur.
> +    return_values: If set to True, RunParallelSteps returns a list containing
> +      the return values of the steps.  Defaults to False.
> +
> +  Returns:
> +    If |return_values| is True, the function will return a list containing the
> +    return values of the steps.
> +
> +  Example:
> +    # This snippet will execute in parallel:
> +    #   somefunc()
> +    #   anotherfunc()
> +    #   funcfunc()
> +    steps = [somefunc, anotherfunc, funcfunc]
> +    RunParallelSteps(steps)
> +    # Blocks until all calls have completed.
> +  """
> +  def ReturnWrapper(queue, fn):
> +    """A function that """
> +    queue.put(fn())
> +
> +  full_steps = []
> +  queues = []
> +  manager = None
> +  if return_values:
> +    # We use a managed queue here, because the child process will wait for the
> +    # queue(pipe) to be flushed (i.e., when items are read from the queue)
> +    # before exiting, and with a regular queue this may result in hangs for
> +    # large return values.  But with a managed queue, the manager process will
> +    # read the items and hold on to them until the managed queue goes out of
> +    # scope and is cleaned up.
> +    manager = multiprocessing.Manager()
> +    for step in steps:
> +      # pylint: disable=E1101
> +      queue = manager.Queue()
> +      queues.append(queue)
> +      full_steps.append(functools.partial(ReturnWrapper, queue, step))
> +  else:
> +    full_steps = steps
> +
> +  with _BackgroundTask.ParallelTasks(full_steps, max_parallel=max_parallel,
> +                                     halt_on_error=halt_on_error):
> +    pass
> +
> +  if return_values:
> +    return [queue.get_nowait() for queue in queues]
> +
> +
> +class _AllTasksComplete(object):
> +  """Sentinel object to indicate that all tasks are complete."""
> +
> +
> +@××××××××××.contextmanager
> +def BackgroundTaskRunner(task, *args, **kwargs):
> +  """Run the specified task on each queued input in a pool of processes.
> +
> +  This context manager starts a set of workers in the background, who each
> +  wait for input on the specified queue. For each input on the queue, these
> +  workers run task(*args + *input, **kwargs). Note that certain kwargs will
> +  not pass through to the task (see Args below for the list).
> +
> +  The output from these tasks is saved to a temporary file. When control
> +  returns to the context manager, the background output is printed in order,
> +  as if the tasks were run in sequence.
> +
> +  If exceptions occur in the steps, we join together the tracebacks and print
> +  them after all parallel tasks have finished running. Further, a
> +  BackgroundFailure is raised with full stack traces of all exceptions.
> +
> +  Example:
> +    # This will run somefunc(1, 'small', 'cow', foo='bar' in the background
> +    # while "more random stuff" is being executed.
> +
> +    def somefunc(arg1, arg2, arg3, foo=None):
> +      ...
> +    ...
> +    with BackgroundTaskRunner(somefunc, 1, foo='bar') as queue:
> +      ... do random stuff ...
> +      queue.put(['small', 'cow'])
> +      ... do more random stuff ...
> +    # Exiting the with statement will block until all calls have completed.
> +
> +  Args:
> +    task: Function to run on each queued input.
> +    queue: A queue of tasks to run. Add tasks to this queue, and they will
> +      be run in the background.  If None, one will be created on the fly.
> +    processes: Number of processes to launch.
> +    onexit: Function to run in each background process after all inputs are
> +      processed.
> +  """
> +
> +  queue = kwargs.pop('queue', None)
> +  processes = kwargs.pop('processes', None)
> +  onexit = kwargs.pop('onexit', None)
> +
> +  if queue is None:
> +    queue = multiprocessing.Queue()
> +
> +  if not processes:
> +    processes = multiprocessing.cpu_count()
> +
> +  child = functools.partial(_BackgroundTask.TaskRunner, queue, task,
> +                            onexit=onexit, task_args=args,
> +                            task_kwargs=kwargs)
> +  steps = [child] * processes
> +  with _BackgroundTask.ParallelTasks(steps):
> +    try:
> +      yield queue
> +    finally:
> +      for _ in xrange(processes):
> +        queue.put(_AllTasksComplete())
> +
> +
> +def RunTasksInProcessPool(task, inputs, processes=None, onexit=None):
> +  """Run the specified function with each supplied input in a pool of processes.
> +
> +  This function runs task(*x) for x in inputs in a pool of processes. This
> +  function blocks until all tasks are completed.
> +
> +  The output from these tasks is saved to a temporary file. When control
> +  returns to the context manager, the background output is printed in order,
> +  as if the tasks were run in sequence.
> +
> +  If exceptions occur in the steps, we join together the tracebacks and print
> +  them after all parallel tasks have finished running. Further, a
> +  BackgroundFailure is raised with full stack traces of all exceptions.
> +
> +  Example:
> +    # This snippet will execute in parallel:
> +    #   somefunc('hi', 'fat', 'code')
> +    #   somefunc('foo', 'bar', 'cow')
> +
> +    def somefunc(arg1, arg2, arg3):
> +      ...
> +    ...
> +    inputs = [
> +      ['hi', 'fat', 'code'],
> +      ['foo', 'bar', 'cow'],
> +    ]
> +    RunTasksInProcessPool(somefunc, inputs)
> +    # Blocks until all calls have completed.
> +
> +  Args:
> +    task: Function to run on each input.
> +    inputs: List of inputs.
> +    processes: Number of processes, at most, to launch.
> +    onexit: Function to run in each background process after all inputs are
> +      processed.
> +  """
> +
> +  if not processes:
> +    processes = min(multiprocessing.cpu_count(), len(inputs))
> +
> +  with BackgroundTaskRunner(task, processes=processes, onexit=onexit) as queue:
> +    for x in inputs:
> +      queue.put(x)

--
Arfrever Frehtes Taifersar Arahesis