1 |
This rewrites the prestrip logic from scratch in python. The bash code |
2 |
was getting way too hairy, and the addition of parallel stripping was |
3 |
(still) racy. |
4 |
|
5 |
The upside is this has unittests :). |
6 |
|
7 |
The downside is that this adds a dep on pyelftools in portage. But I |
8 |
think that's fine as pax-utils has been pulling that in for some time. |
9 |
--- |
10 |
bin/ebuild-helpers/prepstrip | 387 +----------- |
11 |
pym/portage/bin/__init__.py | 0 |
12 |
pym/portage/bin/prepstrip.py | 682 +++++++++++++++++++++ |
13 |
pym/portage/tests/bin/test_prepstrip.py | 253 ++++++++ |
14 |
pym/portage/tests/bin/testdir/.gitignore | 2 + |
15 |
.../bin/testdir/work/somepackage-1.2.3/Makefile | 65 ++ |
16 |
.../testdir/work/somepackage-1.2.3/o/.gitignore | 1 + |
17 |
.../bin/testdir/work/somepackage-1.2.3/src/main.c | 5 + |
18 |
pym/portage/util/parallel.py | 598 ++++++++++++++++++ |
19 |
9 files changed, 1617 insertions(+), 376 deletions(-) |
20 |
create mode 100644 pym/portage/bin/__init__.py |
21 |
create mode 100644 pym/portage/bin/prepstrip.py |
22 |
create mode 100644 pym/portage/tests/bin/test_prepstrip.py |
23 |
create mode 100644 pym/portage/tests/bin/testdir/.gitignore |
24 |
create mode 100644 pym/portage/tests/bin/testdir/work/somepackage-1.2.3/Makefile |
25 |
create mode 100644 pym/portage/tests/bin/testdir/work/somepackage-1.2.3/o/.gitignore |
26 |
create mode 100644 pym/portage/tests/bin/testdir/work/somepackage-1.2.3/src/main.c |
27 |
create mode 100644 pym/portage/util/parallel.py |
28 |
|
29 |
diff --git a/bin/ebuild-helpers/prepstrip b/bin/ebuild-helpers/prepstrip |
30 |
index 9b2d47c..c8df5a5 100755 |
31 |
--- a/bin/ebuild-helpers/prepstrip |
32 |
+++ b/bin/ebuild-helpers/prepstrip |
33 |
@@ -1,386 +1,21 @@ |
34 |
-#!/bin/bash |
35 |
+#!/usr/bin/python |
36 |
# Copyright 1999-2013 Gentoo Foundation |
37 |
# Distributed under the terms of the GNU General Public License v2 |
38 |
|
39 |
-PORTAGE_PYM_PATH=${PORTAGE_PYM_PATH:-/usr/lib/portage/pym} |
40 |
-source "${PORTAGE_BIN_PATH:-/usr/lib/portage/bin}"/helper-functions.sh |
41 |
+"""Helper for stripping installed programs. |
42 |
|
43 |
-# avoid multiple calls to `has`. this creates things like: |
44 |
-# FEATURES_foo=false |
45 |
-# if "foo" is not in $FEATURES |
46 |
-tf() { "$@" && echo true || echo false ; } |
47 |
-exp_tf() { |
48 |
- local flag var=$1 |
49 |
- shift |
50 |
- for flag in "$@" ; do |
51 |
- eval ${var}_${flag}=$(tf has ${flag} ${!var}) |
52 |
- done |
53 |
-} |
54 |
-exp_tf FEATURES compressdebug installsources nostrip splitdebug xattr |
55 |
-exp_tf RESTRICT binchecks installsources splitdebug strip |
56 |
+This handles all the fun things in addition to stripping like splitdebug, |
57 |
+installsources, etc... |
58 |
|
59 |
-if ! ___eapi_has_prefix_variables; then |
60 |
- EPREFIX= ED=${D} |
61 |
-fi |
62 |
+If no paths are specified, then $D is searched. |
63 |
+""" |
64 |
|
65 |
-banner=false |
66 |
-SKIP_STRIP=false |
67 |
-if ${RESTRICT_strip} || ${FEATURES_nostrip} ; then |
68 |
- SKIP_STRIP=true |
69 |
- banner=true |
70 |
- ${FEATURES_installsources} || exit 0 |
71 |
-fi |
72 |
+from __future__ import print_function |
73 |
|
74 |
-PRESERVE_XATTR=false |
75 |
-if [[ ${KERNEL} == linux ]] && ${FEATURES_xattr} ; then |
76 |
- PRESERVE_XATTR=true |
77 |
- if type -P getfattr >/dev/null && type -P setfattr >/dev/null ; then |
78 |
- dump_xattrs() { |
79 |
- getfattr -d --absolute-names "$1" |
80 |
- } |
81 |
- restore_xattrs() { |
82 |
- setfattr --restore=- |
83 |
- } |
84 |
- else |
85 |
- dump_xattrs() { |
86 |
- PYTHONPATH=${PORTAGE_PYTHONPATH:-${PORTAGE_PYM_PATH}} \ |
87 |
- "${PORTAGE_PYTHON:-/usr/bin/python}" \ |
88 |
- "${PORTAGE_BIN_PATH}/xattr-helper.py" --dump < <(echo -n "$1") |
89 |
- } |
90 |
- restore_xattrs() { |
91 |
- PYTHONPATH=${PORTAGE_PYTHONPATH:-${PORTAGE_PYM_PATH}} \ |
92 |
- "${PORTAGE_PYTHON:-/usr/bin/python}" \ |
93 |
- "${PORTAGE_BIN_PATH}/xattr-helper.py" --restore |
94 |
- } |
95 |
- fi |
96 |
-fi |
97 |
+import sys |
98 |
|
99 |
-# look up the tools we might be using |
100 |
-for t in STRIP:strip OBJCOPY:objcopy READELF:readelf ; do |
101 |
- v=${t%:*} # STRIP |
102 |
- t=${t#*:} # strip |
103 |
- eval ${v}=\"${!v:-${CHOST}-${t}}\" |
104 |
- type -P -- ${!v} >/dev/null || eval ${v}=${t} |
105 |
-done |
106 |
+from portage.bin import prepstrip |
107 |
|
108 |
-# Figure out what tool set we're using to strip stuff |
109 |
-unset SAFE_STRIP_FLAGS DEF_STRIP_FLAGS SPLIT_STRIP_FLAGS |
110 |
-case $(${STRIP} --version 2>/dev/null) in |
111 |
-*elfutils*) # dev-libs/elfutils |
112 |
- # elfutils default behavior is always safe, so don't need to specify |
113 |
- # any flags at all |
114 |
- SAFE_STRIP_FLAGS="" |
115 |
- DEF_STRIP_FLAGS="--remove-comment" |
116 |
- SPLIT_STRIP_FLAGS="-f" |
117 |
- ;; |
118 |
-*GNU*) # sys-devel/binutils |
119 |
- # We'll leave out -R .note for now until we can check out the relevance |
120 |
- # of the section when it has the ALLOC flag set on it ... |
121 |
- SAFE_STRIP_FLAGS="--strip-unneeded" |
122 |
- DEF_STRIP_FLAGS="-R .comment -R .GCC.command.line -R .note.gnu.gold-version" |
123 |
- SPLIT_STRIP_FLAGS= |
124 |
- ;; |
125 |
-esac |
126 |
-: ${PORTAGE_STRIP_FLAGS=${SAFE_STRIP_FLAGS} ${DEF_STRIP_FLAGS}} |
127 |
|
128 |
-prepstrip_sources_dir=${EPREFIX}/usr/src/debug/${CATEGORY}/${PF} |
129 |
- |
130 |
-type -P debugedit >/dev/null && debugedit_found=true || debugedit_found=false |
131 |
-debugedit_warned=false |
132 |
- |
133 |
-__multijob_init |
134 |
- |
135 |
-# Setup $T filesystem layout that we care about. |
136 |
-tmpdir="${T}/prepstrip" |
137 |
-rm -rf "${tmpdir}" |
138 |
-mkdir -p "${tmpdir}"/{inodes,splitdebug,sources} |
139 |
- |
140 |
-# Usage: save_elf_sources <elf> |
141 |
-save_elf_sources() { |
142 |
- ${FEATURES_installsources} || return 0 |
143 |
- ${RESTRICT_installsources} && return 0 |
144 |
- if ! ${debugedit_found} ; then |
145 |
- if ! ${debugedit_warned} ; then |
146 |
- debugedit_warned=true |
147 |
- ewarn "FEATURES=installsources is enabled but the debugedit binary could not" |
148 |
- ewarn "be found. This feature will not work unless debugedit is installed!" |
149 |
- fi |
150 |
- return 0 |
151 |
- fi |
152 |
- |
153 |
- local x=$1 |
154 |
- |
155 |
- # since we're editing the ELF here, we should recompute the build-id |
156 |
- # (the -i flag below). save that output so we don't need to recompute |
157 |
- # it later on in the save_elf_debug step. |
158 |
- buildid=$(debugedit -i \ |
159 |
- -b "${WORKDIR}" \ |
160 |
- -d "${prepstrip_sources_dir}" \ |
161 |
- -l "${tmpdir}/sources/${x##*/}.${BASHPID}" \ |
162 |
- "${x}") |
163 |
-} |
164 |
- |
165 |
-# Usage: save_elf_debug <elf> [splitdebug file] |
166 |
-save_elf_debug() { |
167 |
- ${FEATURES_splitdebug} || return 0 |
168 |
- ${RESTRICT_splitdebug} && return 0 |
169 |
- |
170 |
- # NOTE: Debug files must be installed in |
171 |
- # ${EPREFIX}/usr/lib/debug/${EPREFIX} (note that ${EPREFIX} occurs |
172 |
- # twice in this path) in order for gdb's debug-file-directory |
173 |
- # lookup to work correctly. |
174 |
- local x=$1 |
175 |
- local inode_debug=$2 |
176 |
- local splitdebug=$3 |
177 |
- local y=${ED}usr/lib/debug/${x:${#D}}.debug |
178 |
- |
179 |
- # dont save debug info twice |
180 |
- [[ ${x} == *".debug" ]] && return 0 |
181 |
- |
182 |
- mkdir -p "${y%/*}" |
183 |
- |
184 |
- if [ -f "${inode_debug}" ] ; then |
185 |
- ln "${inode_debug}" "${y}" || die "ln failed unexpectedly" |
186 |
- else |
187 |
- if [[ -n ${splitdebug} ]] ; then |
188 |
- mv "${splitdebug}" "${y}" |
189 |
- else |
190 |
- local objcopy_flags="--only-keep-debug" |
191 |
- ${FEATURES_compressdebug} && objcopy_flags+=" --compress-debug-sections" |
192 |
- ${OBJCOPY} ${objcopy_flags} "${x}" "${y}" |
193 |
- ${OBJCOPY} --add-gnu-debuglink="${y}" "${x}" |
194 |
- fi |
195 |
- # Only do the following if the debug file was |
196 |
- # successfully created (see bug #446774). |
197 |
- if [ $? -eq 0 ] ; then |
198 |
- local args="a-x,o-w" |
199 |
- [[ -g ${x} || -u ${x} ]] && args+=",go-r" |
200 |
- chmod ${args} "${y}" |
201 |
- ln "${y}" "${inode_debug}" || die "ln failed unexpectedly" |
202 |
- fi |
203 |
- fi |
204 |
- |
205 |
- # if we don't already have build-id from debugedit, look it up |
206 |
- if [[ -z ${buildid} ]] ; then |
207 |
- # convert the readelf output to something useful |
208 |
- buildid=$(${READELF} -n "${x}" 2>/dev/null | awk '/Build ID:/{ print $NF; exit }') |
209 |
- fi |
210 |
- if [[ -n ${buildid} ]] ; then |
211 |
- local buildid_dir="${ED}usr/lib/debug/.build-id/${buildid:0:2}" |
212 |
- local buildid_file="${buildid_dir}/${buildid:2}" |
213 |
- mkdir -p "${buildid_dir}" |
214 |
- [ -L "${buildid_file}".debug ] || ln -s "../../${x:${#D}}.debug" "${buildid_file}.debug" |
215 |
- [ -L "${buildid_file}" ] || ln -s "/${x:${#D}}" "${buildid_file}" |
216 |
- fi |
217 |
-} |
218 |
- |
219 |
-# Usage: process_elf <elf> |
220 |
-process_elf() { |
221 |
- local x=$1 inode_link=$2 strip_flags=${*:3} |
222 |
- local already_stripped lockfile xt_data |
223 |
- |
224 |
- __vecho " ${x:${#ED}}" |
225 |
- |
226 |
- # If two processes try to debugedit or strip the same hardlink at the |
227 |
- # same time, it may corrupt files or cause loss of splitdebug info. |
228 |
- # So, use a lockfile to prevent interference (easily observed with |
229 |
- # dev-vcs/git which creates ~111 hardlinks to one file in |
230 |
- # /usr/libexec/git-core). |
231 |
- lockfile=${inode_link}_lockfile |
232 |
- if ! ln "${inode_link}" "${lockfile}" 2>/dev/null ; then |
233 |
- while [[ -f ${lockfile} ]] ; do |
234 |
- sleep 1 |
235 |
- done |
236 |
- unset lockfile |
237 |
- fi |
238 |
- |
239 |
- [ -f "${inode_link}_stripped" ] && already_stripped=true || already_stripped=false |
240 |
- |
241 |
- if ! ${already_stripped} ; then |
242 |
- if ${PRESERVE_XATTR} ; then |
243 |
- xt_data=$(dump_xattrs "${x}") |
244 |
- fi |
245 |
- save_elf_sources "${x}" |
246 |
- fi |
247 |
- |
248 |
- if ${strip_this} ; then |
249 |
- |
250 |
- # see if we can split & strip at the same time |
251 |
- if [[ -n ${SPLIT_STRIP_FLAGS} ]] ; then |
252 |
- local shortname="${x##*/}.debug" |
253 |
- local splitdebug="${tmpdir}/splitdebug/${shortname}.${BASHPID}" |
254 |
- ${already_stripped} || \ |
255 |
- ${STRIP} ${strip_flags} \ |
256 |
- -f "${splitdebug}" \ |
257 |
- -F "${shortname}" \ |
258 |
- "${x}" |
259 |
- save_elf_debug "${x}" "${inode_link}_debug" "${splitdebug}" |
260 |
- else |
261 |
- save_elf_debug "${x}" "${inode_link}_debug" |
262 |
- ${already_stripped} || \ |
263 |
- ${STRIP} ${strip_flags} "${x}" |
264 |
- fi |
265 |
- fi |
266 |
- |
267 |
- if ${already_stripped} ; then |
268 |
- rm -f "${x}" || die "rm failed unexpectedly" |
269 |
- ln "${inode_link}_stripped" "${x}" || die "ln failed unexpectedly" |
270 |
- else |
271 |
- ln "${x}" "${inode_link}_stripped" || die "ln failed unexpectedly" |
272 |
- if [[ ${xt_data} ]] ; then |
273 |
- restore_xattrs <<< "${xt_data}" |
274 |
- fi |
275 |
- fi |
276 |
- |
277 |
- [[ -n ${lockfile} ]] && rm -f "${lockfile}" |
278 |
-} |
279 |
- |
280 |
-# The existance of the section .symtab tells us that a binary is stripped. |
281 |
-# We want to log already stripped binaries, as this may be a QA violation. |
282 |
-# They prevent us from getting the splitdebug data. |
283 |
-if ! ${RESTRICT_binchecks} && ! ${RESTRICT_strip} ; then |
284 |
- # We need to do the non-stripped scan serially first before we turn around |
285 |
- # and start stripping the files ourselves. The log parsing can be done in |
286 |
- # parallel though. |
287 |
- log=${tmpdir}/scanelf-already-stripped.log |
288 |
- scanelf -yqRBF '#k%F' -k '!.symtab' "$@" | sed -e "s#^${ED}##" > "${log}" |
289 |
- ( |
290 |
- __multijob_child_init |
291 |
- qa_var="QA_PRESTRIPPED_${ARCH/-/_}" |
292 |
- [[ -n ${!qa_var} ]] && QA_PRESTRIPPED="${!qa_var}" |
293 |
- if [[ -n ${QA_PRESTRIPPED} && -s ${log} && \ |
294 |
- ${QA_STRICT_PRESTRIPPED-unset} = unset ]] ; then |
295 |
- shopts=$- |
296 |
- set -o noglob |
297 |
- for x in ${QA_PRESTRIPPED} ; do |
298 |
- sed -e "s#^${x#/}\$##" -i "${log}" |
299 |
- done |
300 |
- set +o noglob |
301 |
- set -${shopts} |
302 |
- fi |
303 |
- sed -e "/^\$/d" -e "s#^#/#" -i "${log}" |
304 |
- if [[ -s ${log} ]] ; then |
305 |
- __vecho -e "\n" |
306 |
- eqawarn "QA Notice: Pre-stripped files found:" |
307 |
- eqawarn "$(<"${log}")" |
308 |
- else |
309 |
- rm -f "${log}" |
310 |
- fi |
311 |
- ) & |
312 |
- __multijob_post_fork |
313 |
-fi |
314 |
- |
315 |
-# Since strip creates a new inode, we need to know the initial set of |
316 |
-# inodes in advance, so that we can avoid interference due to trying |
317 |
-# to strip the same (hardlinked) file multiple times in parallel. |
318 |
-# See bug #421099. |
319 |
-if [[ ${USERLAND} == BSD ]] ; then |
320 |
- get_inode_number() { stat -f '%i' "$1"; } |
321 |
-else |
322 |
- get_inode_number() { stat -c '%i' "$1"; } |
323 |
-fi |
324 |
-cd "${tmpdir}/inodes" || die "cd failed unexpectedly" |
325 |
-while read -r x ; do |
326 |
- inode_link=$(get_inode_number "${x}") || die "stat failed unexpectedly" |
327 |
- echo "${x}" >> "${inode_link}" || die "echo failed unexpectedly" |
328 |
-done < <( |
329 |
- # Use sort -u to eliminate duplicates for bug #445336. |
330 |
- ( |
331 |
- scanelf -yqRBF '#k%F' -k '.symtab' "$@" |
332 |
- find "$@" -type f ! -type l -name '*.a' |
333 |
- ) | LC_ALL=C sort -u |
334 |
-) |
335 |
- |
336 |
-# Now we look for unstripped binaries. |
337 |
-for inode_link in $(shopt -s nullglob; echo *) ; do |
338 |
-while read -r x |
339 |
-do |
340 |
- |
341 |
- if ! ${banner} ; then |
342 |
- __vecho "strip: ${STRIP} ${PORTAGE_STRIP_FLAGS}" |
343 |
- banner=true |
344 |
- fi |
345 |
- |
346 |
- ( |
347 |
- __multijob_child_init |
348 |
- f=$(file "${x}") || exit 0 |
349 |
- [[ -z ${f} ]] && exit 0 |
350 |
- |
351 |
- if ! ${SKIP_STRIP} ; then |
352 |
- # The noglob funk is to support STRIP_MASK="/*/booga" and to keep |
353 |
- # the for loop from expanding the globs. |
354 |
- # The eval echo is to support STRIP_MASK="/*/{booga,bar}" sex. |
355 |
- set -o noglob |
356 |
- strip_this=true |
357 |
- for m in $(eval echo ${STRIP_MASK}) ; do |
358 |
- [[ /${x#${ED}} == ${m} ]] && strip_this=false && break |
359 |
- done |
360 |
- set +o noglob |
361 |
- else |
362 |
- strip_this=false |
363 |
- fi |
364 |
- |
365 |
- # In Prefix we are usually an unprivileged user, so we can't strip |
366 |
- # unwritable objects. Make them temporarily writable for the |
367 |
- # stripping. |
368 |
- was_not_writable=false |
369 |
- if [[ ! -w ${x} ]] ; then |
370 |
- was_not_writable=true |
371 |
- chmod u+w "${x}" |
372 |
- fi |
373 |
- |
374 |
- # only split debug info for final linked objects |
375 |
- # or kernel modules as debuginfo for intermediatary |
376 |
- # files (think crt*.o from gcc/glibc) is useless and |
377 |
- # actually causes problems. install sources for all |
378 |
- # elf types though cause that stuff is good. |
379 |
- |
380 |
- buildid= |
381 |
- if [[ ${f} == *"current ar archive"* ]] ; then |
382 |
- __vecho " ${x:${#ED}}" |
383 |
- if ${strip_this} ; then |
384 |
- # hmm, can we split debug/sources for .a ? |
385 |
- ${STRIP} -g "${x}" |
386 |
- fi |
387 |
- elif [[ ${f} == *"SB executable"* || ${f} == *"SB shared object"* ]] ; then |
388 |
- process_elf "${x}" "${inode_link}" ${PORTAGE_STRIP_FLAGS} |
389 |
- elif [[ ${f} == *"SB relocatable"* ]] ; then |
390 |
- process_elf "${x}" "${inode_link}" ${SAFE_STRIP_FLAGS} |
391 |
- fi |
392 |
- |
393 |
- if ${was_not_writable} ; then |
394 |
- chmod u-w "${x}" |
395 |
- fi |
396 |
- ) & |
397 |
- __multijob_post_fork |
398 |
- |
399 |
-done < "${inode_link}" |
400 |
-done |
401 |
- |
402 |
-# With a bit more work, we could run the rsync processes below in |
403 |
-# parallel, but not sure that'd be an overall improvement. |
404 |
-__multijob_finish |
405 |
- |
406 |
-cd "${tmpdir}"/sources/ && cat * > "${tmpdir}/debug.sources" 2>/dev/null |
407 |
-if [[ -s ${tmpdir}/debug.sources ]] && \ |
408 |
- ${FEATURES_installsources} && \ |
409 |
- ! ${RESTRICT_installsources} && \ |
410 |
- ${debugedit_found} |
411 |
-then |
412 |
- __vecho "installsources: rsyncing source files" |
413 |
- [[ -d ${D}${prepstrip_sources_dir} ]] || mkdir -p "${D}${prepstrip_sources_dir}" |
414 |
- grep -zv '/<[^/>]*>$' "${tmpdir}"/debug.sources | \ |
415 |
- (cd "${WORKDIR}"; LANG=C sort -z -u | \ |
416 |
- rsync -tL0 --chmod=ugo-st,a+r,go-w,Da+x,Fa-x --files-from=- "${WORKDIR}/" "${D}${prepstrip_sources_dir}/" ) |
417 |
- |
418 |
- # Preserve directory structure. |
419 |
- # Needed after running save_elf_sources. |
420 |
- # https://bugzilla.redhat.com/show_bug.cgi?id=444310 |
421 |
- while read -r -d $'\0' emptydir |
422 |
- do |
423 |
- >> "${emptydir}"/.keepdir |
424 |
- done < <(find "${D}${prepstrip_sources_dir}/" -type d -empty -print0) |
425 |
-fi |
426 |
- |
427 |
-cd "${T}" |
428 |
-rm -rf "${tmpdir}" |
429 |
+if __name__ == '__main__': |
430 |
+ sys.exit(prepstrip.main(sys.argv[1:])) |
431 |
diff --git a/pym/portage/bin/__init__.py b/pym/portage/bin/__init__.py |
432 |
new file mode 100644 |
433 |
index 0000000..e69de29 |
434 |
diff --git a/pym/portage/bin/prepstrip.py b/pym/portage/bin/prepstrip.py |
435 |
new file mode 100644 |
436 |
index 0000000..0f6eb8d |
437 |
--- /dev/null |
438 |
+++ b/pym/portage/bin/prepstrip.py |
439 |
@@ -0,0 +1,682 @@ |
440 |
+#!/usr/bin/python |
441 |
+# Copyright 1999-2013 Gentoo Foundation |
442 |
+# Distributed under the terms of the GNU General Public License v2 |
443 |
+ |
444 |
+"""Helper code for stripping installed programs. |
445 |
+ |
446 |
+This handles all the fun things in addition to stripping like splitdebug, |
447 |
+installsources, etc... |
448 |
+""" |
449 |
+ |
450 |
+from __future__ import print_function |
451 |
+ |
452 |
+import contextlib |
453 |
+import errno |
454 |
+import fcntl |
455 |
+import fnmatch |
456 |
+import multiprocessing |
457 |
+import re |
458 |
+import shutil |
459 |
+import stat |
460 |
+import subprocess |
461 |
+import sys |
462 |
+import tempfile |
463 |
+ |
464 |
+from elftools.elf.elffile import ELFFile |
465 |
+from elftools.common import exceptions |
466 |
+ |
467 |
+from portage import os, _shell_quote |
468 |
+from portage.elog.messages import eqawarn, ewarn |
469 |
+from portage.process import find_binary |
470 |
+from portage.util import parallel, shlex |
471 |
+from portage.util._argparse import ArgumentParser |
472 |
+from portage.util.movefile import _copyxattr |
473 |
+from portage.util._xattr import preserve_xattrs, xattr |
474 |
+ |
475 |
+ |
476 |
+class Paths(object): |
477 |
+ """Object to hold (and cache) various paths""" |
478 |
+ |
479 |
+ _PATH_VARS = ('D', 'ED', 'EPREFIX', 'T', 'WORKDIR') |
480 |
+ _OTHER_VARS = ('CATEGORY', 'PN', 'PF') |
481 |
+ _VARS = _PATH_VARS + _OTHER_VARS |
482 |
+ D = '' |
483 |
+ ED = '' |
484 |
+ EPREFIX = '' |
485 |
+ CATEGORY = '' |
486 |
+ PN = '' |
487 |
+ PF = '' |
488 |
+ T = '' |
489 |
+ WORKDIR = '' |
490 |
+ SOURCES_DIR = '' |
491 |
+ DEBUG_DIR = '' |
492 |
+ BUILDID_DIR = '' |
493 |
+ |
494 |
+ @classmethod |
495 |
+ def cache(cls, environ=os.environ): |
496 |
+ for var in cls._VARS: |
497 |
+ val = environ.get(var, '') |
498 |
+ if var in cls._PATH_VARS: |
499 |
+ val = val.rstrip('/') |
500 |
+ setattr(cls, var, val) |
501 |
+ if cls.D and not cls.ED: |
502 |
+ cls.ED = cls.D |
503 |
+ if not cls.T: |
504 |
+ cls.T = tempfile.gettempdir() |
505 |
+ |
506 |
+ cls.SOURCES_DIR = os.path.normpath( |
507 |
+ '%s/usr/src/debug/%s/%s' % (cls.EPREFIX, cls.CATEGORY, cls.PF)) |
508 |
+ # NOTE: Debug files must be installed in |
509 |
+ # ${EPREFIX}/usr/lib/debug/${EPREFIX} (note that ${EPREFIX} occurs |
510 |
+ # twice in this path) in order for gdb's debug-file-directory |
511 |
+ # lookup to work correctly. |
512 |
+ cls.DEBUG_DIR = os.path.normpath('%s/usr/lib/debug' % (cls.EPREFIX)) |
513 |
+ cls.BUILDID_DIR = '%s/.build-id' % (cls.DEBUG_DIR) |
514 |
+ |
515 |
+ |
516 |
+class Features(object): |
517 |
+ """Object to hold (and cache) FEATURES availability |
518 |
+ |
519 |
+ Once we've been cached, you can simply do: |
520 |
+ if Features.strip: |
521 |
+ ... do strip stuff ... |
522 |
+ """ |
523 |
+ |
524 |
+ # Some features are always enabled even if not explicitly so. |
525 |
+ IMPLICIT_FEATURES = frozenset(( |
526 |
+ 'binchecks', |
527 |
+ 'strip', |
528 |
+ )) |
529 |
+ |
530 |
+ # These are the features we deal with in this file. |
531 |
+ FEATURES = frozenset(IMPLICIT_FEATURES | set(( |
532 |
+ 'compressdebug', |
533 |
+ 'installsources', |
534 |
+ 'splitdebug', |
535 |
+ 'xattr', |
536 |
+ ))) |
537 |
+ |
538 |
+ |
539 |
+ @classmethod |
540 |
+ def cache(cls, features=None, environ=os.environ): |
541 |
+ """Cache |features| tests to avoid processing at runtime""" |
542 |
+ if features is None: |
543 |
+ features = cls.FEATURES |
544 |
+ |
545 |
+ # Portage should have done the incremental thing for us already |
546 |
+ # so we don't have to handle it ourselves (like 'foo -foo foo'). |
547 |
+ env_features = environ.get('FEATURES', '').split() |
548 |
+ env_restrict = environ.get('RESTRICT', '').split() |
549 |
+ env_features += list(cls.IMPLICIT_FEATURES) |
550 |
+ |
551 |
+ for f in features: |
552 |
+ setattr(cls, f, f in env_features and f not in env_restrict) |
553 |
+ |
554 |
+ # Backwards compat support for "nostrip" and such. |
555 |
+ if 'no' + f in env_features: |
556 |
+ setattr(cls, f, False) |
557 |
+ |
558 |
+ @classmethod |
559 |
+ def reset(cls): |
560 |
+ for f in cls.FEATURES: |
561 |
+ delattr(cls, f) |
562 |
+ |
563 |
+ @classmethod |
564 |
+ def __str__(cls): |
565 |
+ return ' '.join('%s=%s' % (f, getattr(cls, f)) for f in cls.FEATURES) |
566 |
+ |
567 |
+ |
568 |
+class Tools(object): |
569 |
+ """Object to hold (and cache) toolchain tools that we'll need |
570 |
+ |
571 |
+ We also need to deal with things like env vars and compiler prefixes.""" |
572 |
+ |
573 |
+ TOOLS = frozenset(( |
574 |
+ 'debugedit', |
575 |
+ 'objcopy', |
576 |
+ 'strip', |
577 |
+ )) |
578 |
+ |
579 |
+ _strip_flags = {} |
580 |
+ _strip_type = None |
581 |
+ |
582 |
+ @staticmethod |
583 |
+ def find_toolchain_tool(tool, environ): |
584 |
+ """Given a toolchain |tool|, look it up via env vars |
585 |
+ |
586 |
+ e.g. We'll get "strip", so check for ${STRIP} and ${CHOST}-strip |
587 |
+ before falling back to "strip". |
588 |
+ """ |
589 |
+ # Look for $STRIP first. |
590 |
+ etool = environ.get(tool.upper()) |
591 |
+ if etool: |
592 |
+ path = find_binary(etool) |
593 |
+ if path: |
594 |
+ return path |
595 |
+ |
596 |
+ # Look for $CHOST-strip next. |
597 |
+ chost = environ.get('CHOST') |
598 |
+ if chost: |
599 |
+ path = find_binary('%s-%s' % (chost, tool)) |
600 |
+ if path: |
601 |
+ return path |
602 |
+ |
603 |
+ # Screw it, you just get `strip` now. |
604 |
+ return tool |
605 |
+ |
606 |
+ @classmethod |
607 |
+ def cache(cls, environ=os.environ): |
608 |
+ for t in cls.TOOLS: |
609 |
+ setattr(cls, t, cls.find_toolchain_tool(t, environ)) |
610 |
+ |
611 |
+ cls._cache_strip(environ=environ) |
612 |
+ try: |
613 |
+ cls.run('debugedit', '--help', stdout=open(os.devnull, 'w')) |
614 |
+ except OSError as e: |
615 |
+ Features.installsources = None |
616 |
+ |
617 |
+ @classmethod |
618 |
+ def strip_type(cls): |
619 |
+ if not cls._strip_type: |
620 |
+ verinfo = subprocess.check_output([cls.strip, '--version'], |
621 |
+ stderr=subprocess.STDOUT) |
622 |
+ verinfo = verinfo.split('\n', 1)[0] |
623 |
+ cls._strip_type = 'elfutils' if 'elfutils' in verinfo else 'GNU' |
624 |
+ |
625 |
+ return cls._strip_type |
626 |
+ |
627 |
+ @classmethod |
628 |
+ def _cache_strip(cls, environ): |
629 |
+ """Handle various strip flags/behavior""" |
630 |
+ if cls.strip_type() == 'elfutils': |
631 |
+ cls._strip_flags = { |
632 |
+ 'safe': '', |
633 |
+ 'default': '--remove-comment', |
634 |
+ } |
635 |
+ elif cls.strip_type() == 'GNU': |
636 |
+ cls._strip_flags = { |
637 |
+ 'safe': '--strip-unneeded', |
638 |
+ 'default': '-R .comment -R .GCC.command.line -R .note.gnu.gold-version', |
639 |
+ } |
640 |
+ cls._strip_flags['debug'] = '-g' |
641 |
+ cls._strip_flags['portage'] = environ.get( |
642 |
+ 'PORTAGE_STRIP_FLAGS', '%s %s' % (cls._strip_flags.get('safe', ''), |
643 |
+ cls._strip_flags.get('default', ''))) |
644 |
+ |
645 |
+ for k, v in cls._strip_flags.iteritems(): |
646 |
+ cls._strip_flags[k] = tuple(cls._strip_flags[k].split()) |
647 |
+ |
648 |
+ @classmethod |
649 |
+ def strip_flags(cls, strip_class): |
650 |
+ return cls._strip_flags[strip_class] |
651 |
+ |
652 |
+ @classmethod |
653 |
+ def run(cls, tool, *args, **kwargs): |
654 |
+ cmd = [getattr(cls, tool)] + list(args) |
655 |
+ proc = subprocess.Popen(cmd, **kwargs) |
656 |
+ proc.wait() |
657 |
+ if proc.returncode: |
658 |
+ raise subprocess.CalledProcessError('command failed (ret=%i): %s' % ( |
659 |
+ proc.returncode, ' '.join(map(repr, cmd)))) |
660 |
+ return proc |
661 |
+ |
662 |
+ @classmethod |
663 |
+ def run_strip(cls, strip_class, *args): |
664 |
+ # If stripping is disabled, then there's nothing for us to do. |
665 |
+ if Features.strip: |
666 |
+ return cls.run('strip', *(cls.strip_flags(strip_class) + args)) |
667 |
+ |
668 |
+ @classmethod |
669 |
+ def __str__(cls): |
670 |
+ return ' '.join('%s=%s' % (t, getattr(cls, t)) for t in cls.TOOLS) |
671 |
+ |
672 |
+ |
673 |
+class Qa(object): |
674 |
+ """Object to hold (and cache) QA settings""" |
675 |
+ |
676 |
+ QA_VARS = frozenset(( |
677 |
+ 'prestripped', |
678 |
+ )) |
679 |
+ |
680 |
+ _stripmask = [] |
681 |
+ _qa_vars = {} |
682 |
+ _qa_re = {} |
683 |
+ |
684 |
+ @classmethod |
685 |
+ def cache(cls, environ=os.environ): |
686 |
+ # Support an arch-specific override QA_XXX_${ARCH} for the QA_XXX var. |
687 |
+ # It's a regex, so precompile it here so we can just execute later on. |
688 |
+ arch = environ.get('ARCH') |
689 |
+ for v in cls.QA_VARS: |
690 |
+ val = None |
691 |
+ if arch: |
692 |
+ val = environ.get('QA_%s_%s' % (v.upper, arch)) |
693 |
+ if val is None: |
694 |
+ val = environ.get('QA_%s' % (v.upper,)) |
695 |
+ if val is None: |
696 |
+ val = '' |
697 |
+ val = val.split() |
698 |
+ cls._qa_vars[v] = val |
699 |
+ cls._qa_re[v] = re.compile(r'^(%s)$' % '|'.join(val)) |
700 |
+ |
701 |
+ # STRIP_MASK supports bash brace expansion (well, it supports all types |
702 |
+ # of expansion, but we only bother with brace). People can also use |
703 |
+ # globs, but we let fnmatch handle that. Paths should be anchored to /. |
704 |
+ brace_re = re.compile(r'([^{]*){([^}]*,[^}]*)}(.*)') |
705 |
+ stripmask = environ.get('STRIP_MASK', '') |
706 |
+ |
707 |
+ def expand(expr): |
708 |
+ # This isn't terribly intelligent, but since the usage in the tree |
709 |
+ # is low (and limited to one or two expansions), we don't have to |
710 |
+ # worry about the pathological cases. |
711 |
+ m = brace_re.match(expr) |
712 |
+ if m: |
713 |
+ for x in m.group(2).split(','): |
714 |
+ expand(m.group(1) + x + m.group(3)) |
715 |
+ else: |
716 |
+ cls._stripmask.append(expr) |
717 |
+ |
718 |
+ for mask in stripmask.split(): |
719 |
+ expand(mask) |
720 |
+ |
721 |
+ @classmethod |
722 |
+ def val(cls, name): |
723 |
+ return cls._qa_vars.get(name, '') |
724 |
+ |
725 |
+ @classmethod |
726 |
+ def regex(cls, name): |
727 |
+ return cls._qa_re.get(name) |
728 |
+ |
729 |
+ @classmethod |
730 |
+ def stripmask(cls, path): |
731 |
+ for mask in cls._stripmask: |
732 |
+ if fnmatch.fnmatchcase(path, mask): |
733 |
+ return True |
734 |
+ return False |
735 |
+ |
736 |
+ |
737 |
+def CheckStripped(q_prestripped, elf, path): |
738 |
+ """Warn about ELF files already stripped |
739 |
+ |
740 |
+ The existance of the section .symtab tells us that a binary is stripped. |
741 |
+ We want to log already stripped binaries, as this may be a QA violation. |
742 |
+ They prevent us from getting the splitdebug data. |
743 |
+ """ |
744 |
+ for section in elf.iter_sections(): |
745 |
+ if section.name == '.symtab': |
746 |
+ return False |
747 |
+ else: |
748 |
+ # No .symtab! |
749 |
+ if q_prestripped: |
750 |
+ regex = Qa.regex('prestripped') |
751 |
+ if not regex.match(path): |
752 |
+ q_prestripped.put(path) |
753 |
+ return True |
754 |
+ |
755 |
+ |
756 |
+@××××××××××.contextmanager |
757 |
+def PreserveFileMetadata(path, mode=0600): |
758 |
+ """Temporarily make |path| readable/writable if need be |
759 |
+ |
760 |
+ In Prefix we are usually an unprivileged user, so we can't strip |
761 |
+ unwritable objects. Make them temporarily writable for the |
762 |
+ stripping. |
763 |
+ """ |
764 |
+ st = os.stat(path) |
765 |
+ usable = ((st.st_mode & mode) == mode) |
766 |
+ if not usable: |
767 |
+ os.chmod(path, st.st_mode | mode) |
768 |
+ try: |
769 |
+ with preserve_xattrs(path): |
770 |
+ yield st.st_mode |
771 |
+ finally: |
772 |
+ if not usable: |
773 |
+ os.chmod(path, st.st_mode) |
774 |
+ |
775 |
+ |
776 |
+def MkdirP(path): |
777 |
+ """Create |path|, but don't fail if it exists""" |
778 |
+ try: |
779 |
+ os.makedirs(path) |
780 |
+ except EnvironmentError as e: |
781 |
+ # We might be doing this in parallel, so don't complain |
782 |
+ # if another thread created the dir for us. |
783 |
+ if e.errno != errno.EEXIST or not os.path.isdir(path): |
784 |
+ raise |
785 |
+ |
786 |
+ |
787 |
+def Relink(src, dsts): |
788 |
+ """Link all the |dsts| paths to |src|""" |
789 |
+ for dst in dsts: |
790 |
+ os.unlink(dst) |
791 |
+ os.link(src, dst) |
792 |
+ |
793 |
+ |
794 |
+def GetBuildId(elf): |
795 |
+ """Extract the build id from |elf|""" |
796 |
+ # TODO: Should add PT_NOTE parsing. |
797 |
+ for section in elf.iter_sections(): |
798 |
+ if section.name == '.note.gnu.build-id': |
799 |
+ return ''.join('%02x' % ord(x) for x in section.data()[16:]) |
800 |
+ |
801 |
+ |
802 |
+def InstallSourcesProc(): |
803 |
+ """Launch a process for copying source files to the right place""" |
804 |
+ if not Features.installsources: |
805 |
+ return |
806 |
+ |
807 |
+ d_sources_dir = '%s%s' % (Paths.D, Paths.SOURCES_DIR) |
808 |
+ MkdirP(d_sources_dir) |
809 |
+ proc = subprocess.Popen([ |
810 |
+ 'rsync', '-tL0', '--chmod=ugo-st,a+r,go-w,Da+x,Fa-x', |
811 |
+ '--files-from=-', Paths.WORKDIR, '%s/' % (d_sources_dir), |
812 |
+ ], cwd=Paths.WORKDIR, stdin=subprocess.PIPE) |
813 |
+ setattr(proc, 'stdin_lock', multiprocessing.Lock()) |
814 |
+ return proc |
815 |
+ |
816 |
+ |
817 |
+def RewriteElfSources(proc_installsources, path): |
818 |
+ """Save the sources for this file""" |
819 |
+ if not Features.installsources: |
820 |
+ return |
821 |
+ |
822 |
+ # Since we're editing the ELF here, we should recompute the build-id |
823 |
+ # (the -i flag below). Save that output so we don't need to recompute |
824 |
+ # it later on in the save_elf_debug step. |
825 |
+ with tempfile.NamedTemporaryFile(dir=Paths.T) as tmpfile: |
826 |
+ proc = Tools.run('debugedit', |
827 |
+ '-i', |
828 |
+ '-b', Paths.WORKDIR, |
829 |
+ '-d', Paths.SOURCES_DIR, |
830 |
+ '-l', tmpfile.name, |
831 |
+ path, stdout=subprocess.PIPE) |
832 |
+ with open(tmpfile.name) as f: |
833 |
+ proc_installsources.stdin_lock.acquire() |
834 |
+ proc_installsources.stdin.write(f.read()) |
835 |
+ proc_installsources.stdin_lock.release() |
836 |
+ return proc.stdout.read().strip() |
837 |
+ |
838 |
+ |
839 |
+def SaveElfDebug(elf, path, linked_paths, mode, buildid=None, splitdebug=None): |
840 |
+ """Split off the debugging info for this file""" |
841 |
+ if not Features.splitdebug: |
842 |
+ return |
843 |
+ |
844 |
+ # Don't save debug info twice. |
845 |
+ if os.path.splitext(path)[1] == '.debug': |
846 |
+ return |
847 |
+ |
848 |
+ def _paths(p): |
849 |
+ root_path = p[len(Paths.D):] |
850 |
+ root_debug_path = '%s.debug' % (root_path) |
851 |
+ d_debug_path = '%s%s%s' % (Paths.ED, Paths.DEBUG_DIR, root_debug_path) |
852 |
+ MkdirP(os.path.dirname(d_debug_path)) |
853 |
+ return (root_path, root_debug_path, d_debug_path) |
854 |
+ |
855 |
+ root_path, root_debug_path, d_debug_path = _paths(path) |
856 |
+ |
857 |
+ # Set up the .debug file in /usr/lib/debug/. |
858 |
+ if splitdebug: |
859 |
+ os.rename(splitdebug, d_debug_path) |
860 |
+ else: |
861 |
+ # Split out the .debug file. |
862 |
+ flags = ['--only-keep-debug'] |
863 |
+ if Features.compressdebug: |
864 |
+ flags += ['--compress-debug-sections'] |
865 |
+ flags += [path, d_debug_path] |
866 |
+ Tools.run('objcopy', *flags) |
867 |
+ |
868 |
+ # Now link the ELF to the .debug file. Strip out the |
869 |
+ # old section name in case there was one (otherwise |
870 |
+ # objcopy will crap itself). |
871 |
+ flags = [ |
872 |
+ '--remove-section', '.gnu_debuglink', |
873 |
+ '--add-gnu-debuglink', d_debug_path, |
874 |
+ path, |
875 |
+ ] |
876 |
+ Tools.run('objcopy', *flags) |
877 |
+ |
878 |
+ # If we don't already have build-id from debugedit, look it up |
879 |
+ if not buildid: |
880 |
+ buildid = GetBuildId(elf) |
881 |
+ if buildid: |
882 |
+ buildid_dir = '%s%s/%s' % (Paths.ED, Paths.BUILDID_DIR, buildid[0:2]) |
883 |
+ buildid_file = '%s/%s' % (buildid_dir, buildid[2:]) |
884 |
+ MkdirP(buildid_dir) |
885 |
+ os.symlink('../../%s' % (root_debug_path.lstrip('/')), |
886 |
+ '%s.debug' % (buildid_file)) |
887 |
+ os.symlink(root_debug_path, buildid_file) |
888 |
+ |
889 |
+ # Create links for all the .debug files. |
890 |
+ for dst_path in linked_paths: |
891 |
+ _, _, dst_d_debug_path = _paths(dst_path) |
892 |
+ os.link(d_debug_path, dst_d_debug_path) |
893 |
+ # Make sure the .debug file has same perms as the original. |
894 |
+ os.chmod(d_debug_path, mode) |
895 |
+ |
896 |
+ |
897 |
+def _StripFile(q_stripped, proc_installsources, prestripped, elf, path, |
898 |
+ strip_class, linked_paths, mode): |
899 |
+ """Do the actual stripping/splitdebug/etc...""" |
900 |
+ buildid = RewriteElfSources(proc_installsources, path) |
901 |
+ |
902 |
+ if not prestripped: |
903 |
+ if Features.strip: |
904 |
+ q_stripped.put((path, '')) |
905 |
+ else: |
906 |
+ q_stripped.put((path, 'not stripping due to FEATURES=nostrip')) |
907 |
+ |
908 |
+ # We don't copy xattrs from the source file to the splitdebug. |
909 |
+ # This is most likely what we want since the debug file is not |
910 |
+ # executable ... |
911 |
+ |
912 |
+ # See if we can split & strip at the same time. |
913 |
+ if Tools.strip_type() == 'elfutils': |
914 |
+ splitdebug = tempfile.NamedTemporaryFile(dir=Paths.T) |
915 |
+ shortname = '%s.debug' % (os.path.basename(path),) |
916 |
+ args = [ |
917 |
+ '-f', splitdebug.name, |
918 |
+ '-F', shortname, |
919 |
+ path, |
920 |
+ ] |
921 |
+ if not prestripped: |
922 |
+ Tools.run_strip(strip_class, *args) |
923 |
+ SaveElfDebug(elf, path, linked_paths, mode, buildid=buildid, splitdebug=splitdebug.name) |
924 |
+ |
925 |
+ else: # GNU |
926 |
+ SaveElfDebug(elf, path, linked_paths, mode, buildid=buildid) |
927 |
+ if not prestripped: |
928 |
+ Tools.run_strip(strip_class, path) |
929 |
+ |
930 |
+ |
931 |
+def StripFile(q_stripped, q_prestripped, proc_installsources, path, linked_paths): |
932 |
+ """Strip |path|""" |
933 |
+ with PreserveFileMetadata(path) as mode: |
934 |
+ if path.endswith('.a'): |
935 |
+ # hmm, can we split debug/sources for .a ? |
936 |
+ q_stripped.put((path, '')) |
937 |
+ if Features.strip: |
938 |
+ Tools.run_strip(path, 'debug') |
939 |
+ Relink(path, linked_paths) |
940 |
+ return |
941 |
+ |
942 |
+ with open(path, 'rb+') as f: |
943 |
+ # Make sure this open fd doesn't bleed into children (`strip`). |
944 |
+ fcntl.fcntl(f, fcntl.F_SETFD, |
945 |
+ fcntl.fcntl(f, fcntl.F_GETFD) | fcntl.FD_CLOEXEC) |
946 |
+ # Grab a lock on this file so we can handle hardlinks #421099. |
947 |
+ #fcntl.lockf(f, fcntl.LOCK_EX) |
948 |
+ |
949 |
+ try: |
950 |
+ elf = ELFFile(f) |
951 |
+ except exceptions.ELFError: |
952 |
+ # Skip non-ELF files. |
953 |
+ return |
954 |
+ |
955 |
+ # If it's already stripped, there's nothing for us to do. |
956 |
+ # Check it here before FEATURES for the QA aspect. |
957 |
+ prestripped = CheckStripped(q_prestripped, elf, path) |
958 |
+ |
959 |
+ # First see if this thing has been masked. |
960 |
+ if Qa.stripmask(path): |
961 |
+ # Should pass down for sources saving ... |
962 |
+ q_stripped.put((path, 'skipped due to $STRIP_MASK')) |
963 |
+ for p in linked_paths: |
964 |
+ q_stripped.put((p, 'skipped due to hardlink to %s' % path)) |
965 |
+ return |
966 |
+ |
967 |
+ e_type = elf.header.e_type |
968 |
+ if e_type == 'ET_EXEC' or e_type == 'ET_DYN': |
969 |
+ strip_class = 'portage' |
970 |
+ elif e_type == 'ET_REL': |
971 |
+ strip_class = 'safe' |
972 |
+ else: |
973 |
+ strip_class = None |
974 |
+ q_stripped.put((path, 'unknown ELF type %s' % e_type)) |
975 |
+ |
976 |
+ if strip_class: |
977 |
+ _StripFile(q_stripped, proc_installsources, prestripped, elf, |
978 |
+ path, strip_class, linked_paths, mode) |
979 |
+ Relink(path, linked_paths) |
980 |
+ |
981 |
+ |
982 |
+def ProcessFile(queue, hardlinks, path, ignore_symlink=True): |
983 |
+ """Queue |path| for stripping""" |
984 |
+ # Now queue the file immediately if it has no hardlinks, else |
985 |
+ # delay it in the hardlinks dict for later processing. |
986 |
+ st = os.lstat(path) |
987 |
+ if not stat.S_ISLNK(st.st_mode) or not ignore_symlink: |
988 |
+ if st.st_nlink > 1: |
989 |
+ hardlinks.setdefault(st.st_ino, []) |
990 |
+ hardlinks[st.st_ino].append(path) |
991 |
+ else: |
992 |
+ queue.put((path, [])) |
993 |
+ |
994 |
+ |
995 |
+def ProcessDir(queue, hardlinks, path): |
996 |
+ """Queue all files found in |path| for stripping |
997 |
+ |
998 |
+ Recursively descend into |path| and locate all files for stripping |
999 |
+ (ignoring symlinks and such). |
1000 |
+ """ |
1001 |
+ for root, _, files in os.walk(path, topdown=False): |
1002 |
+ for f in files: |
1003 |
+ ProcessFile(queue, hardlinks, os.path.join(root, f)) |
1004 |
+ |
1005 |
+ |
1006 |
+def ProcessPaths(queue, hardlinks, paths): |
1007 |
+ """Queue all files found in |paths| for stripping |
1008 |
+ |
1009 |
+ This accepts both directories (which will be walked) and files. |
1010 |
+ Symlinks to files are processed at this point. |
1011 |
+ """ |
1012 |
+ for p in paths: |
1013 |
+ if os.path.isdir(p): |
1014 |
+ ProcessDir(queue, hardlinks, p) |
1015 |
+ else: |
1016 |
+ ProcessFile(queue, hardlinks, p, ignore_symlink=False) |
1017 |
+ |
1018 |
+ |
1019 |
+def Prepstrip(paths, jobs=None, out=None): |
1020 |
+ """Do the stripping on |paths| in parallel""" |
1021 |
+ q_stripped = multiprocessing.Queue() |
1022 |
+ q_prestripped = None |
1023 |
+ if Features.binchecks: |
1024 |
+ q_prestripped = multiprocessing.Queue() |
1025 |
+ |
1026 |
+ proc_installsources = InstallSourcesProc() |
1027 |
+ |
1028 |
+ # Now do the actual stripping. |
1029 |
+ with parallel.BackgroundTaskRunner(StripFile, q_stripped, q_prestripped, |
1030 |
+ proc_installsources, processes=jobs) as queue: |
1031 |
+ # First queue up all files that are not hardlinks and strip them |
1032 |
+ # in the background. Hardlinks will be processed specially. |
1033 |
+ hardlinks = {} |
1034 |
+ ProcessPaths(queue, hardlinks, paths) |
1035 |
+ |
1036 |
+ # Since strip creates a new inode, we need to know the initial set of |
1037 |
+ # inodes in advance, so that we can avoid interference due to trying |
1038 |
+ # to strip the same (hardlinked) file multiple times in parallel. |
1039 |
+ # See bug #421099. |
1040 |
+ for paths in hardlinks.itervalues(): |
1041 |
+ queue.put((paths[0], paths[1:])) |
1042 |
+ |
1043 |
+ # Print out the summary. |
1044 |
+ stripped = [] |
1045 |
+ align = 0 |
1046 |
+ while not q_stripped.empty(): |
1047 |
+ path, reason = q_stripped.get() |
1048 |
+ path = path[len(Paths.D) + 1:] |
1049 |
+ align = max([align, len(path)]) |
1050 |
+ stripped.append((path, reason)) |
1051 |
+ if stripped: |
1052 |
+ stripped.sort(key=lambda x: x[0]) |
1053 |
+ flags = ' '.join(_shell_quote(x) for x in Tools.strip_flags('portage')) |
1054 |
+ print('%s: %s' % (Tools.strip, flags), file=out) |
1055 |
+ for path, reason in stripped: |
1056 |
+ if not reason: |
1057 |
+ print(' %s' % path, file=out) |
1058 |
+ else: |
1059 |
+ print(' %-*s # %s' % (align, path, reason), file=out) |
1060 |
+ |
1061 |
+ prestripped = [] |
1062 |
+ if q_prestripped: |
1063 |
+ while not q_prestripped.empty(): |
1064 |
+ prestripped.append(q_prestripped.get()) |
1065 |
+ prestripped.sort() |
1066 |
+ if prestripped: |
1067 |
+ eqawarn('QA Notice: Pre-stripped files found:', out=out) |
1068 |
+ for p in prestripped: |
1069 |
+ eqawarn(p, out=out) |
1070 |
+ |
1071 |
+ if Features.installsources is None: |
1072 |
+ ewarn('FEATURES=installsources is enabled but the debugedit binary could not', out=out) |
1073 |
+ ewarn('be found. This feature will not work unless debugedit is installed!', out=out) |
1074 |
+ elif Features.installsources: |
1075 |
+ # Preserve directory structure. |
1076 |
+ # Needed after running save_elf_sources. |
1077 |
+ # https://bugzilla.redhat.com/show_bug.cgi?id=444310 |
1078 |
+ for root, dirs, files in os.walk('%s%s' % (Paths.D, Paths.SOURCES_DIR)): |
1079 |
+ if not files and not dirs: |
1080 |
+ open(os.path.join(root, '.keepdir'), 'w').close() |
1081 |
+ |
1082 |
+ proc_installsources.stdin.close() |
1083 |
+ proc_installsources.wait() |
1084 |
+ |
1085 |
+ |
1086 |
+def main(argv, environ=os.environ, out=None): |
1087 |
+ parser = ArgumentParser(description=__doc__) |
1088 |
+ parser.add_argument('paths', nargs='*') |
1089 |
+ parser.add_argument('-j', '--jobs', default=None, type=int, |
1090 |
+ help='Number of jobs to run in parallel ' |
1091 |
+ '(default: -j flag in $MAKEOPTS, else 1)') |
1092 |
+ parser.add_argument('--clean-debugdir', default=False, action='store_true', |
1093 |
+ help='Delete /usr/lib/debug first (useful for testing)') |
1094 |
+ opts = parser.parse_args(argv) |
1095 |
+ |
1096 |
+ Paths.cache(environ=environ) |
1097 |
+ Features.cache(environ=environ) |
1098 |
+ Tools.cache(environ=environ) |
1099 |
+ Qa.cache(environ=environ) |
1100 |
+ |
1101 |
+ if not opts.paths: |
1102 |
+ opts.paths = [Paths.ED] |
1103 |
+ if not opts.paths: |
1104 |
+ parser.error('need some paths to strip') |
1105 |
+ |
1106 |
+ if opts.jobs is None: |
1107 |
+ # XXX: Use a common func for this. |
1108 |
+ for flag in environ.get('MAKEOPTS', '').split(): |
1109 |
+ if flag.startswith('-j'): |
1110 |
+ opts.jobs = int(flag[2:].strip()) |
1111 |
+ break |
1112 |
+ else: |
1113 |
+ opts.jobs = 1 |
1114 |
+ |
1115 |
+ if opts.clean_debugdir: |
1116 |
+ for d in (Paths.SOURCES_DIR, Paths.DEBUG_DIR, Paths.BUILDID_DIR): |
1117 |
+ shutil.rmtree('%s%s' % (Paths.ED, d), ignore_errors=True) |
1118 |
+ |
1119 |
+ Prepstrip(opts.paths, jobs=opts.jobs, out=out) |
1120 |
+ |
1121 |
+ return os.EX_OK |
1122 |
diff --git a/pym/portage/tests/bin/test_prepstrip.py b/pym/portage/tests/bin/test_prepstrip.py |
1123 |
new file mode 100644 |
1124 |
index 0000000..0bdff62 |
1125 |
--- /dev/null |
1126 |
+++ b/pym/portage/tests/bin/test_prepstrip.py |
1127 |
@@ -0,0 +1,253 @@ |
1128 |
+# test_prepstrip.py -- Portage Unit Testing Functionality |
1129 |
+# Copyright 2007-2013 Gentoo Foundation |
1130 |
+# Distributed under the terms of the GNU General Public License v2 |
1131 |
+ |
1132 |
+from copy import deepcopy |
1133 |
+import cStringIO |
1134 |
+import glob |
1135 |
+import inspect |
1136 |
+import shutil |
1137 |
+import sys |
1138 |
+ |
1139 |
+from portage import os |
1140 |
+from portage.bin import prepstrip |
1141 |
+from portage.process import find_binary |
1142 |
+from portage.tests.bin.setup_env import BinTestCase, dobin, exists_in_D |
1143 |
+from portage.tests import TestCase |
1144 |
+ |
1145 |
+ |
1146 |
+class PrepStrip(BinTestCase): |
1147 |
+ """Simple/directed tests of the interface (as seen by ebuilds)""" |
1148 |
+ |
1149 |
+ def testPrepStrip(self): |
1150 |
+ self.init() |
1151 |
+ try: |
1152 |
+ dobin("/bin/bash") |
1153 |
+ exists_in_D("/usr/bin/bash") |
1154 |
+ finally: |
1155 |
+ self.cleanup() |
1156 |
+ |
1157 |
+ |
1158 |
+class PrepStripFull(TestCase): |
1159 |
+ """Full integration tests of the interface (as seen by ebuilds)""" |
1160 |
+ |
1161 |
+ CATEGORY = 'cat' |
1162 |
+ PN = 'somepackage' |
1163 |
+ PV = '1.2.3' |
1164 |
+ P = '%s-%s' % (PN, PV) |
1165 |
+ PF = P |
1166 |
+ |
1167 |
+ TESTDIR = os.path.realpath(__file__ + '/../testdir') |
1168 |
+ WORKDIR = os.path.join(TESTDIR, 'work') |
1169 |
+ S = os.path.join(WORKDIR, P) |
1170 |
+ T = '' |
1171 |
+ D = '' |
1172 |
+ |
1173 |
+ # We'll join this to D during setup. |
1174 |
+ DEBUG_DIR = 'usr/lib/debug' |
1175 |
+ SOURCES_DIR = 'usr/src/debug' |
1176 |
+ |
1177 |
+ def _setUp(self): |
1178 |
+ """Install the files to a test-specific root""" |
1179 |
+ name = inspect.stack()[1][3] |
1180 |
+ for v, d in (('D', 'image'), ('T', 'temp')): |
1181 |
+ d = os.path.join(self.TESTDIR, '%s.%s' % (d, name)) |
1182 |
+ setattr(self, v, d) |
1183 |
+ shutil.rmtree(d, ignore_errors=True) |
1184 |
+ os.makedirs(d) |
1185 |
+ for v in ('DEBUG_DIR', 'SOURCES_DIR'): |
1186 |
+ setattr(self, v, os.path.join(self.D, getattr(self, v))) |
1187 |
+ self._make('install', 'DESTDIR=%s' % self.D) |
1188 |
+ |
1189 |
+ def _make(self, *args): |
1190 |
+ """Run make!""" |
1191 |
+ cmd = ( |
1192 |
+ os.environ.get('MAKE', 'make'), |
1193 |
+ '-s', '-C', self.S, |
1194 |
+ ) + args |
1195 |
+ os.system(' '.join(cmd)) |
1196 |
+ |
1197 |
+ def _prepstrip(self, args, features='', restrict=''): |
1198 |
+ """Run prepstrip""" |
1199 |
+ environ = { |
1200 |
+ 'MAKEOPTS': '-j1', |
1201 |
+ |
1202 |
+ 'CATEGORY': self.CATEGORY, |
1203 |
+ 'PN': self.PN, |
1204 |
+ 'PV': self.PV, |
1205 |
+ 'P': self.P, |
1206 |
+ 'PF': self.PF, |
1207 |
+ |
1208 |
+ 'WORKDIR': self.WORKDIR, |
1209 |
+ 'S': self.S, |
1210 |
+ 'D': self.D, |
1211 |
+ 'T': self.T, |
1212 |
+ |
1213 |
+ 'FEATURES': features, |
1214 |
+ 'RESTRICT': restrict, |
1215 |
+ } |
1216 |
+ output = cStringIO.StringIO() |
1217 |
+ prepstrip.main(args, environ=environ, out=output) |
1218 |
+ return output |
1219 |
+ |
1220 |
+ def _sizes(self): |
1221 |
+ d = os.path.join(self.D, 'bin') |
1222 |
+ return [os.path.getsize(os.path.join(d, x)) for x in os.listdir(d)] |
1223 |
+ |
1224 |
+ @staticmethod |
1225 |
+ def _inode(path): |
1226 |
+ """Return the inode number for |path|""" |
1227 |
+ return os.stat(path).st_ino |
1228 |
+ |
1229 |
+ def _assertHardlinks(self, debugdir=False): |
1230 |
+ """Make sure hardlinks are still hardlinks""" |
1231 |
+ inodes = set() |
1232 |
+ dinodes = set() |
1233 |
+ for sfx in ('', '-1', '-2'): |
1234 |
+ p = os.path.join(self.D, 'bin', 'debug-hardlinked%s' % sfx) |
1235 |
+ inodes.add(self._inode(p)) |
1236 |
+ if debugdir: |
1237 |
+ p = os.path.join(self.DEBUG_DIR, 'bin', 'debug-hardlinked%s.debug' % sfx) |
1238 |
+ dinodes.add(self._inode(p)) |
1239 |
+ self.assertEqual(len(inodes), 1) |
1240 |
+ if debugdir: |
1241 |
+ self.assertEqual(len(dinodes), 1) |
1242 |
+ |
1243 |
+ def testStripSimple(self): |
1244 |
+ """Only strip objects""" |
1245 |
+ self._setUp() |
1246 |
+ before = self._sizes() |
1247 |
+ output = self._prepstrip([]) |
1248 |
+ after = self._sizes() |
1249 |
+ # Verify things were stripped by checking the file size. |
1250 |
+ self.assertNotEqual(before, after) |
1251 |
+ # We didn't split debug, so the dir should not exist. |
1252 |
+ self.assertNotExists(self.DEBUG_DIR) |
1253 |
+ # Make sure hardlinks didn't get messed up. |
1254 |
+ self._assertHardlinks() |
1255 |
+ # Verify QA pre-stripped check kicks in. |
1256 |
+ self.assertIn('QA Notice: Pre-stripped', output.getvalue()) |
1257 |
+ |
1258 |
+ def testNoStrip(self): |
1259 |
+ """Verify FEATURES=nostrip behavior""" |
1260 |
+ self._setUp() |
1261 |
+ before = self._sizes() |
1262 |
+ self._prepstrip([], features='nostrip') |
1263 |
+ after = self._sizes() |
1264 |
+ # Verify nothing was stripped by checking the file size. |
1265 |
+ self.assertEqual(before, after) |
1266 |
+ # Make sure hardlinks didn't get messed up. |
1267 |
+ self._assertHardlinks() |
1268 |
+ |
1269 |
+ def testNoBinChecks(self): |
1270 |
+ """Verify RESTRICT=binchecks behavior""" |
1271 |
+ self._setUp() |
1272 |
+ output = self._prepstrip([], restrict='binchecks') |
1273 |
+ # Verify QA pre-stripped checks were skipped. |
1274 |
+ self.assertNotIn('QA Notice: Pre-stripped', output.getvalue()) |
1275 |
+ # Make sure hardlinks didn't get messed up. |
1276 |
+ self._assertHardlinks() |
1277 |
+ |
1278 |
+ def testSplitdebug(self): |
1279 |
+ """Strip objects and check splitdebug""" |
1280 |
+ self._setUp() |
1281 |
+ self._prepstrip([], features='splitdebug') |
1282 |
+ # Verify things got split. |
1283 |
+ self.assertExists(os.path.join(self.DEBUG_DIR, 'bin', 'debug-unreadable.debug')) |
1284 |
+ self.assertExists(os.path.join(self.DEBUG_DIR, '.build-id')) |
1285 |
+ # Make sure hardlinks didn't get messed up. |
1286 |
+ self._assertHardlinks(debugdir=True) |
1287 |
+ |
1288 |
+ def testInstallSources(self): |
1289 |
+ """Strip objects and check sources""" |
1290 |
+ self._setUp() |
1291 |
+ self._prepstrip([], features='installsources') |
1292 |
+ # We didn't split debug, so the dir should not exist. |
1293 |
+ self.assertNotExists(self.DEBUG_DIR) |
1294 |
+ # Verify sources got copied. |
1295 |
+ self.assertExists(os.path.join( |
1296 |
+ self.SOURCES_DIR, self.CATEGORY, self.PF, self.PF, 'src', 'main.c')) |
1297 |
+ # Make sure hardlinks didn't get messed up. |
1298 |
+ self._assertHardlinks() |
1299 |
+ |
1300 |
+ |
1301 |
+class PrepStripApiFeatures(TestCase): |
1302 |
+ """Unittests for FEATURES logic""" |
1303 |
+ |
1304 |
+ def _cache(self, features, env_features, env_restrict): |
1305 |
+ features.cache(environ={ |
1306 |
+ 'FEATURES': ' '.join(env_features), |
1307 |
+ 'RESTRICT': ' '.join(env_restrict), |
1308 |
+ }) |
1309 |
+ |
1310 |
+ def testDefault(self): |
1311 |
+ """Verify default Features works""" |
1312 |
+ features = deepcopy(prepstrip.Features) |
1313 |
+ self._cache(features, [], []) |
1314 |
+ self.assertTrue(features.binchecks) |
1315 |
+ self.assertFalse(features.compressdebug) |
1316 |
+ self.assertTrue(features.strip) |
1317 |
+ |
1318 |
+ def testRestrict(self): |
1319 |
+ """Check RESTRICT handling""" |
1320 |
+ features = deepcopy(prepstrip.Features) |
1321 |
+ |
1322 |
+ self._cache(features, [], []) |
1323 |
+ self.assertFalse(features.xattr) |
1324 |
+ features.reset() |
1325 |
+ |
1326 |
+ self._cache(features, ['xattr'], []) |
1327 |
+ self.assertTrue(features.xattr) |
1328 |
+ features.reset() |
1329 |
+ |
1330 |
+ self._cache(features, ['xattr'], ['xattr']) |
1331 |
+ self.assertFalse(features.xattr) |
1332 |
+ |
1333 |
+ def testNegatives(self): |
1334 |
+ """Check handling of nostrip""" |
1335 |
+ features = deepcopy(prepstrip.Features) |
1336 |
+ |
1337 |
+ self._cache(features, ['strip'], ['']) |
1338 |
+ self.assertTrue(features.strip) |
1339 |
+ features.reset() |
1340 |
+ |
1341 |
+ self._cache(features, ['strip'], ['strip']) |
1342 |
+ self.assertFalse(features.strip) |
1343 |
+ features.reset() |
1344 |
+ |
1345 |
+ self._cache(features, ['nostrip'], ['']) |
1346 |
+ self.assertFalse(features.strip) |
1347 |
+ |
1348 |
+ |
1349 |
+class PrepStripApiTools(TestCase): |
1350 |
+ """Unittests for helper tool logic""" |
1351 |
+ |
1352 |
+ def testDefault(self): |
1353 |
+ """Verify basic sanity""" |
1354 |
+ tools = deepcopy(prepstrip.Tools) |
1355 |
+ tools.cache(environ={}) |
1356 |
+ self.assertEqual(tools.strip, 'strip') |
1357 |
+ |
1358 |
+ def testChost(self): |
1359 |
+ """Check looking up by CHOST prefix""" |
1360 |
+ tools = deepcopy(prepstrip.Tools) |
1361 |
+ objcopy = glob.glob('/usr/bin/*-objcopy') |
1362 |
+ if not objcopy: |
1363 |
+ # Maybe we should mock this stuff out. |
1364 |
+ return |
1365 |
+ objcopy = objcopy[0] |
1366 |
+ tools.cache(environ={'CHOST': objcopy[:-8]}) |
1367 |
+ self.assertEqual(tools.objcopy, objcopy) |
1368 |
+ |
1369 |
+ def testEnv(self): |
1370 |
+ """Check overriding by specific env var names""" |
1371 |
+ tools = deepcopy(prepstrip.Tools) |
1372 |
+ tools.cache(environ={'STRIP': 'true'}) |
1373 |
+ true = find_binary('true') |
1374 |
+ self.assertEqual(tools.strip, true) |
1375 |
+ |
1376 |
+ def testMissing(self): |
1377 |
+ """Check we get a sane value when user gives us crap""" |
1378 |
+ tools = deepcopy(prepstrip.Tools) |
1379 |
+ tools.cache(environ={'DEBUGEDIT': 'asldk19sdfj*!@af'}) |
1380 |
+ self.assertEqual(tools.debugedit, 'debugedit') |
1381 |
diff --git a/pym/portage/tests/bin/testdir/.gitignore b/pym/portage/tests/bin/testdir/.gitignore |
1382 |
new file mode 100644 |
1383 |
index 0000000..31dbb9d |
1384 |
--- /dev/null |
1385 |
+++ b/pym/portage/tests/bin/testdir/.gitignore |
1386 |
@@ -0,0 +1,2 @@ |
1387 |
+image*/ |
1388 |
+temp*/ |
1389 |
diff --git a/pym/portage/tests/bin/testdir/work/somepackage-1.2.3/Makefile b/pym/portage/tests/bin/testdir/work/somepackage-1.2.3/Makefile |
1390 |
new file mode 100644 |
1391 |
index 0000000..3e73f61 |
1392 |
--- /dev/null |
1393 |
+++ b/pym/portage/tests/bin/testdir/work/somepackage-1.2.3/Makefile |
1394 |
@@ -0,0 +1,65 @@ |
1395 |
+src = src/main.c |
1396 |
+L = $(CC) $(CFLAGS) $(CPPFLAGS) $(LDFLAGS) -o $@ $< |
1397 |
+ |
1398 |
+C_NO_DBG = -g0 |
1399 |
+C_DBG = -g -ggdb |
1400 |
+C_STRIP = -s |
1401 |
+L_NO_BILD = -Wl,--build-id=none |
1402 |
+L_BILD = -Wl,--build-id |
1403 |
+ |
1404 |
+_PROGS = \ |
1405 |
+ debug-buildid \ |
1406 |
+ debug-no-buildid \ |
1407 |
+ no-debug-buildid \ |
1408 |
+ no-debug-no-buildid \ |
1409 |
+ strip-buildid \ |
1410 |
+ strip-no-buildid \ |
1411 |
+ debug-hardlinked \ |
1412 |
+ debug-hardlinked-1 \ |
1413 |
+ debug-hardlinked-2 \ |
1414 |
+ debug-softlinked \ |
1415 |
+ debug-softlinked-1 \ |
1416 |
+ debug-softlinked-2 \ |
1417 |
+ debug-unreadable |
1418 |
+PROGS = $(addprefix o/,$(_PROGS)) |
1419 |
+ |
1420 |
+all: $(PROGS) |
1421 |
+clean:; rm -f o/* |
1422 |
+%: $(src); $(L) |
1423 |
+ |
1424 |
+o/debug-buildid: CFLAGS += $(C_DBG) |
1425 |
+o/debug-buildid: LDFLAGS += $(L_BILD) |
1426 |
+o/debug-no-buildid: CFLAGS += $(C_DBG) |
1427 |
+o/debug-no-buildid: LDFLAGS += $(L_NO_BILD) |
1428 |
+o/no-debug-buildid: CFLAGS += $(C_NO_DBG) |
1429 |
+o/no-debug-buildid: LDFLAGS += $(L_BILD) |
1430 |
+o/no-debug-no-buildid: CFLAGS += $(C_NO_DBG) |
1431 |
+o/no-debug-no-buildid: LDFLAGS += $(L_NO_BILD) |
1432 |
+o/strip-buildid: CFLAGS += $(C_STRIP) |
1433 |
+o/strip-buildid: LDFLAGS += $(L_BILD) |
1434 |
+o/strip-no-buildid: CFLAGS += $(C_STRIP) |
1435 |
+o/strip-no-buildid: LDFLAGS += $(L_NO_BILD) |
1436 |
+ |
1437 |
+o/debug-hardlinked: CFLAGS += $(C_DBG) |
1438 |
+o/debug-hardlinked-1: o/debug-hardlinked; ln -f $< $@ |
1439 |
+o/debug-hardlinked-2: o/debug-hardlinked; ln -f $< $@ |
1440 |
+o/debug-softlinked: CFLAGS += $(C_DBG) |
1441 |
+o/debug-softlinked-1: o/debug-softlinked; ln -sf $(<F) $@ |
1442 |
+o/debug-softlinked-2: o/debug-softlinked; ln -sf $(<F) $@ |
1443 |
+ |
1444 |
+o/debug-unreadable: CFLAGS += $(C_DBG) |
1445 |
+#debug-unreadable: $(src) |
1446 |
+# $(L) |
1447 |
+# chmod 000 $@ |
1448 |
+ |
1449 |
+#gnulink-debug-no-buildid |
1450 |
+#--add-gnu-debuglink=path-to-file |
1451 |
+ |
1452 |
+DESTDIR = $(PWD)/../../image |
1453 |
+install: $(PROGS) |
1454 |
+ rm -rf $(DESTDIR) |
1455 |
+ mkdir -p $(DESTDIR)/bin |
1456 |
+ rsync -aH $(PROGS) $(DESTDIR)/bin/ |
1457 |
+ chmod 000 $(DESTDIR)/bin/debug-unreadable |
1458 |
+ |
1459 |
+.PHONY: all clean install |
1460 |
diff --git a/pym/portage/tests/bin/testdir/work/somepackage-1.2.3/o/.gitignore b/pym/portage/tests/bin/testdir/work/somepackage-1.2.3/o/.gitignore |
1461 |
new file mode 100644 |
1462 |
index 0000000..72e8ffc |
1463 |
--- /dev/null |
1464 |
+++ b/pym/portage/tests/bin/testdir/work/somepackage-1.2.3/o/.gitignore |
1465 |
@@ -0,0 +1 @@ |
1466 |
+* |
1467 |
diff --git a/pym/portage/tests/bin/testdir/work/somepackage-1.2.3/src/main.c b/pym/portage/tests/bin/testdir/work/somepackage-1.2.3/src/main.c |
1468 |
new file mode 100644 |
1469 |
index 0000000..9989d20 |
1470 |
--- /dev/null |
1471 |
+++ b/pym/portage/tests/bin/testdir/work/somepackage-1.2.3/src/main.c |
1472 |
@@ -0,0 +1,5 @@ |
1473 |
+#include <stdio.h> |
1474 |
+int main() { |
1475 |
+ puts("hi"); |
1476 |
+ return 0; |
1477 |
+} |
1478 |
diff --git a/pym/portage/util/parallel.py b/pym/portage/util/parallel.py |
1479 |
new file mode 100644 |
1480 |
index 0000000..068f0ae |
1481 |
--- /dev/null |
1482 |
+++ b/pym/portage/util/parallel.py |
1483 |
@@ -0,0 +1,598 @@ |
1484 |
+# Copyright (c) 2012 The Chromium OS Authors. All rights reserved. |
1485 |
+# Use of this source code is governed by a BSD-style license that can be |
1486 |
+# found in the LICENSE file. |
1487 |
+ |
1488 |
+"""Module for running cbuildbot stages in the background.""" |
1489 |
+ |
1490 |
+import collections |
1491 |
+import contextlib |
1492 |
+import errno |
1493 |
+import functools |
1494 |
+import logging |
1495 |
+import multiprocessing |
1496 |
+import os |
1497 |
+import Queue |
1498 |
+import signal |
1499 |
+import sys |
1500 |
+import tempfile |
1501 |
+import time |
1502 |
+import traceback |
1503 |
+ |
1504 |
+ |
1505 |
+_BUFSIZE = 1024 |
1506 |
+ |
1507 |
+logger = logging.getLogger(__name__) |
1508 |
+ |
1509 |
+ |
1510 |
+class BackgroundFailure(Exception): |
1511 |
+ """A failure happened in the background""" |
1512 |
+ |
1513 |
+ |
1514 |
+class _BackgroundTask(multiprocessing.Process): |
1515 |
+ """Run a task in the background. |
1516 |
+ |
1517 |
+ This task may be the 'Run' function from a buildbot stage or just a plain |
1518 |
+ function. It will be run in the background. Output from this task is saved |
1519 |
+ to a temporary file and is printed when the 'Wait' function is called. |
1520 |
+ """ |
1521 |
+ |
1522 |
+ # The time we give Python to startup and exit. |
1523 |
+ STARTUP_TIMEOUT = 60 * 5 |
1524 |
+ EXIT_TIMEOUT = 60 * 10 |
1525 |
+ |
1526 |
+ # The time we allow processes to be silent. This must be greater than the |
1527 |
+ # hw_test_timeout set in cbuildbot_config.py, and less than the timeout set |
1528 |
+ # by buildbot itself (typically, 150 minutes.) |
1529 |
+ SILENT_TIMEOUT = 60 * 145 |
1530 |
+ |
1531 |
+ # The amount by which we reduce the SILENT_TIMEOUT every time we launch |
1532 |
+ # a subprocess. This helps ensure that children get a chance to enforce the |
1533 |
+ # SILENT_TIMEOUT prior to the parents enforcing it. |
1534 |
+ SILENT_TIMEOUT_STEP = 30 |
1535 |
+ MINIMUM_SILENT_TIMEOUT = 60 * 135 |
1536 |
+ |
1537 |
+ # The time before terminating or killing a task. |
1538 |
+ SIGTERM_TIMEOUT = 30 |
1539 |
+ SIGKILL_TIMEOUT = 60 |
1540 |
+ |
1541 |
+ # Interval we check for updates from print statements. |
1542 |
+ PRINT_INTERVAL = 1 |
1543 |
+ |
1544 |
+ def __init__(self, task, semaphore=None, task_args=None, task_kwargs=None): |
1545 |
+ """Create a new _BackgroundTask object. |
1546 |
+ |
1547 |
+ If semaphore is supplied, it will be acquired for the duration of the |
1548 |
+ steps that are run in the background. This can be used to limit the |
1549 |
+ number of simultaneous parallel tasks. |
1550 |
+ |
1551 |
+ Args: |
1552 |
+ task: The task (a functor) to run in the background. |
1553 |
+ semaphore: The lock to hold while |task| runs. |
1554 |
+ task_args: A list of args to pass to the |task|. |
1555 |
+ task_kwargs: A dict of optional args to pass to the |task|. |
1556 |
+ """ |
1557 |
+ multiprocessing.Process.__init__(self) |
1558 |
+ self._task = task |
1559 |
+ self._queue = multiprocessing.Queue() |
1560 |
+ self._semaphore = semaphore |
1561 |
+ self._started = multiprocessing.Event() |
1562 |
+ self._killing = multiprocessing.Event() |
1563 |
+ self._output = None |
1564 |
+ self._parent_pid = None |
1565 |
+ self._task_args = task_args if task_args else () |
1566 |
+ self._task_kwargs = task_kwargs if task_kwargs else {} |
1567 |
+ |
1568 |
+ def _WaitForStartup(self): |
1569 |
+ # TODO(davidjames): Use python-2.7 syntax to simplify this. |
1570 |
+ self._started.wait(self.STARTUP_TIMEOUT) |
1571 |
+ msg = 'Process failed to start in %d seconds' % self.STARTUP_TIMEOUT |
1572 |
+ assert self._started.is_set(), msg |
1573 |
+ |
1574 |
+ def Kill(self, sig, log_level): |
1575 |
+ """Kill process with signal, ignoring if the process is dead. |
1576 |
+ |
1577 |
+ Args: |
1578 |
+ sig: Signal to send. |
1579 |
+ log_level: The log level of log messages. |
1580 |
+ """ |
1581 |
+ self._killing.set() |
1582 |
+ self._WaitForStartup() |
1583 |
+ if logger.isEnabledFor(log_level): |
1584 |
+ logger.log(log_level, 'Killing %r (sig=%r)', self.pid, sig) |
1585 |
+ |
1586 |
+ try: |
1587 |
+ os.kill(self.pid, sig) |
1588 |
+ except OSError as ex: |
1589 |
+ if ex.errno != errno.ESRCH: |
1590 |
+ raise |
1591 |
+ |
1592 |
+ def Cleanup(self, silent=False): |
1593 |
+ """Wait for a process to exit.""" |
1594 |
+ if os.getpid() != self._parent_pid or self._output is None: |
1595 |
+ return |
1596 |
+ try: |
1597 |
+ # Print output from subprocess. |
1598 |
+ if not silent and logger.isEnabledFor(logging.DEBUG): |
1599 |
+ with open(self._output.name, 'r') as f: |
1600 |
+ for line in f: |
1601 |
+ logging.debug(line.rstrip('\n')) |
1602 |
+ finally: |
1603 |
+ # Clean up our temporary file. |
1604 |
+ os.unlink(self._output.name) |
1605 |
+ self._output.close() |
1606 |
+ self._output = None |
1607 |
+ |
1608 |
+ def Wait(self): |
1609 |
+ """Wait for the task to complete. |
1610 |
+ |
1611 |
+ Output from the task is printed as it runs. |
1612 |
+ |
1613 |
+ If an exception occurs, return a string containing the traceback. |
1614 |
+ """ |
1615 |
+ try: |
1616 |
+ # Flush stdout and stderr to be sure no output is interleaved. |
1617 |
+ sys.stdout.flush() |
1618 |
+ sys.stderr.flush() |
1619 |
+ |
1620 |
+ # File position pointers are shared across processes, so we must open |
1621 |
+ # our own file descriptor to ensure output is not lost. |
1622 |
+ self._WaitForStartup() |
1623 |
+ silent_death_time = time.time() + self.SILENT_TIMEOUT |
1624 |
+ results = [] |
1625 |
+ with open(self._output.name, 'r') as output: |
1626 |
+ pos = 0 |
1627 |
+ running, exited_cleanly, msg, error = (True, False, None, None) |
1628 |
+ while running: |
1629 |
+ # Check whether the process is still alive. |
1630 |
+ running = self.is_alive() |
1631 |
+ |
1632 |
+ try: |
1633 |
+ error = self._queue.get(True, self.PRINT_INTERVAL)[0] |
1634 |
+ running = False |
1635 |
+ exited_cleanly = True |
1636 |
+ except Queue.Empty: |
1637 |
+ pass |
1638 |
+ |
1639 |
+ if not running: |
1640 |
+ # Wait for the process to actually exit. If the child doesn't exit |
1641 |
+ # in a timely fashion, kill it. |
1642 |
+ self.join(self.EXIT_TIMEOUT) |
1643 |
+ if self.exitcode is None: |
1644 |
+ msg = '%r hung for %r seconds' % (self, self.EXIT_TIMEOUT) |
1645 |
+ self._KillChildren([self]) |
1646 |
+ elif not exited_cleanly: |
1647 |
+ msg = ('%r exited unexpectedly with code %s' % |
1648 |
+ (self, self.EXIT_TIMEOUT)) |
1649 |
+ # Read output from process. |
1650 |
+ output.seek(pos) |
1651 |
+ buf = output.read(_BUFSIZE) |
1652 |
+ |
1653 |
+ if len(buf) > 0: |
1654 |
+ silent_death_time = time.time() + self.SILENT_TIMEOUT |
1655 |
+ elif running and time.time() > silent_death_time: |
1656 |
+ msg = ('No output from %r for %r seconds' % |
1657 |
+ (self, self.SILENT_TIMEOUT)) |
1658 |
+ self._KillChildren([self]) |
1659 |
+ |
1660 |
+ # Read remaining output from the process. |
1661 |
+ output.seek(pos) |
1662 |
+ buf = output.read(_BUFSIZE) |
1663 |
+ running = False |
1664 |
+ |
1665 |
+ # Print output so far. |
1666 |
+ while len(buf) > 0: |
1667 |
+ sys.stdout.write(buf) |
1668 |
+ pos += len(buf) |
1669 |
+ if len(buf) < _BUFSIZE: |
1670 |
+ break |
1671 |
+ buf = output.read(_BUFSIZE) |
1672 |
+ |
1673 |
+ # Print error messages if anything exceptional occurred. |
1674 |
+ if msg: |
1675 |
+ error = '\n'.join(x for x in (error, msg) if x) |
1676 |
+ logger.warning(error) |
1677 |
+ traceback.print_stack() |
1678 |
+ |
1679 |
+ sys.stdout.flush() |
1680 |
+ sys.stderr.flush() |
1681 |
+ |
1682 |
+ finally: |
1683 |
+ self.Cleanup(silent=True) |
1684 |
+ |
1685 |
+ # If a traceback occurred, return it. |
1686 |
+ return error |
1687 |
+ |
1688 |
+ def start(self): |
1689 |
+ """Invoke multiprocessing.Process.start after flushing output/err.""" |
1690 |
+ if self.SILENT_TIMEOUT < self.MINIMUM_SILENT_TIMEOUT: |
1691 |
+ raise AssertionError('Maximum recursion depth exceeded in %r' % self) |
1692 |
+ |
1693 |
+ sys.stdout.flush() |
1694 |
+ sys.stderr.flush() |
1695 |
+ self._output = tempfile.NamedTemporaryFile(delete=False, bufsize=0, |
1696 |
+ prefix='chromite-parallel-') |
1697 |
+ self._parent_pid = os.getpid() |
1698 |
+ return multiprocessing.Process.start(self) |
1699 |
+ |
1700 |
+ def run(self): |
1701 |
+ """Run the list of steps.""" |
1702 |
+ if self._semaphore is not None: |
1703 |
+ self._semaphore.acquire() |
1704 |
+ |
1705 |
+ error = 'Unexpected exception in %r' % self |
1706 |
+ pid = os.getpid() |
1707 |
+ try: |
1708 |
+ error = self._Run() |
1709 |
+ finally: |
1710 |
+ if not self._killing.is_set() and os.getpid() == pid: |
1711 |
+ self._queue.put((error,)) |
1712 |
+ if self._semaphore is not None: |
1713 |
+ self._semaphore.release() |
1714 |
+ |
1715 |
+ def _Run(self): |
1716 |
+ """Internal method for running the list of steps.""" |
1717 |
+ |
1718 |
+ # The default handler for SIGINT sometimes forgets to actually raise the |
1719 |
+ # exception (and we can reproduce this using unit tests), so we define a |
1720 |
+ # custom one instead. |
1721 |
+ def kill_us(_sig_num, _frame): |
1722 |
+ raise KeyboardInterrupt('SIGINT received') |
1723 |
+ signal.signal(signal.SIGINT, kill_us) |
1724 |
+ |
1725 |
+ sys.stdout.flush() |
1726 |
+ sys.stderr.flush() |
1727 |
+ # Send all output to a named temporary file. |
1728 |
+ with open(self._output.name, 'w', 0) as output: |
1729 |
+ # Back up sys.std{err,out}. These aren't used, but we keep a copy so |
1730 |
+ # that they aren't garbage collected. We intentionally don't restore |
1731 |
+ # the old stdout and stderr at the end, because we want shutdown errors |
1732 |
+ # to also be sent to the same log file. |
1733 |
+ _orig_stdout, _orig_stderr = sys.stdout, sys.stderr |
1734 |
+ |
1735 |
+ # Replace std{out,err} with unbuffered file objects. |
1736 |
+ os.dup2(output.fileno(), sys.__stdout__.fileno()) |
1737 |
+ os.dup2(output.fileno(), sys.__stderr__.fileno()) |
1738 |
+ sys.stdout = os.fdopen(sys.__stdout__.fileno(), 'w', 0) |
1739 |
+ sys.stderr = os.fdopen(sys.__stderr__.fileno(), 'w', 0) |
1740 |
+ |
1741 |
+ error = None |
1742 |
+ try: |
1743 |
+ self._started.set() |
1744 |
+ |
1745 |
+ # Reduce the silent timeout by the prescribed amount. |
1746 |
+ cls = self.__class__ |
1747 |
+ cls.SILENT_TIMEOUT -= cls.SILENT_TIMEOUT_STEP |
1748 |
+ |
1749 |
+ # Actually launch the task. |
1750 |
+ self._task(*self._task_args, **self._task_kwargs) |
1751 |
+ except BaseException as ex: |
1752 |
+ error = traceback.format_exc() |
1753 |
+ if self._killing.is_set(): |
1754 |
+ traceback.print_exc() |
1755 |
+ finally: |
1756 |
+ sys.stdout.flush() |
1757 |
+ sys.stderr.flush() |
1758 |
+ |
1759 |
+ return error |
1760 |
+ |
1761 |
+ @classmethod |
1762 |
+ def _KillChildren(cls, bg_tasks, log_level=logging.WARNING): |
1763 |
+ """Kill a deque of background tasks. |
1764 |
+ |
1765 |
+ This is needed to prevent hangs in the case where child processes refuse |
1766 |
+ to exit. |
1767 |
+ |
1768 |
+ Arguments: |
1769 |
+ bg_tasks: A list filled with _BackgroundTask objects. |
1770 |
+ log_level: The log level of log messages. |
1771 |
+ """ |
1772 |
+ logger.log(log_level, 'Killing tasks: %r', bg_tasks) |
1773 |
+ signals = ((signal.SIGINT, cls.SIGTERM_TIMEOUT), |
1774 |
+ (signal.SIGTERM, cls.SIGKILL_TIMEOUT), |
1775 |
+ (signal.SIGKILL, None)) |
1776 |
+ for sig, timeout in signals: |
1777 |
+ # Send signal to all tasks. |
1778 |
+ for task in bg_tasks: |
1779 |
+ task.Kill(sig, log_level) |
1780 |
+ |
1781 |
+ # Wait for all tasks to exit, if requested. |
1782 |
+ if timeout is None: |
1783 |
+ for task in bg_tasks: |
1784 |
+ task.join() |
1785 |
+ task.Cleanup() |
1786 |
+ break |
1787 |
+ |
1788 |
+ # Wait until timeout expires. |
1789 |
+ end_time = time.time() + timeout |
1790 |
+ while bg_tasks: |
1791 |
+ time_left = end_time - time.time() |
1792 |
+ if time_left <= 0: |
1793 |
+ break |
1794 |
+ task = bg_tasks[-1] |
1795 |
+ task.join(time_left) |
1796 |
+ if task.exitcode is not None: |
1797 |
+ task.Cleanup() |
1798 |
+ bg_tasks.pop() |
1799 |
+ |
1800 |
+ @classmethod |
1801 |
+ @contextlib.contextmanager |
1802 |
+ def ParallelTasks(cls, steps, max_parallel=None, halt_on_error=False): |
1803 |
+ """Run a list of functions in parallel. |
1804 |
+ |
1805 |
+ This function launches the provided functions in the background, yields, |
1806 |
+ and then waits for the functions to exit. |
1807 |
+ |
1808 |
+ The output from the functions is saved to a temporary file and printed as if |
1809 |
+ they were run in sequence. |
1810 |
+ |
1811 |
+ If exceptions occur in the steps, we join together the tracebacks and print |
1812 |
+ them after all parallel tasks have finished running. Further, a |
1813 |
+ BackgroundFailure is raised with full stack traces of all exceptions. |
1814 |
+ |
1815 |
+ Args: |
1816 |
+ steps: A list of functions to run. |
1817 |
+ max_parallel: The maximum number of simultaneous tasks to run in parallel. |
1818 |
+ By default, run all tasks in parallel. |
1819 |
+ halt_on_error: After the first exception occurs, halt any running steps, |
1820 |
+ and squelch any further output, including any exceptions that might |
1821 |
+ occur. |
1822 |
+ """ |
1823 |
+ |
1824 |
+ semaphore = None |
1825 |
+ if max_parallel is not None: |
1826 |
+ semaphore = multiprocessing.Semaphore(max_parallel) |
1827 |
+ |
1828 |
+ # First, start all the steps. |
1829 |
+ bg_tasks = collections.deque() |
1830 |
+ for step in steps: |
1831 |
+ task = cls(step, semaphore=semaphore) |
1832 |
+ task.start() |
1833 |
+ bg_tasks.append(task) |
1834 |
+ |
1835 |
+ try: |
1836 |
+ yield |
1837 |
+ finally: |
1838 |
+ # Wait for each step to complete. |
1839 |
+ tracebacks = [] |
1840 |
+ while bg_tasks: |
1841 |
+ task = bg_tasks.popleft() |
1842 |
+ error = task.Wait() |
1843 |
+ if error is not None: |
1844 |
+ tracebacks.append(error) |
1845 |
+ if halt_on_error: |
1846 |
+ break |
1847 |
+ |
1848 |
+ # If there are still tasks left, kill them. |
1849 |
+ if bg_tasks: |
1850 |
+ cls._KillChildren(bg_tasks, log_level=logging.DEBUG) |
1851 |
+ |
1852 |
+ # Propagate any exceptions. |
1853 |
+ if tracebacks: |
1854 |
+ raise BackgroundFailure('\n' + ''.join(tracebacks)) |
1855 |
+ |
1856 |
+ @staticmethod |
1857 |
+ def TaskRunner(queue, task, onexit=None, task_args=None, task_kwargs=None): |
1858 |
+ """Run task(*input) for each input in the queue. |
1859 |
+ |
1860 |
+ Returns when it encounters an _AllTasksComplete object on the queue. |
1861 |
+ If exceptions occur, save them off and re-raise them as a |
1862 |
+ BackgroundFailure once we've finished processing the items in the queue. |
1863 |
+ |
1864 |
+ Args: |
1865 |
+ queue: A queue of tasks to run. Add tasks to this queue, and they will |
1866 |
+ be run. |
1867 |
+ task: Function to run on each queued input. |
1868 |
+ onexit: Function to run after all inputs are processed. |
1869 |
+ task_args: A list of args to pass to the |task|. |
1870 |
+ task_kwargs: A dict of optional args to pass to the |task|. |
1871 |
+ """ |
1872 |
+ if task_args is None: |
1873 |
+ task_args = [] |
1874 |
+ elif not isinstance(task_args, list): |
1875 |
+ task_args = list(task_args) |
1876 |
+ if task_kwargs is None: |
1877 |
+ task_kwargs = {} |
1878 |
+ |
1879 |
+ tracebacks = [] |
1880 |
+ while True: |
1881 |
+ # Wait for a new item to show up on the queue. This is a blocking wait, |
1882 |
+ # so if there's nothing to do, we just sit here. |
1883 |
+ x = queue.get() |
1884 |
+ if isinstance(x, _AllTasksComplete): |
1885 |
+ # All tasks are complete, so we should exit. |
1886 |
+ break |
1887 |
+ elif not isinstance(x, list): |
1888 |
+ x = task_args + list(x) |
1889 |
+ else: |
1890 |
+ x = task_args + x |
1891 |
+ |
1892 |
+ # If no tasks failed yet, process the remaining tasks. |
1893 |
+ if not tracebacks: |
1894 |
+ try: |
1895 |
+ task(*x, **task_kwargs) |
1896 |
+ except BaseException: |
1897 |
+ tracebacks.append(traceback.format_exc()) |
1898 |
+ |
1899 |
+ # Run exit handlers. |
1900 |
+ if onexit: |
1901 |
+ onexit() |
1902 |
+ |
1903 |
+ # Propagate any exceptions. |
1904 |
+ if tracebacks: |
1905 |
+ raise BackgroundFailure('\n' + ''.join(tracebacks)) |
1906 |
+ |
1907 |
+ |
1908 |
+def RunParallelSteps(steps, max_parallel=None, halt_on_error=False, |
1909 |
+ return_values=False): |
1910 |
+ """Run a list of functions in parallel. |
1911 |
+ |
1912 |
+ This function blocks until all steps are completed. |
1913 |
+ |
1914 |
+ The output from the functions is saved to a temporary file and printed as if |
1915 |
+ they were run in sequence. |
1916 |
+ |
1917 |
+ If exceptions occur in the steps, we join together the tracebacks and print |
1918 |
+ them after all parallel tasks have finished running. Further, a |
1919 |
+ BackgroundFailure is raised with full stack traces of all exceptions. |
1920 |
+ |
1921 |
+ Args: |
1922 |
+ steps: A list of functions to run. |
1923 |
+ max_parallel: The maximum number of simultaneous tasks to run in parallel. |
1924 |
+ By default, run all tasks in parallel. |
1925 |
+ halt_on_error: After the first exception occurs, halt any running steps, |
1926 |
+ and squelch any further output, including any exceptions that might occur. |
1927 |
+ return_values: If set to True, RunParallelSteps returns a list containing |
1928 |
+ the return values of the steps. Defaults to False. |
1929 |
+ |
1930 |
+ Returns: |
1931 |
+ If |return_values| is True, the function will return a list containing the |
1932 |
+ return values of the steps. |
1933 |
+ |
1934 |
+ Example: |
1935 |
+ # This snippet will execute in parallel: |
1936 |
+ # somefunc() |
1937 |
+ # anotherfunc() |
1938 |
+ # funcfunc() |
1939 |
+ steps = [somefunc, anotherfunc, funcfunc] |
1940 |
+ RunParallelSteps(steps) |
1941 |
+ # Blocks until all calls have completed. |
1942 |
+ """ |
1943 |
+ def ReturnWrapper(queue, fn): |
1944 |
+ """A function that """ |
1945 |
+ queue.put(fn()) |
1946 |
+ |
1947 |
+ full_steps = [] |
1948 |
+ queues = [] |
1949 |
+ manager = None |
1950 |
+ if return_values: |
1951 |
+ # We use a managed queue here, because the child process will wait for the |
1952 |
+ # queue(pipe) to be flushed (i.e., when items are read from the queue) |
1953 |
+ # before exiting, and with a regular queue this may result in hangs for |
1954 |
+ # large return values. But with a managed queue, the manager process will |
1955 |
+ # read the items and hold on to them until the managed queue goes out of |
1956 |
+ # scope and is cleaned up. |
1957 |
+ manager = multiprocessing.Manager() |
1958 |
+ for step in steps: |
1959 |
+ # pylint: disable=E1101 |
1960 |
+ queue = manager.Queue() |
1961 |
+ queues.append(queue) |
1962 |
+ full_steps.append(functools.partial(ReturnWrapper, queue, step)) |
1963 |
+ else: |
1964 |
+ full_steps = steps |
1965 |
+ |
1966 |
+ with _BackgroundTask.ParallelTasks(full_steps, max_parallel=max_parallel, |
1967 |
+ halt_on_error=halt_on_error): |
1968 |
+ pass |
1969 |
+ |
1970 |
+ if return_values: |
1971 |
+ return [queue.get_nowait() for queue in queues] |
1972 |
+ |
1973 |
+ |
1974 |
+class _AllTasksComplete(object): |
1975 |
+ """Sentinel object to indicate that all tasks are complete.""" |
1976 |
+ |
1977 |
+ |
1978 |
+@××××××××××.contextmanager |
1979 |
+def BackgroundTaskRunner(task, *args, **kwargs): |
1980 |
+ """Run the specified task on each queued input in a pool of processes. |
1981 |
+ |
1982 |
+ This context manager starts a set of workers in the background, who each |
1983 |
+ wait for input on the specified queue. For each input on the queue, these |
1984 |
+ workers run task(*args + *input, **kwargs). Note that certain kwargs will |
1985 |
+ not pass through to the task (see Args below for the list). |
1986 |
+ |
1987 |
+ The output from these tasks is saved to a temporary file. When control |
1988 |
+ returns to the context manager, the background output is printed in order, |
1989 |
+ as if the tasks were run in sequence. |
1990 |
+ |
1991 |
+ If exceptions occur in the steps, we join together the tracebacks and print |
1992 |
+ them after all parallel tasks have finished running. Further, a |
1993 |
+ BackgroundFailure is raised with full stack traces of all exceptions. |
1994 |
+ |
1995 |
+ Example: |
1996 |
+ # This will run somefunc(1, 'small', 'cow', foo='bar' in the background |
1997 |
+ # while "more random stuff" is being executed. |
1998 |
+ |
1999 |
+ def somefunc(arg1, arg2, arg3, foo=None): |
2000 |
+ ... |
2001 |
+ ... |
2002 |
+ with BackgroundTaskRunner(somefunc, 1, foo='bar') as queue: |
2003 |
+ ... do random stuff ... |
2004 |
+ queue.put(['small', 'cow']) |
2005 |
+ ... do more random stuff ... |
2006 |
+ # Exiting the with statement will block until all calls have completed. |
2007 |
+ |
2008 |
+ Args: |
2009 |
+ task: Function to run on each queued input. |
2010 |
+ queue: A queue of tasks to run. Add tasks to this queue, and they will |
2011 |
+ be run in the background. If None, one will be created on the fly. |
2012 |
+ processes: Number of processes to launch. |
2013 |
+ onexit: Function to run in each background process after all inputs are |
2014 |
+ processed. |
2015 |
+ """ |
2016 |
+ |
2017 |
+ queue = kwargs.pop('queue', None) |
2018 |
+ processes = kwargs.pop('processes', None) |
2019 |
+ onexit = kwargs.pop('onexit', None) |
2020 |
+ |
2021 |
+ if queue is None: |
2022 |
+ queue = multiprocessing.Queue() |
2023 |
+ |
2024 |
+ if not processes: |
2025 |
+ processes = multiprocessing.cpu_count() |
2026 |
+ |
2027 |
+ child = functools.partial(_BackgroundTask.TaskRunner, queue, task, |
2028 |
+ onexit=onexit, task_args=args, |
2029 |
+ task_kwargs=kwargs) |
2030 |
+ steps = [child] * processes |
2031 |
+ with _BackgroundTask.ParallelTasks(steps): |
2032 |
+ try: |
2033 |
+ yield queue |
2034 |
+ finally: |
2035 |
+ for _ in xrange(processes): |
2036 |
+ queue.put(_AllTasksComplete()) |
2037 |
+ |
2038 |
+ |
2039 |
+def RunTasksInProcessPool(task, inputs, processes=None, onexit=None): |
2040 |
+ """Run the specified function with each supplied input in a pool of processes. |
2041 |
+ |
2042 |
+ This function runs task(*x) for x in inputs in a pool of processes. This |
2043 |
+ function blocks until all tasks are completed. |
2044 |
+ |
2045 |
+ The output from these tasks is saved to a temporary file. When control |
2046 |
+ returns to the context manager, the background output is printed in order, |
2047 |
+ as if the tasks were run in sequence. |
2048 |
+ |
2049 |
+ If exceptions occur in the steps, we join together the tracebacks and print |
2050 |
+ them after all parallel tasks have finished running. Further, a |
2051 |
+ BackgroundFailure is raised with full stack traces of all exceptions. |
2052 |
+ |
2053 |
+ Example: |
2054 |
+ # This snippet will execute in parallel: |
2055 |
+ # somefunc('hi', 'fat', 'code') |
2056 |
+ # somefunc('foo', 'bar', 'cow') |
2057 |
+ |
2058 |
+ def somefunc(arg1, arg2, arg3): |
2059 |
+ ... |
2060 |
+ ... |
2061 |
+ inputs = [ |
2062 |
+ ['hi', 'fat', 'code'], |
2063 |
+ ['foo', 'bar', 'cow'], |
2064 |
+ ] |
2065 |
+ RunTasksInProcessPool(somefunc, inputs) |
2066 |
+ # Blocks until all calls have completed. |
2067 |
+ |
2068 |
+ Args: |
2069 |
+ task: Function to run on each input. |
2070 |
+ inputs: List of inputs. |
2071 |
+ processes: Number of processes, at most, to launch. |
2072 |
+ onexit: Function to run in each background process after all inputs are |
2073 |
+ processed. |
2074 |
+ """ |
2075 |
+ |
2076 |
+ if not processes: |
2077 |
+ processes = min(multiprocessing.cpu_count(), len(inputs)) |
2078 |
+ |
2079 |
+ with BackgroundTaskRunner(task, processes=processes, onexit=onexit) as queue: |
2080 |
+ for x in inputs: |
2081 |
+ queue.put(x) |
2082 |
-- |
2083 |
1.8.3.2 |