1 |
regenerating autotools in packages that have a lot of AC_CONFIG_SUBDIRS is |
2 |
really slow due to the serialization of all the dirs (which really isn't |
3 |
required). so i took some code that i merged into portage semi-recently |
4 |
(which is based on work by Brian, although i'm not sure he wants to admit it) |
5 |
and put it into a new multiprocessing.eclass. this way people can generically |
6 |
utilize this in their own eclasses/ebuilds. |
7 |
|
8 |
it doesn't currently support nesting. not sure if i should fix that. |
9 |
|
10 |
i'll follow up with an example of parallelizing of eautoreconf. for |
11 |
mail-filter/maildrop on my 4 core system, it cuts the time needed to run from |
12 |
~2.5 min to ~1 min. |
13 |
-mike |
14 |
|
15 |
# Copyright 1999-2012 Gentoo Foundation |
16 |
# Distributed under the terms of the GNU General Public License v2 |
17 |
# $Header: $ |
18 |
|
19 |
# @ECLASS: multiprocessing.eclass |
20 |
# @MAINTAINER: |
21 |
# base-system@g.o |
22 |
# @AUTHORS: |
23 |
# Brian Harring <ferringb@g.o> |
24 |
# Mike Frysinger <vapier@g.o> |
25 |
# @BLURB: parallelization with bash (wtf?) |
26 |
# @DESCRIPTION: |
27 |
# The multiprocessing eclass contains a suite of functions that allow ebuilds |
28 |
# to quickly run things in parallel using shell code. |
29 |
|
30 |
if [[ ${___ECLASS_ONCE_MULTIPROCESSING} != "recur -_+^+_- spank" ]] ; then |
31 |
___ECLASS_ONCE_MULTIPROCESSING="recur -_+^+_- spank" |
32 |
|
33 |
# @FUNCTION: makeopts_jobs |
34 |
# @USAGE: [${MAKEOPTS}] |
35 |
# @DESCRIPTION: |
36 |
# Searches the arguments (defaults to ${MAKEOPTS}) and extracts the jobs number |
37 |
# specified therein. Useful for running non-make tools in parallel too. |
38 |
# i.e. if the user has MAKEOPTS=-j9, this will show "9". |
39 |
# We can't return the number as bash normalizes it to [0, 255]. If the flags |
40 |
# haven't specified a -j flag, then "1" is shown as that is the default `make` |
41 |
# uses. Since there's no way to represent infinity, we return 999 if the user |
42 |
# has -j without a number. |
43 |
makeopts_jobs() { |
44 |
[[ $# -eq 0 ]] && set -- ${MAKEOPTS} |
45 |
# This assumes the first .* will be more greedy than the second .* |
46 |
# since POSIX doesn't specify a non-greedy match (i.e. ".*?"). |
47 |
local jobs=$(echo " $* " | sed -r -n \ |
48 |
-e 's:.*[[:space:]](-j|--jobs[=[:space:]])[[:space:]]*([0-9]+).*:\2:p' \ |
49 |
-e 's:.*[[:space:]](-j|--jobs)[[:space:]].*:999:p') |
50 |
echo ${jobs:-1} |
51 |
} |
52 |
|
53 |
# @FUNCTION: multijob_init |
54 |
# @USAGE: [${MAKEOPTS}] |
55 |
# @DESCRIPTION: |
56 |
# Setup the environment for executing things in parallel. |
57 |
# You must call this before any other multijob function. |
58 |
multijob_init() { |
59 |
# Setup a pipe for children to write their pids to when they finish. |
60 |
mj_control_pipe="${T}/multijob.pipe" |
61 |
mkfifo "${mj_control_pipe}" |
62 |
exec {mj_control_fd}<>${mj_control_pipe} |
63 |
rm -f "${mj_control_pipe}" |
64 |
|
65 |
# See how many children we can fork based on the user's settings. |
66 |
mj_max_jobs=$(makeopts_jobs "$@") |
67 |
mj_num_jobs=0 |
68 |
} |
69 |
|
70 |
# @FUNCTION: multijob_child_init |
71 |
# @DESCRIPTION: |
72 |
# You must call this first in the forked child process. |
73 |
multijob_child_init() { |
74 |
[[ $# -eq 0 ]] || die "${FUNCNAME} takes no arguments" |
75 |
|
76 |
trap 'echo ${BASHPID} $? >&'${mj_control_fd} EXIT |
77 |
trap 'exit 1' INT TERM |
78 |
} |
79 |
|
80 |
# @FUNCTION: multijob_post_fork |
81 |
# @DESCRIPTION: |
82 |
# You must call this in the parent process after forking a child process. |
83 |
# If the parallel limit has been hit, it will wait for one to finish and |
84 |
# return the child's exit status. |
85 |
multijob_post_fork() { |
86 |
[[ $# -eq 0 ]] || die "${FUNCNAME} takes no arguments" |
87 |
|
88 |
: $(( ++mj_num_jobs )) |
89 |
if [[ ${mj_num_jobs} -ge ${mj_max_jobs} ]] ; then |
90 |
multijob_finish_one |
91 |
fi |
92 |
return $? |
93 |
} |
94 |
|
95 |
# @FUNCTION: multijob_finish_one |
96 |
# @DESCRIPTION: |
97 |
# Wait for a single process to exit and return its exit code. |
98 |
multijob_finish_one() { |
99 |
[[ $# -eq 0 ]] || die "${FUNCNAME} takes no arguments" |
100 |
|
101 |
local pid ret |
102 |
read -r -u ${mj_control_fd} pid ret |
103 |
: $(( --mj_num_jobs )) |
104 |
return ${ret} |
105 |
} |
106 |
|
107 |
# @FUNCTION: multijob_finish |
108 |
# @DESCRIPTION: |
109 |
# Wait for all pending processes to exit and return the bitwise or |
110 |
# of all their exit codes. |
111 |
multijob_finish() { |
112 |
[[ $# -eq 0 ]] || die "${FUNCNAME} takes no arguments" |
113 |
|
114 |
local ret=0 |
115 |
while [[ ${mj_num_jobs} -gt 0 ]] ; do |
116 |
multijob_finish_one |
117 |
: $(( ret |= $? )) |
118 |
done |
119 |
# Let bash clean up its internal child tracking state. |
120 |
wait |
121 |
return ${ret} |
122 |
} |
123 |
|
124 |
fi |