1 |
vapier 12/06/07 04:59:41 |
2 |
|
3 |
Added: multiprocessing.eclass |
4 |
Log: |
5 |
initial multiprocessing eclass |
6 |
|
7 |
Revision Changes Path |
8 |
1.1 eclass/multiprocessing.eclass |
9 |
|
10 |
file : http://sources.gentoo.org/viewvc.cgi/gentoo-x86/eclass/multiprocessing.eclass?rev=1.1&view=markup |
11 |
plain: http://sources.gentoo.org/viewvc.cgi/gentoo-x86/eclass/multiprocessing.eclass?rev=1.1&content-type=text/plain |
12 |
|
13 |
Index: multiprocessing.eclass |
14 |
=================================================================== |
15 |
# Copyright 1999-2012 Gentoo Foundation |
16 |
# Distributed under the terms of the GNU General Public License v2 |
17 |
# $Header: /var/cvsroot/gentoo-x86/eclass/multiprocessing.eclass,v 1.1 2012/06/07 04:59:40 vapier Exp $ |
18 |
|
19 |
# @ECLASS: multiprocessing.eclass |
20 |
# @MAINTAINER: |
21 |
# base-system@g.o |
22 |
# @AUTHOR: |
23 |
# Brian Harring <ferringb@g.o> |
24 |
# Mike Frysinger <vapier@g.o> |
25 |
# @BLURB: parallelization with bash (wtf?) |
26 |
# @DESCRIPTION: |
27 |
# The multiprocessing eclass contains a suite of functions that allow ebuilds |
28 |
# to quickly run things in parallel using shell code. |
29 |
# |
30 |
# It has two modes: pre-fork and post-fork. If you don't want to dive into any |
31 |
# more nuts & bolts, just use the pre-fork mode. For main threads that mostly |
32 |
# spawn children and then wait for them to finish, use the pre-fork mode. For |
33 |
# main threads that do a bit of processing themselves, use the post-fork mode. |
34 |
# You may mix & match them for longer computation loops. |
35 |
# @EXAMPLE: |
36 |
# |
37 |
# @CODE |
38 |
# # First initialize things: |
39 |
# multijob_init |
40 |
# |
41 |
# # Then hash a bunch of files in parallel: |
42 |
# for n in {0..20} ; do |
43 |
# multijob_child_init md5sum data.${n} > data.${n} |
44 |
# done |
45 |
# |
46 |
# # Then wait for all the children to finish: |
47 |
# multijob_finish |
48 |
# @CODE |
49 |
|
50 |
if [[ ${___ECLASS_ONCE_MULTIPROCESSING} != "recur -_+^+_- spank" ]] ; then |
51 |
___ECLASS_ONCE_MULTIPROCESSING="recur -_+^+_- spank" |
52 |
|
53 |
# @FUNCTION: makeopts_jobs |
54 |
# @USAGE: [${MAKEOPTS}] |
55 |
# @DESCRIPTION: |
56 |
# Searches the arguments (defaults to ${MAKEOPTS}) and extracts the jobs number |
57 |
# specified therein. Useful for running non-make tools in parallel too. |
58 |
# i.e. if the user has MAKEOPTS=-j9, this will echo "9" -- we can't return the |
59 |
# number as bash normalizes it to [0, 255]. If the flags haven't specified a |
60 |
# -j flag, then "1" is shown as that is the default `make` uses. Since there's |
61 |
# no way to represent infinity, we return 999 if the user has -j without a number. |
62 |
makeopts_jobs() { |
63 |
[[ $# -eq 0 ]] && set -- ${MAKEOPTS} |
64 |
# This assumes the first .* will be more greedy than the second .* |
65 |
# since POSIX doesn't specify a non-greedy match (i.e. ".*?"). |
66 |
local jobs=$(echo " $* " | sed -r -n \ |
67 |
-e 's:.*[[:space:]](-j|--jobs[=[:space:]])[[:space:]]*([0-9]+).*:\2:p' \ |
68 |
-e 's:.*[[:space:]](-j|--jobs)[[:space:]].*:999:p') |
69 |
echo ${jobs:-1} |
70 |
} |
71 |
|
72 |
# @FUNCTION: multijob_init |
73 |
# @USAGE: [${MAKEOPTS}] |
74 |
# @DESCRIPTION: |
75 |
# Setup the environment for executing code in parallel. |
76 |
# You must call this before any other multijob function. |
77 |
multijob_init() { |
78 |
# When something goes wrong, try to wait for all the children so we |
79 |
# don't leave any zombies around. |
80 |
has wait ${EBUILD_DEATH_HOOKS} || EBUILD_DEATH_HOOKS+=" wait" |
81 |
|
82 |
# Setup a pipe for children to write their pids to when they finish. |
83 |
local pipe="${T}/multijob.pipe" |
84 |
mkfifo "${pipe}" |
85 |
redirect_alloc_fd mj_control_fd "${pipe}" |
86 |
rm -f "${pipe}" |
87 |
|
88 |
# See how many children we can fork based on the user's settings. |
89 |
mj_max_jobs=$(makeopts_jobs "$@") |
90 |
mj_num_jobs=0 |
91 |
} |
92 |
|
93 |
# @FUNCTION: multijob_child_init |
94 |
# @USAGE: [--pre|--post] [command to run in background] |
95 |
# @DESCRIPTION: |
96 |
# This function has two forms. You can use it to execute a simple command |
97 |
# in the background (and it takes care of everything else), or you must |
98 |
# call this first thing in your forked child process. |
99 |
# |
100 |
# The --pre/--post options allow you to select the child generation mode. |
101 |
# |
102 |
# @CODE |
103 |
# # 1st form: pass the command line as arguments: |
104 |
# multijob_child_init ls /dev |
105 |
# # Or if you want to use pre/post fork modes: |
106 |
# multijob_child_init --pre ls /dev |
107 |
# multijob_child_init --post ls /dev |
108 |
# |
109 |
# # 2nd form: execute multiple stuff in the background (post fork): |
110 |
# ( |
111 |
# multijob_child_init |
112 |
# out=`ls` |
113 |
# if echo "${out}" | grep foo ; then |
114 |
# echo "YEAH" |
115 |
# fi |
116 |
# ) & |
117 |
# multijob_post_fork |
118 |
# |
119 |
# # 2nd form: execute multiple stuff in the background (pre fork): |
120 |
# multijob_pre_fork |
121 |
# ( |
122 |
# multijob_child_init |
123 |
# out=`ls` |
124 |
# if echo "${out}" | grep foo ; then |
125 |
# echo "YEAH" |
126 |
# fi |
127 |
# ) & |
128 |
# @CODE |
129 |
multijob_child_init() { |
130 |
local mode="pre" |
131 |
case $1 in |
132 |
--pre) mode="pre" ; shift ;; |
133 |
--post) mode="post"; shift ;; |
134 |
esac |
135 |
|
136 |
if [[ $# -eq 0 ]] ; then |
137 |
trap 'echo ${BASHPID} $? >&'${mj_control_fd} EXIT |
138 |
trap 'exit 1' INT TERM |
139 |
else |
140 |
local ret |
141 |
[[ ${mode} == "pre" ]] && { multijob_pre_fork; ret=$?; } |
142 |
( multijob_child_init ; "$@" ) & |
143 |
[[ ${mode} == "post" ]] && { multijob_post_fork; ret=$?; } |
144 |
return ${ret} |
145 |
fi |
146 |
} |
147 |
|
148 |
# @FUNCTION: _multijob_fork |
149 |
# @INTERNAL |
150 |
# @DESCRIPTION: |
151 |
# Do the actual book keeping. |
152 |
_multijob_fork() { |
153 |
[[ $# -eq 1 ]] || die "incorrect number of arguments" |
154 |
|
155 |
local ret=0 |
156 |
[[ $1 == "post" ]] && : $(( ++mj_num_jobs )) |
157 |
if [[ ${mj_num_jobs} -ge ${mj_max_jobs} ]] ; then |
158 |
multijob_finish_one |
159 |
ret=$? |
160 |
fi |
161 |
[[ $1 == "pre" ]] && : $(( ++mj_num_jobs )) |
162 |
return ${ret} |
163 |
} |
164 |
|
165 |
# @FUNCTION: multijob_pre_fork |
166 |
# @DESCRIPTION: |
167 |
# You must call this in the parent process before forking a child process. |
168 |
# If the parallel limit has been hit, it will wait for one child to finish |
169 |
# and return its exit status. |
170 |
multijob_pre_fork() { _multijob_fork pre "$@" ; } |
171 |
|
172 |
# @FUNCTION: multijob_post_fork |
173 |
# @DESCRIPTION: |
174 |
# You must call this in the parent process after forking a child process. |
175 |
# If the parallel limit has been hit, it will wait for one child to finish |
176 |
# and return its exit status. |
177 |
multijob_post_fork() { _multijob_fork post "$@" ; } |
178 |
|
179 |
# @FUNCTION: multijob_finish_one |
180 |
# @DESCRIPTION: |
181 |
# Wait for a single process to exit and return its exit code. |
182 |
multijob_finish_one() { |
183 |
[[ $# -eq 0 ]] || die "${FUNCNAME} takes no arguments" |
184 |
|
185 |
local pid ret |
186 |
read -r -u ${mj_control_fd} pid ret || die |
187 |
: $(( --mj_num_jobs )) |
188 |
return ${ret} |
189 |
} |
190 |
|
191 |
# @FUNCTION: multijob_finish |
192 |
# @DESCRIPTION: |
193 |
# Wait for all pending processes to exit and return the bitwise or |
194 |
# of all their exit codes. |
195 |
multijob_finish() { |
196 |
local ret=0 |
197 |
while [[ ${mj_num_jobs} -gt 0 ]] ; do |
198 |
multijob_finish_one |
199 |
: $(( ret |= $? )) |
200 |
done |
201 |
# Let bash clean up its internal child tracking state. |
202 |
wait |
203 |
|
204 |
# Do this after reaping all the children. |
205 |
[[ $# -eq 0 ]] || die "${FUNCNAME} takes no arguments" |
206 |
|
207 |
return ${ret} |
208 |
} |
209 |
|
210 |
# @FUNCTION: redirect_alloc_fd |
211 |
# @USAGE: <var> <file> [redirection] |
212 |
# @DESCRIPTION: |
213 |
# Find a free fd and redirect the specified file via it. Store the new |
214 |
# fd in the specified variable. Useful for the cases where we don't care |
215 |
# about the exact fd #. |
216 |
redirect_alloc_fd() { |
217 |
local var=$1 file=$2 redir=${3:-"<>"} |
218 |
|
219 |
if [[ $(( (BASH_VERSINFO[0] << 8) + BASH_VERSINFO[1] )) -ge $(( (4 << 8) + 1 )) ]] ; then |
220 |
# Newer bash provides this functionality. |
221 |
eval "exec {${var}}${redir}'${file}'" |
222 |
else |
223 |
# Need to provide the functionality ourselves. |
224 |
local fd=10 |
225 |
while :; do |
226 |
# Make sure the fd isn't open. It could be a char device, |
227 |
# or a symlink (possibly broken) to something else. |
228 |
if [[ ! -e /dev/fd/${fd} ]] && [[ ! -L /dev/fd/${fd} ]] ; then |
229 |
eval "exec ${fd}${redir}'${file}'" && break |
230 |
fi |
231 |
[[ ${fd} -gt 1024 ]] && die 'could not locate a free temp fd !?' |
232 |
: $(( ++fd )) |
233 |
done |
234 |
: $(( ${var} = fd )) |
235 |
fi |
236 |
} |
237 |
|
238 |
fi |