1 |
Remove the multijob functions that were used to run bash code |
2 |
in parallel. The code was very complex, fragile and unmaintained. It has |
3 |
been used scarcely, and pretty much by a single developer. It gave very |
4 |
little gain, usually at the cost of losing readability and violating |
5 |
PMS. |
6 |
|
7 |
Closes: https://bugs.gentoo.org/613322 |
8 |
--- |
9 |
eclass/multiprocessing.eclass | 178 +++------------------------------------- |
10 |
eclass/tests/multiprocessing.sh | 42 ---------- |
11 |
2 files changed, 10 insertions(+), 210 deletions(-) |
12 |
delete mode 100755 eclass/tests/multiprocessing.sh |
13 |
|
14 |
// note: this is still waiting for ACK from hwoarang for removing |
15 |
// it from dev-tcl/jimtcl |
16 |
|
17 |
diff --git a/eclass/multiprocessing.eclass b/eclass/multiprocessing.eclass |
18 |
index b6e92976f73e..cfe22303043e 100644 |
19 |
--- a/eclass/multiprocessing.eclass |
20 |
+++ b/eclass/multiprocessing.eclass |
21 |
@@ -1,4 +1,4 @@ |
22 |
-# Copyright 1999-2014 Gentoo Foundation |
23 |
+# Copyright 1999-2017 Gentoo Foundation |
24 |
# Distributed under the terms of the GNU General Public License v2 |
25 |
|
26 |
# @ECLASS: multiprocessing.eclass |
27 |
@@ -7,29 +7,20 @@ |
28 |
# @AUTHOR: |
29 |
# Brian Harring <ferringb@g.o> |
30 |
# Mike Frysinger <vapier@g.o> |
31 |
-# @BLURB: parallelization with bash (wtf?) |
32 |
+# @BLURB: multiprocessing helper functions |
33 |
# @DESCRIPTION: |
34 |
-# The multiprocessing eclass contains a suite of functions that allow ebuilds |
35 |
-# to quickly run things in parallel using shell code. |
36 |
+# The multiprocessing eclass contains a suite of utility functions |
37 |
+# that could be helpful to controlling parallel multiple job execution. |
38 |
+# The most common use is processing MAKEOPTS in order to obtain job |
39 |
+# count. |
40 |
# |
41 |
-# It has two modes: pre-fork and post-fork. If you don't want to dive into any |
42 |
-# more nuts & bolts, just use the pre-fork mode. For main threads that mostly |
43 |
-# spawn children and then wait for them to finish, use the pre-fork mode. For |
44 |
-# main threads that do a bit of processing themselves, use the post-fork mode. |
45 |
-# You may mix & match them for longer computation loops. |
46 |
# @EXAMPLE: |
47 |
# |
48 |
# @CODE |
49 |
-# # First initialize things: |
50 |
-# multijob_init |
51 |
-# |
52 |
-# # Then hash a bunch of files in parallel: |
53 |
-# for n in {0..20} ; do |
54 |
-# multijob_child_init md5sum data.${n} > data.${n} |
55 |
-# done |
56 |
-# |
57 |
-# # Then wait for all the children to finish: |
58 |
-# multijob_finish |
59 |
+# src_compile() { |
60 |
+# # custom build system that does not support most of MAKEOPTS |
61 |
+# ./mybs -j$(makeopts_jobs) |
62 |
+# } |
63 |
# @CODE |
64 |
|
65 |
if [[ -z ${_MULTIPROCESSING_ECLASS} ]]; then |
66 |
@@ -126,155 +117,6 @@ makeopts_loadavg() { |
67 |
echo ${lavg:-${2:-999}} |
68 |
} |
69 |
|
70 |
-# @FUNCTION: multijob_init |
71 |
-# @USAGE: [${MAKEOPTS}] |
72 |
-# @DESCRIPTION: |
73 |
-# Setup the environment for executing code in parallel. |
74 |
-# You must call this before any other multijob function. |
75 |
-multijob_init() { |
76 |
- # When something goes wrong, try to wait for all the children so we |
77 |
- # don't leave any zombies around. |
78 |
- has wait ${EBUILD_DEATH_HOOKS} || EBUILD_DEATH_HOOKS+=" wait " |
79 |
- |
80 |
- # Setup a pipe for children to write their pids to when they finish. |
81 |
- # We have to allocate two fd's because POSIX has undefined behavior |
82 |
- # when using one single fd for both read and write. #487056 |
83 |
- # However, opening an fd for read or write only will block until the |
84 |
- # opposite end is opened as well. Thus we open the first fd for both |
85 |
- # read and write to not block ourselve, but use it for reading only. |
86 |
- # The second fd really is opened for write only, as Cygwin supports |
87 |
- # just one single read fd per FIFO. #583962 |
88 |
- local pipe="${T}/multijob.pipe" |
89 |
- mkfifo -m 600 "${pipe}" |
90 |
- redirect_alloc_fd mj_read_fd "${pipe}" |
91 |
- redirect_alloc_fd mj_write_fd "${pipe}" '>' |
92 |
- rm -f "${pipe}" |
93 |
- |
94 |
- # See how many children we can fork based on the user's settings. |
95 |
- mj_max_jobs=$(makeopts_jobs "$@") |
96 |
- mj_num_jobs=0 |
97 |
-} |
98 |
- |
99 |
-# @FUNCTION: multijob_child_init |
100 |
-# @USAGE: [--pre|--post] [command to run in background] |
101 |
-# @DESCRIPTION: |
102 |
-# This function has two forms. You can use it to execute a simple command |
103 |
-# in the background (and it takes care of everything else), or you must |
104 |
-# call this first thing in your forked child process. |
105 |
-# |
106 |
-# The --pre/--post options allow you to select the child generation mode. |
107 |
-# |
108 |
-# @CODE |
109 |
-# # 1st form: pass the command line as arguments: |
110 |
-# multijob_child_init ls /dev |
111 |
-# # Or if you want to use pre/post fork modes: |
112 |
-# multijob_child_init --pre ls /dev |
113 |
-# multijob_child_init --post ls /dev |
114 |
-# |
115 |
-# # 2nd form: execute multiple stuff in the background (post fork): |
116 |
-# ( |
117 |
-# multijob_child_init |
118 |
-# out=`ls` |
119 |
-# if echo "${out}" | grep foo ; then |
120 |
-# echo "YEAH" |
121 |
-# fi |
122 |
-# ) & |
123 |
-# multijob_post_fork |
124 |
-# |
125 |
-# # 2nd form: execute multiple stuff in the background (pre fork): |
126 |
-# multijob_pre_fork |
127 |
-# ( |
128 |
-# multijob_child_init |
129 |
-# out=`ls` |
130 |
-# if echo "${out}" | grep foo ; then |
131 |
-# echo "YEAH" |
132 |
-# fi |
133 |
-# ) & |
134 |
-# @CODE |
135 |
-multijob_child_init() { |
136 |
- local mode="pre" |
137 |
- case $1 in |
138 |
- --pre) mode="pre" ; shift ;; |
139 |
- --post) mode="post"; shift ;; |
140 |
- esac |
141 |
- |
142 |
- if [[ $# -eq 0 ]] ; then |
143 |
- trap 'echo ${BASHPID:-$(bashpid)} $? >&'${mj_write_fd} EXIT |
144 |
- trap 'exit 1' INT TERM |
145 |
- else |
146 |
- local ret |
147 |
- [[ ${mode} == "pre" ]] && { multijob_pre_fork; ret=$?; } |
148 |
- ( multijob_child_init ; "$@" ) & |
149 |
- [[ ${mode} == "post" ]] && { multijob_post_fork; ret=$?; } |
150 |
- return ${ret} |
151 |
- fi |
152 |
-} |
153 |
- |
154 |
-# @FUNCTION: _multijob_fork |
155 |
-# @INTERNAL |
156 |
-# @DESCRIPTION: |
157 |
-# Do the actual book keeping. |
158 |
-_multijob_fork() { |
159 |
- [[ $# -eq 1 ]] || die "incorrect number of arguments" |
160 |
- |
161 |
- local ret=0 |
162 |
- [[ $1 == "post" ]] && : $(( ++mj_num_jobs )) |
163 |
- if [[ ${mj_num_jobs} -ge ${mj_max_jobs} ]] ; then |
164 |
- multijob_finish_one |
165 |
- ret=$? |
166 |
- fi |
167 |
- [[ $1 == "pre" ]] && : $(( ++mj_num_jobs )) |
168 |
- return ${ret} |
169 |
-} |
170 |
- |
171 |
-# @FUNCTION: multijob_pre_fork |
172 |
-# @DESCRIPTION: |
173 |
-# You must call this in the parent process before forking a child process. |
174 |
-# If the parallel limit has been hit, it will wait for one child to finish |
175 |
-# and return its exit status. |
176 |
-multijob_pre_fork() { _multijob_fork pre "$@" ; } |
177 |
- |
178 |
-# @FUNCTION: multijob_post_fork |
179 |
-# @DESCRIPTION: |
180 |
-# You must call this in the parent process after forking a child process. |
181 |
-# If the parallel limit has been hit, it will wait for one child to finish |
182 |
-# and return its exit status. |
183 |
-multijob_post_fork() { _multijob_fork post "$@" ; } |
184 |
- |
185 |
-# @FUNCTION: multijob_finish_one |
186 |
-# @DESCRIPTION: |
187 |
-# Wait for a single process to exit and return its exit code. |
188 |
-multijob_finish_one() { |
189 |
- [[ $# -eq 0 ]] || die "${FUNCNAME} takes no arguments" |
190 |
- |
191 |
- local pid ret |
192 |
- read -r -u ${mj_read_fd} pid ret || die |
193 |
- : $(( --mj_num_jobs )) |
194 |
- return ${ret} |
195 |
-} |
196 |
- |
197 |
-# @FUNCTION: multijob_finish |
198 |
-# @DESCRIPTION: |
199 |
-# Wait for all pending processes to exit and return the bitwise or |
200 |
-# of all their exit codes. |
201 |
-multijob_finish() { |
202 |
- local ret=0 |
203 |
- while [[ ${mj_num_jobs} -gt 0 ]] ; do |
204 |
- multijob_finish_one |
205 |
- : $(( ret |= $? )) |
206 |
- done |
207 |
- # Let bash clean up its internal child tracking state. |
208 |
- wait |
209 |
- |
210 |
- # Do this after reaping all the children. |
211 |
- [[ $# -eq 0 ]] || die "${FUNCNAME} takes no arguments" |
212 |
- |
213 |
- # No need to hook anymore. |
214 |
- EBUILD_DEATH_HOOKS=${EBUILD_DEATH_HOOKS/ wait / } |
215 |
- |
216 |
- return ${ret} |
217 |
-} |
218 |
- |
219 |
# @FUNCTION: redirect_alloc_fd |
220 |
# @USAGE: <var> <file> [redirection] |
221 |
# @DESCRIPTION: |
222 |
diff --git a/eclass/tests/multiprocessing.sh b/eclass/tests/multiprocessing.sh |
223 |
deleted file mode 100755 |
224 |
index 1ceb7f7fa7b2..000000000000 |
225 |
--- a/eclass/tests/multiprocessing.sh |
226 |
+++ /dev/null |
227 |
@@ -1,42 +0,0 @@ |
228 |
-#!/bin/bash |
229 |
-# Copyright 1999-2015 Gentoo Foundation |
230 |
-# Distributed under the terms of the GNU General Public License v2 |
231 |
- |
232 |
-source tests-common.sh |
233 |
- |
234 |
-inherit multiprocessing |
235 |
- |
236 |
-tbegin "simple" |
237 |
-MAKEOPTS="-j1" multijob_init |
238 |
-multijob_child_init ls -d / >/dev/null || die "fail!" |
239 |
-multijob_finish |
240 |
-tend $? |
241 |
- |
242 |
-tbegin "less simple" |
243 |
-multijob_init -j3 |
244 |
-multijob_child_init true || die "fail!" |
245 |
-multijob_child_init false || die "fail!" |
246 |
-multijob_child_init true || die "fail!" |
247 |
-multijob_finish |
248 |
-tend $(( $? == 1 ? 0 : 1 )) |
249 |
- |
250 |
-tbegin "less less simple" |
251 |
-multijob_init -j1 |
252 |
-multijob_child_init true || die "fail!" |
253 |
-multijob_child_init false || die "fail!" |
254 |
-multijob_child_init true && die "fail!" |
255 |
-multijob_finish |
256 |
-tend $? |
257 |
- |
258 |
-tbegin "less less less simple" |
259 |
-multijob_init -j10 |
260 |
-multijob_child_init true || die "fail!" |
261 |
-multijob_finish_one || die "fail!" |
262 |
-multijob_child_init false || die "fail!" |
263 |
-multijob_finish_one && die "fail!" |
264 |
-multijob_child_init true || die "fail!" |
265 |
-multijob_finish_one || die "fail!" |
266 |
-multijob_finish |
267 |
-tend $? |
268 |
- |
269 |
-texit |
270 |
-- |
271 |
2.16.0.rc0 |