[gentoo-commits] repo/gentoo:master commit in: eclass/tests/, eclass/

[Michał Górny]

commit:     0e50583c165feedf24152dae1973af88e0005191
Author:     Michał Górny <mgorny <AT> gentoo <DOT> org>
AuthorDate: Sun Dec 31 10:29:42 2017 +0000
Commit:     Michał Górny <mgorny <AT> gentoo <DOT> org>
CommitDate: Thu Jan  4 21:56:14 2018 +0000
URL:        https://gitweb.gentoo.org/repo/gentoo.git/commit/?id=0e50583c

multiprocessing.eclass: Remove multijob_* functions

Remove the multijob functions that were used to run bash code
in parallel. The code was very complex, fragile and unmaintained. It has
been used scarcely, and pretty much by a single developer. It gave very
little gain, usually at the cost of losing readability and violating
PMS.

Closes: https://bugs.gentoo.org/613322

 eclass/multiprocessing.eclass   | 178 +++-------------------------------------
 eclass/tests/multiprocessing.sh |  42 ----------
 2 files changed, 10 insertions(+), 210 deletions(-)

diff --git a/eclass/multiprocessing.eclass b/eclass/multiprocessing.eclass
index b6e92976f73..cfe22303043 100644
--- a/eclass/multiprocessing.eclass
+++ b/eclass/multiprocessing.eclass
@@ -1,4 +1,4 @@
-# Copyright 1999-2014 Gentoo Foundation
+# Copyright 1999-2017 Gentoo Foundation
 # Distributed under the terms of the GNU General Public License v2
 
 # @ECLASS: multiprocessing.eclass
@@ -7,29 +7,20 @@
 # @AUTHOR:
 # Brian Harring <ferringb@g.o>
 # Mike Frysinger <vapier@g.o>
-# @BLURB: parallelization with bash (wtf?)
+# @BLURB: multiprocessing helper functions
 # @DESCRIPTION:
-# The multiprocessing eclass contains a suite of functions that allow ebuilds
-# to quickly run things in parallel using shell code.
+# The multiprocessing eclass contains a suite of utility functions
+# that could be helpful to controlling parallel multiple job execution.
+# The most common use is processing MAKEOPTS in order to obtain job
+# count.
 #
-# It has two modes: pre-fork and post-fork.  If you don't want to dive into any
-# more nuts & bolts, just use the pre-fork mode.  For main threads that mostly
-# spawn children and then wait for them to finish, use the pre-fork mode.  For
-# main threads that do a bit of processing themselves, use the post-fork mode.
-# You may mix & match them for longer computation loops.
 # @EXAMPLE:
 #
 # @CODE
-# # First initialize things:
-# multijob_init
-#
-# # Then hash a bunch of files in parallel:
-# for n in {0..20} ; do
-# 	multijob_child_init md5sum data.${n} > data.${n}
-# done
-#
-# # Then wait for all the children to finish:
-# multijob_finish
+# src_compile() {
+#   # custom build system that does not support most of MAKEOPTS
+#   ./mybs -j$(makeopts_jobs)
+# }
 # @CODE
 
 if [[ -z ${_MULTIPROCESSING_ECLASS} ]]; then
@@ -126,155 +117,6 @@ makeopts_loadavg() {
 	echo ${lavg:-${2:-999}}
 }
 
-# @FUNCTION: multijob_init
-# @USAGE: [${MAKEOPTS}]
-# @DESCRIPTION:
-# Setup the environment for executing code in parallel.
-# You must call this before any other multijob function.
-multijob_init() {
-	# When something goes wrong, try to wait for all the children so we
-	# don't leave any zombies around.
-	has wait ${EBUILD_DEATH_HOOKS} || EBUILD_DEATH_HOOKS+=" wait "
-
-	# Setup a pipe for children to write their pids to when they finish.
-	# We have to allocate two fd's because POSIX has undefined behavior
-	# when using one single fd for both read and write. #487056
-	# However, opening an fd for read or write only will block until the
-	# opposite end is opened as well. Thus we open the first fd for both
-	# read and write to not block ourselve, but use it for reading only.
-	# The second fd really is opened for write only, as Cygwin supports
-	# just one single read fd per FIFO. #583962
-	local pipe="${T}/multijob.pipe"
-	mkfifo -m 600 "${pipe}"
-	redirect_alloc_fd mj_read_fd "${pipe}"
-	redirect_alloc_fd mj_write_fd "${pipe}" '>'
-	rm -f "${pipe}"
-
-	# See how many children we can fork based on the user's settings.
-	mj_max_jobs=$(makeopts_jobs "$@")
-	mj_num_jobs=0
-}
-
-# @FUNCTION: multijob_child_init
-# @USAGE: [--pre|--post] [command to run in background]
-# @DESCRIPTION:
-# This function has two forms.  You can use it to execute a simple command
-# in the background (and it takes care of everything else), or you must
-# call this first thing in your forked child process.
-#
-# The --pre/--post options allow you to select the child generation mode.
-#
-# @CODE
-# # 1st form: pass the command line as arguments:
-# multijob_child_init ls /dev
-# # Or if you want to use pre/post fork modes:
-# multijob_child_init --pre ls /dev
-# multijob_child_init --post ls /dev
-#
-# # 2nd form: execute multiple stuff in the background (post fork):
-# (
-# multijob_child_init
-# out=`ls`
-# if echo "${out}" | grep foo ; then
-# 	echo "YEAH"
-# fi
-# ) &
-# multijob_post_fork
-#
-# # 2nd form: execute multiple stuff in the background (pre fork):
-# multijob_pre_fork
-# (
-# multijob_child_init
-# out=`ls`
-# if echo "${out}" | grep foo ; then
-# 	echo "YEAH"
-# fi
-# ) &
-# @CODE
-multijob_child_init() {
-	local mode="pre"
-	case $1 in
-	--pre)  mode="pre" ; shift ;;
-	--post) mode="post"; shift ;;
-	esac
-
-	if [[ $# -eq 0 ]] ; then
-		trap 'echo ${BASHPID:-$(bashpid)} $? >&'${mj_write_fd} EXIT
-		trap 'exit 1' INT TERM
-	else
-		local ret
-		[[ ${mode} == "pre" ]] && { multijob_pre_fork; ret=$?; }
-		( multijob_child_init ; "$@" ) &
-		[[ ${mode} == "post" ]] && { multijob_post_fork; ret=$?; }
-		return ${ret}
-	fi
-}
-
-# @FUNCTION: _multijob_fork
-# @INTERNAL
-# @DESCRIPTION:
-# Do the actual book keeping.
-_multijob_fork() {
-	[[ $# -eq 1 ]] || die "incorrect number of arguments"
-
-	local ret=0
-	[[ $1 == "post" ]] && : $(( ++mj_num_jobs ))
-	if [[ ${mj_num_jobs} -ge ${mj_max_jobs} ]] ; then
-		multijob_finish_one
-		ret=$?
-	fi
-	[[ $1 == "pre" ]] && : $(( ++mj_num_jobs ))
-	return ${ret}
-}
-
-# @FUNCTION: multijob_pre_fork
-# @DESCRIPTION:
-# You must call this in the parent process before forking a child process.
-# If the parallel limit has been hit, it will wait for one child to finish
-# and return its exit status.
-multijob_pre_fork() { _multijob_fork pre "$@" ; }
-
-# @FUNCTION: multijob_post_fork
-# @DESCRIPTION:
-# You must call this in the parent process after forking a child process.
-# If the parallel limit has been hit, it will wait for one child to finish
-# and return its exit status.
-multijob_post_fork() { _multijob_fork post "$@" ; }
-
-# @FUNCTION: multijob_finish_one
-# @DESCRIPTION:
-# Wait for a single process to exit and return its exit code.
-multijob_finish_one() {
-	[[ $# -eq 0 ]] || die "${FUNCNAME} takes no arguments"
-
-	local pid ret
-	read -r -u ${mj_read_fd} pid ret || die
-	: $(( --mj_num_jobs ))
-	return ${ret}
-}
-
-# @FUNCTION: multijob_finish
-# @DESCRIPTION:
-# Wait for all pending processes to exit and return the bitwise or
-# of all their exit codes.
-multijob_finish() {
-	local ret=0
-	while [[ ${mj_num_jobs} -gt 0 ]] ; do
-		multijob_finish_one
-		: $(( ret |= $? ))
-	done
-	# Let bash clean up its internal child tracking state.
-	wait
-
-	# Do this after reaping all the children.
-	[[ $# -eq 0 ]] || die "${FUNCNAME} takes no arguments"
-
-	# No need to hook anymore.
-	EBUILD_DEATH_HOOKS=${EBUILD_DEATH_HOOKS/ wait / }
-
-	return ${ret}
-}
-
 # @FUNCTION: redirect_alloc_fd
 # @USAGE: <var> <file> [redirection]
 # @DESCRIPTION:

diff --git a/eclass/tests/multiprocessing.sh b/eclass/tests/multiprocessing.sh
deleted file mode 100755
index 1ceb7f7fa7b..00000000000
--- a/eclass/tests/multiprocessing.sh
+++ /dev/null
@@ -1,42 +0,0 @@
-#!/bin/bash
-# Copyright 1999-2015 Gentoo Foundation
-# Distributed under the terms of the GNU General Public License v2
-
-source tests-common.sh
-
-inherit multiprocessing
-
-tbegin "simple"
-MAKEOPTS="-j1" multijob_init
-multijob_child_init ls -d / >/dev/null || die "fail!"
-multijob_finish
-tend $?
-
-tbegin "less simple"
-multijob_init -j3
-multijob_child_init true  || die "fail!"
-multijob_child_init false || die "fail!"
-multijob_child_init true  || die "fail!"
-multijob_finish
-tend $(( $? == 1 ? 0 : 1 ))
-
-tbegin "less less simple"
-multijob_init -j1
-multijob_child_init true  || die "fail!"
-multijob_child_init false || die "fail!"
-multijob_child_init true  && die "fail!"
-multijob_finish
-tend $?
-
-tbegin "less less less simple"
-multijob_init -j10
-multijob_child_init true  || die "fail!"
-multijob_finish_one       || die "fail!"
-multijob_child_init false || die "fail!"
-multijob_finish_one       && die "fail!"
-multijob_child_init true  || die "fail!"
-multijob_finish_one       || die "fail!"
-multijob_finish
-tend $?
-
-texit