Gentoo Archives: gentoo-dev

From: "Michał Górny" <mgorny@g.o>
To: gentoo-dev@l.g.o
Cc: "Michał Górny" <mgorny@g.o>
Subject: [gentoo-dev] [PATCH v2] multiprocessing.eclass: Default makeopts_jobs to inf=nproc+1
Date: Thu, 28 Apr 2022 13:35:23
Message-Id: 20220428133510.17866-1-mgorny@gentoo.org
1 Change the default value for 'inf' argument to makeopts_jobs from 999
2 to $(get_nproc) + 1. This means that if MAKEOPTS specifies a `-j`
3 argument without a specific value, nproc will be used rather than
4 infinity-ish number of jobs.
5
6 The old default made sense for ebuilds using both makeopts_jobs
7 and makeopts_loadavg. However, these are very rare — only 4 packages
8 and 3 eclass at this time. For the remaining ebuilds, they meant
9 uncontrollably using up to 999 jobs.
10
11 The new default is both safer and more correct for the vast majority
12 of Gentoo packages, removing the necessity of repeating:
13
14 $(makeopts_jobs "${MAKEOPTS}" "$(get_nproc)")
15
16 The ebuilds and eclasses using makeopts_loadavg have been updated
17 to pass the old default.
18
19 Signed-off-by: Michał Górny <mgorny@g.o>
20 ---
21 eclass/multiprocessing.eclass | 9 ++++-----
22 eclass/tests/multiprocessing_makeopts_jobs.sh | 15 ++++++++++-----
23 2 files changed, 14 insertions(+), 10 deletions(-)
24
25 Changes in v2:
26 - changed the default to nproc+1 as suggested by Flow, thanks!
27
28 diff --git a/eclass/multiprocessing.eclass b/eclass/multiprocessing.eclass
29 index c32bfaac2e6b..b235c74a5baa 100644
30 --- a/eclass/multiprocessing.eclass
31 +++ b/eclass/multiprocessing.eclass
32 @@ -65,22 +65,21 @@ get_nproc() {
33 }
34
35 # @FUNCTION: makeopts_jobs
36 -# @USAGE: [${MAKEOPTS}] [${inf:-999}]
37 +# @USAGE: [${MAKEOPTS}] [${inf:-$(( $(get_nproc) + 1 ))}]
38 # @DESCRIPTION:
39 # Searches the arguments (defaults to ${MAKEOPTS}) and extracts the jobs number
40 # specified therein. Useful for running non-make tools in parallel too.
41 # i.e. if the user has MAKEOPTS=-j9, this will echo "9" -- we can't return the
42 # number as bash normalizes it to [0, 255]. If the flags haven't specified a
43 -# -j flag, then "1" is shown as that is the default `make` uses. Since there's
44 -# no way to represent infinity, we return ${inf} (defaults to 999) if the user
45 -# has -j without a number.
46 +# -j flag, then "1" is shown as that is the default `make` uses. If the flags
47 +# specify -j without a number, ${inf} is returned (defaults to nproc).
48 makeopts_jobs() {
49 [[ $# -eq 0 ]] && set -- "${MAKEOPTS}"
50 # This assumes the first .* will be more greedy than the second .*
51 # since POSIX doesn't specify a non-greedy match (i.e. ".*?").
52 local jobs=$(echo " $* " | sed -r -n \
53 -e 's:.*[[:space:]](-[a-z]*j|--jobs[=[:space:]])[[:space:]]*([0-9]+).*:\2:p' \
54 - -e "s:.*[[:space:]](-[a-z]*j|--jobs)[[:space:]].*:${2:-999}:p")
55 + -e "s:.*[[:space:]](-[a-z]*j|--jobs)[[:space:]].*:${2:-$(( $(get_nproc) + 1 ))}:p")
56 echo ${jobs:-1}
57 }
58
59 diff --git a/eclass/tests/multiprocessing_makeopts_jobs.sh b/eclass/tests/multiprocessing_makeopts_jobs.sh
60 index 70a6085d5362..37d5a7257775 100755
61 --- a/eclass/tests/multiprocessing_makeopts_jobs.sh
62 +++ b/eclass/tests/multiprocessing_makeopts_jobs.sh
63 @@ -16,14 +16,19 @@ test-makeopts_jobs() {
64 tend 1 "Mismatch between MAKEOPTS/cli: '${indirect}' != '${direct}'"
65 else
66 [[ ${direct} == "${exp}" ]]
67 - tend $? "Got back: ${act}"
68 + tend $? "Got back: ${direct}"
69 fi
70 }
71
72 +# override to avoid relying on a specific value
73 +get_nproc() {
74 + echo 41
75 +}
76 +
77 tests=(
78 - 999 "-j"
79 - 999 "--jobs"
80 - 999 "-j -l9"
81 + 42 "-j"
82 + 42 "--jobs"
83 + 42 "-j -l9"
84 1 ""
85 1 "-l9 -w"
86 1 "-l9 -w-j4"
87 @@ -37,7 +42,7 @@ tests=(
88 7 "-l3 --jobs 7 -w"
89 4 "-j1 -j 2 --jobs 3 --jobs=4"
90 8 " -j 8 "
91 - 999 "-kj"
92 + 42 "-kj"
93 4 "-kj4"
94 5 "-kj 5"
95 )
96 --
97 2.35.1