1 |
From: Douglas Anderson <dianders@××××××××.org> |
2 |
|
3 |
When trying to figure out why it took so long to do a no-op kernel |
4 |
build (re-build when nothing changed) on Chrome OS, I tracked down one |
5 |
slowdown to cros-kernel2_src_configure(). This function was taking |
6 |
~900 ms to execute. |
7 |
|
8 |
The bulk of that slowdown was in iterating over the list of config |
9 |
fragments, specifically the "use ${fragment}" test. We currently have |
10 |
77 fragments so we were effectively calling the "use" function 77 |
11 |
times. |
12 |
|
13 |
Digging through the portage code, the slow part of the "use" function |
14 |
was the block of code to confirm that you specified each USE flag in |
15 |
your IUSE. Commenting out the whole "elif" block of code there sped |
16 |
things up so that the entire cros-kernel2_src_configure() was now |
17 |
taking ~130 ms. This means that each call to the "use" function was |
18 |
taking about 10 ms. |
19 |
|
20 |
The specific part of the test that was slow was testing against the |
21 |
regular expression. It was specifically slow in the Chrome OS kernel |
22 |
build because we inherit the "cros-board" eclass which populates a |
23 |
huge number of boards in the USE flag, making the regular expression |
24 |
totally unwieldly. |
25 |
|
26 |
One way to speed this whole thing up is to use a bash associative |
27 |
array. Unfortunately arrays can't come in through environment |
28 |
variables, so we'll write a function that declares the array the first |
29 |
time it's needed. |
30 |
|
31 |
With this version of the code cros-kernel2_src_configure() now takes |
32 |
~190 ms which seems like it's OK. AKA 77 checks against IUSE took 60 |
33 |
ms or less than 1 ms per check. |
34 |
|
35 |
NOTE: to keep EAPI 4 and older working, we keep doing the regular |
36 |
expression tests there, though we now do it in the __in_portage_iuse() |
37 |
function. In at least one test the extra overhead of the function |
38 |
made testing USE flags on EAPI 4 ~15% slower, but presumably this is |
39 |
OK as we want to encourage folks to move to the newer EAPIs. |
40 |
|
41 |
BUG=chromium:767073 |
42 |
TEST=Time some builds; confirm bad use flags still caught. |
43 |
|
44 |
Change-Id: Ic74fa49bdf002399ba0d6c41f42d4632b07127a9 |
45 |
Reviewed-on: https://chromium-review.googlesource.com/1524641 |
46 |
Commit-Ready: Douglas Anderson <dianders@××××××××.org> |
47 |
Tested-by: Douglas Anderson <dianders@××××××××.org> |
48 |
Reviewed-by: Douglas Anderson <dianders@××××××××.org> |
49 |
See: https://chromium.googlesource.com/chromiumos/third_party/portage_tool/+/82a0776602df5707606de2099b93b8b7b1cc34a1 |
50 |
Bug: https://bugs.gentoo.org/680810 |
51 |
Signed-off-by: Zac Medico <zmedico@g.o> |
52 |
--- |
53 |
bin/ebuild.sh | 4 ++-- |
54 |
bin/phase-helpers.sh | 6 +++--- |
55 |
.../ebuild/_config/special_env_vars.py | 7 ++++--- |
56 |
lib/portage/package/ebuild/config.py | 20 ++++++++++++++++--- |
57 |
4 files changed, 26 insertions(+), 11 deletions(-) |
58 |
|
59 |
diff --git a/bin/ebuild.sh b/bin/ebuild.sh |
60 |
index d3bf0fd29..20dff6f3f 100755 |
61 |
--- a/bin/ebuild.sh |
62 |
+++ b/bin/ebuild.sh |
63 |
@@ -763,8 +763,8 @@ else |
64 |
|
65 |
# If ${EBUILD_FORCE_TEST} == 1 and USE came from ${T}/environment |
66 |
# then it might not have USE=test like it's supposed to here. |
67 |
- if [[ ${EBUILD_PHASE} == test && ${EBUILD_FORCE_TEST} == 1 && |
68 |
- test =~ ${PORTAGE_IUSE} ]] && ! has test ${USE} ; then |
69 |
+ if [[ ${EBUILD_PHASE} == test && ${EBUILD_FORCE_TEST} == 1 ]] && |
70 |
+ ___in_portage_iuse test && ! has test ${USE} ; then |
71 |
export USE="${USE} test" |
72 |
fi |
73 |
declare -r USE |
74 |
diff --git a/bin/phase-helpers.sh b/bin/phase-helpers.sh |
75 |
index ba3f27930..64a82b4b7 100644 |
76 |
--- a/bin/phase-helpers.sh |
77 |
+++ b/bin/phase-helpers.sh |
78 |
@@ -237,9 +237,9 @@ use() { |
79 |
# Make sure we have this USE flag in IUSE, but exempt binary |
80 |
# packages for API consumers like Entropy which do not require |
81 |
# a full profile with IUSE_IMPLICIT and stuff (see bug #456830). |
82 |
- elif [[ -n $PORTAGE_IUSE && -n $EBUILD_PHASE && |
83 |
- -n $PORTAGE_INTERNAL_CALLER ]] ; then |
84 |
- if [[ ! $u =~ $PORTAGE_IUSE ]] ; then |
85 |
+ elif declare -f ___in_portage_iuse >/dev/null && |
86 |
+ [[ -n ${EBUILD_PHASE} && -n ${PORTAGE_INTERNAL_CALLER} ]] ; then |
87 |
+ if ! ___in_portage_iuse "${u}"; then |
88 |
if [[ ${EMERGE_FROM} != binary && |
89 |
! ${EAPI} =~ ^(0|1|2|3|4|4-python|4-slot-abi)$ ]] ; then |
90 |
# This is only strict starting with EAPI 5, since implicit IUSE |
91 |
diff --git a/lib/portage/package/ebuild/_config/special_env_vars.py b/lib/portage/package/ebuild/_config/special_env_vars.py |
92 |
index f9a0c3c0e..e72049e33 100644 |
93 |
--- a/lib/portage/package/ebuild/_config/special_env_vars.py |
94 |
+++ b/lib/portage/package/ebuild/_config/special_env_vars.py |
95 |
@@ -14,8 +14,8 @@ import re |
96 |
# to enter the config instance from the external environment or |
97 |
# configuration files. |
98 |
env_blacklist = frozenset(( |
99 |
- "A", "AA", "BDEPEND", "BROOT", "CATEGORY", "DEPEND", "DESCRIPTION", |
100 |
- "DOCS", "EAPI", |
101 |
+ "A", "AA", "BASH_FUNC____in_portage_iuse%%", "BDEPEND", "BROOT", |
102 |
+ "CATEGORY", "DEPEND", "DESCRIPTION", "DOCS", "EAPI", |
103 |
"EBUILD_FORCE_TEST", "EBUILD_PHASE", |
104 |
"EBUILD_PHASE_FUNC", "EBUILD_SKIP_MANIFEST", |
105 |
"ED", "EMERGE_FROM", "EPREFIX", "EROOT", |
106 |
@@ -42,7 +42,8 @@ environ_whitelist = [] |
107 |
# environment in order to prevent sandbox from sourcing /etc/profile |
108 |
# in it's bashrc (causing major leakage). |
109 |
environ_whitelist += [ |
110 |
- "ACCEPT_LICENSE", "BASH_ENV", "BROOT", "BUILD_PREFIX", "COLUMNS", "D", |
111 |
+ "ACCEPT_LICENSE", "BASH_ENV", "BASH_FUNC____in_portage_iuse%%", |
112 |
+ "BROOT", "BUILD_PREFIX", "COLUMNS", "D", |
113 |
"DISTDIR", "DOC_SYMLINKS_DIR", "EAPI", "EBUILD", |
114 |
"EBUILD_FORCE_TEST", |
115 |
"EBUILD_PHASE", "EBUILD_PHASE_FUNC", "ECLASSDIR", "ECLASS_DEPTH", "ED", |
116 |
diff --git a/lib/portage/package/ebuild/config.py b/lib/portage/package/ebuild/config.py |
117 |
index 2981f7e31..cc2413989 100644 |
118 |
--- a/lib/portage/package/ebuild/config.py |
119 |
+++ b/lib/portage/package/ebuild/config.py |
120 |
@@ -1762,13 +1762,27 @@ class config(object): |
121 |
portage_iuse.update(built_use) |
122 |
self.configdict["pkg"]["IUSE_EFFECTIVE"] = \ |
123 |
" ".join(sorted(portage_iuse)) |
124 |
+ |
125 |
+ self.configdict["env"]["BASH_FUNC____in_portage_iuse%%"] = ( |
126 |
+ "() { " |
127 |
+ "if [[ ${#___PORTAGE_IUSE_HASH[@]} -lt 1 ]]; then " |
128 |
+ " declare -gA ___PORTAGE_IUSE_HASH=(%s); " |
129 |
+ "fi; " |
130 |
+ "[[ -n ${___PORTAGE_IUSE_HASH[$1]} ]]; " |
131 |
+ "}" ) % " ".join('["%s"]=1' % x for x in portage_iuse) |
132 |
else: |
133 |
portage_iuse = self._get_implicit_iuse() |
134 |
portage_iuse.update(explicit_iuse) |
135 |
|
136 |
- # PORTAGE_IUSE is not always needed so it's lazily evaluated. |
137 |
- self.configdict["env"].addLazySingleton( |
138 |
- "PORTAGE_IUSE", _lazy_iuse_regex, portage_iuse) |
139 |
+ # The _get_implicit_iuse() returns a regular expression |
140 |
+ # so we can't use the (faster) map. Fall back to |
141 |
+ # implementing ___in_portage_iuse() the older/slower way. |
142 |
+ |
143 |
+ # PORTAGE_IUSE is not always needed so it's lazily evaluated. |
144 |
+ self.configdict["env"].addLazySingleton( |
145 |
+ "PORTAGE_IUSE", _lazy_iuse_regex, portage_iuse) |
146 |
+ self.configdict["env"]["BASH_FUNC____in_portage_iuse%%"] = \ |
147 |
+ "() { [[ $1 =~ ${PORTAGE_IUSE} ]]; }" |
148 |
|
149 |
ebuild_force_test = not restrict_test and \ |
150 |
self.get("EBUILD_FORCE_TEST") == "1" |
151 |
-- |
152 |
2.21.0 |