Gentoo Archives: gentoo-portage-dev

From: Alec Warner <antarus@g.o>
To: gentoo-portage-dev@l.g.o
Cc: Zac Medico <zmedico@g.o>
Subject: Re: [gentoo-portage-dev] [PATCH] config.environ: delay export of A and AA (bug 720180)
Date: Tue, 26 May 2020 08:43:24
Message-Id: CAAr7Pr9Z9xWg3H51Z3NAYYQVYVgy6UGKsT-9wWWm+kF4p-ZuYg@mail.gmail.com
In Reply to: [gentoo-portage-dev] [PATCH] config.environ: delay export of A and AA (bug 720180) by Zac Medico
1 On Mon, May 25, 2020 at 9:34 PM Zac Medico <zmedico@g.o> wrote:
2
3 > Since variables like A and AA can contain extremely large values which
4 > may trigger E2BIG errors during attempts to execute subprocesses, delay
5 > export until the last moment, and unexport when appropriate.
6 >
7
8 So I think if you want to do this because PMS says:
9 AA should not be visible in EAPI > 3.
10 A should only be visible in src_*, pkg_nofetch.
11
12 That part of the patch makes sense to me. The part that is confusing to me
13 is the 'delay' part; can you explain that further? When you say "delay
14 until the last moment" what do you mean by that and what value is it
15 delivering?
16
17 Is it simply that we don't export these variables on the python side, and
18 we only use them in the shell portion?
19
20 -A
21
22
23 > Bug: https://bugs.gentoo.org/720180
24 > Signed-off-by: Zac Medico <zmedico@g.o>
25 > ---
26 > bin/eapi.sh | 9 +++++
27 > bin/isolated-functions.sh | 34 ++++++++++++++++
28 > bin/phase-functions.sh | 13 +++++++
29 > .../ebuild/_config/special_env_vars.py | 7 +++-
30 > lib/portage/package/ebuild/config.py | 39 ++++++++++++++-----
31 > 5 files changed, 91 insertions(+), 11 deletions(-)
32 >
33 > diff --git a/bin/eapi.sh b/bin/eapi.sh
34 > index 29dfb008c..f56468e4a 100644
35 > --- a/bin/eapi.sh
36 > +++ b/bin/eapi.sh
37 > @@ -26,6 +26,15 @@ ___eapi_has_S_WORKDIR_fallback() {
38 >
39 > # VARIABLES
40 >
41 > +___eapi_exports_A() {
42 > + # https://bugs.gentoo.org/721088
43 > + true
44 > +}
45 > +
46 > +___eapi_exports_AA() {
47 > + [[ ${1-${EAPI-0}} =~ ^(0|1|2|3)$ ]]
48 > +}
49 > +
50 > ___eapi_has_prefix_variables() {
51 > [[ ! ${1-${EAPI-0}} =~ ^(0|1|2)$ || " ${FEATURES} " == *"
52 > force-prefix "* ]]
53 > }
54 > diff --git a/bin/isolated-functions.sh b/bin/isolated-functions.sh
55 > index fde684013..973450d86 100644
56 > --- a/bin/isolated-functions.sh
57 > +++ b/bin/isolated-functions.sh
58 > @@ -107,6 +107,39 @@ __bashpid() {
59 > sh -c 'echo ${PPID}'
60 > }
61 >
62 > +# @FUNCTION: ___eapi_vars_export
63 > +# @DESCRIPTION:
64 > +# Export variables for the current EAPI. Calls to this function should
65 > +# be delayed until the last moment, since exporting these variables
66 > +# may trigger E2BIG errors suring attempts to execute subprocesses.
67 > +___eapi_vars_export() {
68 > + source "${T}/environment.unexported" || die
69 > +
70 > + if ___eapi_exports_A; then
71 > + export A
72 > + fi
73 > +
74 > + if ___eapi_exports_AA; then
75 > + export AA
76 > + fi
77 > +}
78 > +
79 > +# @FUNCTION: ___eapi_vars_unexport
80 > +# @DESCRIPTION:
81 > +# Unexport variables that were exported for the current EAPI. This
82 > +# function should be called after an ebuild phase, in order to unexport
83 > +# variables that may trigger E2BIG errors during attempts to execute
84 > +# subprocesses.
85 > +___eapi_vars_unexport() {
86 > + if ___eapi_exports_A; then
87 > + export -n A
88 > + fi
89 > +
90 > + if ___eapi_exports_AA; then
91 > + export -n AA
92 > + fi
93 > +}
94 > +
95 > __helpers_die() {
96 > if ___eapi_helpers_can_die && [[ ${PORTAGE_NONFATAL} != 1 ]]; then
97 > die "$@"
98 > @@ -122,6 +155,7 @@ die() {
99 >
100 > set +x # tracing only produces useless noise here
101 > local IFS=$' \t\n'
102 > + ___eapi_vars_unexport
103 >
104 > if ___eapi_die_can_respect_nonfatal && [[ $1 == -n ]]; then
105 > shift
106 > diff --git a/bin/phase-functions.sh b/bin/phase-functions.sh
107 > index 90e622e75..df2c0d8de 100644
108 > --- a/bin/phase-functions.sh
109 > +++ b/bin/phase-functions.sh
110 > @@ -146,6 +146,7 @@ __filter_readonly_variables() {
111 > fi
112 > fi
113 >
114 > + ___eapi_vars_unexport
115 > "${PORTAGE_PYTHON:-/usr/bin/python}"
116 > "${PORTAGE_BIN_PATH}"/filter-bash-environment.py "${filtered_vars}" || die
117 > "filter-bash-environment.py failed"
118 > }
119 >
120 > @@ -212,6 +213,7 @@ __ebuild_phase() {
121 >
122 > __ebuild_phase_with_hooks() {
123 > local x phase_name=${1}
124 > + ___eapi_vars_export
125 > for x in {pre_,,post_}${phase_name} ; do
126 > __ebuild_phase ${x}
127 > done
128 > @@ -223,6 +225,7 @@ __dyn_pretend() {
129 > __vecho ">>> Remove '$PORTAGE_BUILDDIR/.pretended' to
130 > force pretend."
131 > return 0
132 > fi
133 > + ___eapi_vars_export
134 > __ebuild_phase pre_pkg_pretend
135 > __ebuild_phase pkg_pretend
136 > >> "$PORTAGE_BUILDDIR/.pretended" || \
137 > @@ -236,6 +239,7 @@ __dyn_setup() {
138 > __vecho ">>> Remove '$PORTAGE_BUILDDIR/.setuped' to force
139 > setup."
140 > return 0
141 > fi
142 > + ___eapi_vars_export
143 > __ebuild_phase pre_pkg_setup
144 > __ebuild_phase pkg_setup
145 > >> "$PORTAGE_BUILDDIR/.setuped" || \
146 > @@ -252,6 +256,7 @@ __dyn_unpack() {
147 > install -m${PORTAGE_WORKDIR_MODE:-0700} -d "${WORKDIR}" ||
148 > die "Failed to create dir '${WORKDIR}'"
149 > fi
150 > cd "${WORKDIR}" || die "Directory change failed: \`cd
151 > '${WORKDIR}'\`"
152 > + ___eapi_vars_export
153 > __ebuild_phase pre_src_unpack
154 > __vecho ">>> Unpacking source..."
155 > __ebuild_phase src_unpack
156 > @@ -386,6 +391,7 @@ __dyn_prepare() {
157 >
158 > trap __abort_prepare SIGINT SIGQUIT
159 >
160 > + ___eapi_vars_export
161 > __ebuild_phase pre_src_prepare
162 > __vecho ">>> Preparing source in $PWD ..."
163 > __ebuild_phase src_prepare
164 > @@ -423,6 +429,7 @@ __dyn_configure() {
165 >
166 > trap __abort_configure SIGINT SIGQUIT
167 >
168 > + ___eapi_vars_export
169 > __ebuild_phase pre_src_configure
170 >
171 > __vecho ">>> Configuring source in $PWD ..."
172 > @@ -456,6 +463,7 @@ __dyn_compile() {
173 >
174 > trap __abort_compile SIGINT SIGQUIT
175 >
176 > + ___eapi_vars_export
177 > __ebuild_phase pre_src_compile
178 >
179 > __vecho ">>> Compiling source in $PWD ..."
180 > @@ -500,6 +508,7 @@ __dyn_test() {
181 > else
182 > local save_sp=${SANDBOX_PREDICT}
183 > addpredict /
184 > + ___eapi_vars_export
185 > __ebuild_phase pre_src_test
186 >
187 > __vecho ">>> Test phase: ${CATEGORY}/${PF}"
188 > @@ -553,6 +562,7 @@ __dyn_install() {
189 > eval "[[ -n \$QA_PRESTRIPPED_${ARCH/-/_} ]] && \
190 > export QA_PRESTRIPPED_${ARCH/-/_}"
191 >
192 > + ___eapi_vars_export
193 > __ebuild_phase pre_src_install
194 >
195 > if ___eapi_has_prefix_variables; then
196 > @@ -695,6 +705,7 @@ __dyn_install() {
197 > --filter-path --filter-sandbox --allow-extra-vars > \
198 > "${PORTAGE_BUILDDIR}"/build-info/environment
199 > assert "__save_ebuild_env failed"
200 > + ___eapi_vars_unexport
201 > cd "${PORTAGE_BUILDDIR}"/build-info || die
202 >
203 > ${PORTAGE_BZIP2_COMMAND} -f9 environment
204 > @@ -1087,11 +1098,13 @@ __ebuild_main() {
205 > __save_ebuild_env | __filter_readonly_variables \
206 > --filter-features > "$T/environment"
207 > assert "__save_ebuild_env failed"
208 > + ___eapi_vars_unexport
209 > chgrp "${PORTAGE_GRPNAME:-portage}" "$T/environment"
210 > chmod g+w "$T/environment"
211 > fi
212 > [[ -n $PORTAGE_EBUILD_EXIT_FILE ]] && > "$PORTAGE_EBUILD_EXIT_FILE"
213 > if [[ -n $PORTAGE_IPC_DAEMON ]] ; then
214 > + ___eapi_vars_unexport
215 > [[ ! -s $SANDBOX_LOG ]]
216 > "$PORTAGE_BIN_PATH"/ebuild-ipc exit $?
217 > fi
218 > diff --git a/lib/portage/package/ebuild/_config/special_env_vars.py
219 > b/lib/portage/package/ebuild/_config/special_env_vars.py
220 > index 440dd00b2..92824e15f 100644
221 > --- a/lib/portage/package/ebuild/_config/special_env_vars.py
222 > +++ b/lib/portage/package/ebuild/_config/special_env_vars.py
223 > @@ -89,7 +89,7 @@ environ_whitelist += [
224 > ]
225 >
226 > environ_whitelist += [
227 > - "A", "AA", "CATEGORY", "P", "PF", "PN", "PR", "PV", "PVR"
228 > + "CATEGORY", "P", "PF", "PN", "PR", "PV", "PVR"
229 > ]
230 >
231 > # misc variables inherited from the calling environment
232 > @@ -124,6 +124,10 @@ environ_whitelist = frozenset(environ_whitelist)
233 >
234 > environ_whitelist_re = re.compile(r'^(CCACHE_|DISTCC_).*')
235 >
236 > +environ_unexported = frozenset([
237 > + 'A', 'AA',
238 > +])
239 > +
240 > # Filter selected variables in the config.environ() method so that
241 > # they don't needlessly propagate down into the ebuild environment.
242 > environ_filter = []
243 > @@ -131,6 +135,7 @@ environ_filter = []
244 > # Exclude anything that could be extremely long here (like SRC_URI)
245 > # since that could cause execve() calls to fail with E2BIG errors. For
246 > # example, see bug #262647.
247 > +environ_filter.extend(environ_unexported)
248 > environ_filter += [
249 > 'DEPEND', 'RDEPEND', 'PDEPEND', 'SRC_URI',
250 > ]
251 > diff --git a/lib/portage/package/ebuild/config.py
252 > b/lib/portage/package/ebuild/config.py
253 > index 47c180c12..a386dc031 100644
254 > --- a/lib/portage/package/ebuild/config.py
255 > +++ b/lib/portage/package/ebuild/config.py
256 > @@ -10,6 +10,7 @@ __all__ = [
257 > import copy
258 > from itertools import chain
259 > import grp
260 > +import io
261 > import logging
262 > import platform
263 > import pwd
264 > @@ -29,7 +30,7 @@ portage.proxy.lazyimport.lazyimport(globals(),
265 > 'portage.util.locale:check_locale,split_LC_ALL',
266 > )
267 > from portage import bsd_chflags, \
268 > - load_mod, os, selinux, _unicode_decode
269 > + load_mod, os, selinux, _encodings, _unicode_decode
270 > from portage.const import CACHE_PATH, \
271 > DEPCACHE_PATH, INCREMENTALS, MAKE_CONF_FILE, \
272 > MODULES_FILE_PATH, PORTAGE_BASE_PATH, \
273 > @@ -2755,14 +2756,36 @@ class config(object):
274 > eapi_attrs = _get_eapi_attrs(eapi)
275 > phase = self.get('EBUILD_PHASE')
276 > emerge_from = self.get('EMERGE_FROM')
277 > + temp_dir = None
278 > filter_calling_env = False
279 > if self.mycpv is not None and \
280 > - not (emerge_from == 'ebuild' and phase == 'setup')
281 > and \
282 > + 'PORTAGE_BUILDDIR_LOCKED' in self and \
283 > phase not in ('clean', 'cleanrm', 'depend',
284 > 'fetch'):
285 > - temp_dir = self.get('T')
286 > - if temp_dir is not None and \
287 > - os.path.exists(os.path.join(temp_dir,
288 > 'environment')):
289 > - filter_calling_env = True
290 > + temp_dir = self['T']
291 > + # These variables will exported by ebuild.sh if
292 > appropriate
293 > + # for the current EAPI, but export is delayed
294 > since large
295 > + # values may trigger E2BIG errors during attempts
296 > to spawn
297 > + # subprocesses.
298 > + unexported = []
299 > + for key in special_env_vars.environ_unexported:
300 > + # Don't export AA for EAPIs that forbid it.
301 > + if key == 'AA' and not
302 > eapi_exports_AA(eapi):
303 > + continue
304 > + value = self.get(key)
305 > + if value is not None:
306 > + unexported.append((key, value))
307 > +
308 > + # Write this file even if it's empty, so that
309 > ebuild.sh can
310 > + # rely on its existence.
311 > + with io.open(os.path.join(temp_dir,
312 > 'environment.unexported'),
313 > + mode='wt',
314 > encoding=_encodings['repo.content']) as f:
315 > + for key, value in unexported:
316 > + f.write('%s="%s"\n' % (key,
317 > value.replace('"', '\\"')))
318 > +
319 > + if temp_dir is not None and \
320 > + not (emerge_from == 'ebuild' and phase == 'setup')
321 > and \
322 > + os.path.exists(os.path.join(temp_dir,
323 > 'environment')):
324 > + filter_calling_env = True
325 >
326 > environ_whitelist = self._environ_whitelist
327 > for x, myvalue in self.iteritems():
328 > @@ -2805,10 +2828,6 @@ class config(object):
329 > # Filtered by IUSE and implicit IUSE.
330 > mydict["USE"] = self.get("PORTAGE_USE", "")
331 >
332 > - # Don't export AA to the ebuild environment in EAPIs that
333 > forbid it
334 > - if not eapi_exports_AA(eapi):
335 > - mydict.pop("AA", None)
336 > -
337 > if not eapi_exports_merge_type(eapi):
338 > mydict.pop("MERGE_TYPE", None)
339 >
340 > --
341 > 2.25.3
342 >
343 >
344 >

Replies