1 |
commit: 73f72f526a66b9953a46868cc1390fde2820997f |
2 |
Author: Zac Medico <zmedico <AT> gentoo <DOT> org> |
3 |
AuthorDate: Sun Mar 1 02:17:52 2020 +0000 |
4 |
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org> |
5 |
CommitDate: Sun Mar 1 04:29:41 2020 +0000 |
6 |
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=73f72f52 |
7 |
|
8 |
Support PORTAGE_LOG_FILTER_FILE (bug 709746) |
9 |
|
10 |
This variable specifies a command that filters build log output to a |
11 |
log file. The plan is to extend this to support a separate filter for |
12 |
tty output in the future. |
13 |
|
14 |
In order to enable the EbuildPhase class to write elog messages to |
15 |
the build log with PORTAGE_LOG_FILTER_FILE support, convert its _elog |
16 |
method to a coroutine, and add a SchedulerInterface async_output |
17 |
method for it to use. |
18 |
|
19 |
Bug: https://bugs.gentoo.org/709746 |
20 |
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org> |
21 |
|
22 |
lib/_emerge/AbstractEbuildProcess.py | 2 + |
23 |
lib/_emerge/EbuildPhase.py | 36 +++++-- |
24 |
lib/_emerge/SpawnProcess.py | 32 ++++-- |
25 |
.../package/ebuild/_config/special_env_vars.py | 8 +- |
26 |
lib/portage/util/_async/BuildLogger.py | 116 +++++++++++++++++++++ |
27 |
lib/portage/util/_async/SchedulerInterface.py | 32 +++++- |
28 |
man/make.conf.5 | 7 +- |
29 |
7 files changed, 213 insertions(+), 20 deletions(-) |
30 |
|
31 |
diff --git a/lib/_emerge/AbstractEbuildProcess.py b/lib/_emerge/AbstractEbuildProcess.py |
32 |
index d1a6d1c4e..3732f80ed 100644 |
33 |
--- a/lib/_emerge/AbstractEbuildProcess.py |
34 |
+++ b/lib/_emerge/AbstractEbuildProcess.py |
35 |
@@ -181,6 +181,8 @@ class AbstractEbuildProcess(SpawnProcess): |
36 |
null_fd = os.open('/dev/null', os.O_RDONLY) |
37 |
self.fd_pipes[0] = null_fd |
38 |
|
39 |
+ self.log_filter_file = self.settings.get('PORTAGE_LOG_FILTER_FILE') |
40 |
+ |
41 |
try: |
42 |
yield SpawnProcess._async_start(self) |
43 |
finally: |
44 |
|
45 |
diff --git a/lib/_emerge/EbuildPhase.py b/lib/_emerge/EbuildPhase.py |
46 |
index f6b380e05..927a74b98 100644 |
47 |
--- a/lib/_emerge/EbuildPhase.py |
48 |
+++ b/lib/_emerge/EbuildPhase.py |
49 |
@@ -26,6 +26,8 @@ from portage.package.ebuild.prepare_build_dirs import (_prepare_workdir, |
50 |
from portage.util.futures.compat_coroutine import coroutine, coroutine_return |
51 |
from portage.util import writemsg |
52 |
from portage.util._async.AsyncTaskFuture import AsyncTaskFuture |
53 |
+from portage.util._async.BuildLogger import BuildLogger |
54 |
+from portage.util.futures import asyncio |
55 |
from portage.util.futures.executor.fork import ForkExecutor |
56 |
|
57 |
try: |
58 |
@@ -130,7 +132,7 @@ class EbuildPhase(CompositeTask): |
59 |
# Force background=True for this header since it's intended |
60 |
# for the log and it doesn't necessarily need to be visible |
61 |
# elsewhere. |
62 |
- self._elog('einfo', msg, background=True) |
63 |
+ yield self._elog('einfo', msg, background=True) |
64 |
|
65 |
if self.phase == 'package': |
66 |
if 'PORTAGE_BINPKG_TMPFILE' not in self.settings: |
67 |
@@ -392,6 +394,7 @@ class EbuildPhase(CompositeTask): |
68 |
self.returncode = 1 |
69 |
self.wait() |
70 |
|
71 |
+ @coroutine |
72 |
def _elog(self, elog_funcname, lines, background=None): |
73 |
if background is None: |
74 |
background = self.background |
75 |
@@ -408,11 +411,30 @@ class EbuildPhase(CompositeTask): |
76 |
portage.output.havecolor = global_havecolor |
77 |
msg = out.getvalue() |
78 |
if msg: |
79 |
- log_path = None |
80 |
- if self.settings.get("PORTAGE_BACKGROUND") != "subprocess": |
81 |
- log_path = self.settings.get("PORTAGE_LOG_FILE") |
82 |
- self.scheduler.output(msg, log_path=log_path, |
83 |
- background=background) |
84 |
+ build_logger = None |
85 |
+ try: |
86 |
+ log_file = None |
87 |
+ log_path = None |
88 |
+ if self.settings.get("PORTAGE_BACKGROUND") != "subprocess": |
89 |
+ log_path = self.settings.get("PORTAGE_LOG_FILE") |
90 |
+ if log_path: |
91 |
+ build_logger = BuildLogger(env=self.settings.environ(), |
92 |
+ log_path=log_path, |
93 |
+ log_filter_file=self.settings.get('PORTAGE_LOG_FILTER_FILE'), |
94 |
+ scheduler=self.scheduler) |
95 |
+ yield build_logger.async_start() |
96 |
+ log_file = build_logger.stdin |
97 |
+ |
98 |
+ yield self.scheduler.async_output(msg, log_file=log_file, |
99 |
+ background=background) |
100 |
+ |
101 |
+ if build_logger is not None: |
102 |
+ build_logger.stdin.close() |
103 |
+ yield build_logger.async_wait() |
104 |
+ except asyncio.CancelledError: |
105 |
+ if build_logger is not None: |
106 |
+ build_logger.cancel() |
107 |
+ raise |
108 |
|
109 |
|
110 |
class _PostPhaseCommands(CompositeTask): |
111 |
@@ -481,4 +503,4 @@ class _PostPhaseCommands(CompositeTask): |
112 |
qa_msg.extend("\t%s: %s" % (filename, " ".join(sorted(soname_deps))) |
113 |
for filename, soname_deps in unresolved) |
114 |
qa_msg.append("") |
115 |
- self.elog("eqawarn", qa_msg) |
116 |
+ yield self.elog("eqawarn", qa_msg) |
117 |
|
118 |
diff --git a/lib/_emerge/SpawnProcess.py b/lib/_emerge/SpawnProcess.py |
119 |
index ab7971ca8..34668b287 100644 |
120 |
--- a/lib/_emerge/SpawnProcess.py |
121 |
+++ b/lib/_emerge/SpawnProcess.py |
122 |
@@ -19,6 +19,7 @@ from portage.const import BASH_BINARY |
123 |
from portage.localization import _ |
124 |
from portage.output import EOutput |
125 |
from portage.util import writemsg_level |
126 |
+from portage.util._async.BuildLogger import BuildLogger |
127 |
from portage.util._async.PipeLogger import PipeLogger |
128 |
from portage.util.futures import asyncio |
129 |
from portage.util.futures.compat_coroutine import coroutine |
130 |
@@ -36,7 +37,7 @@ class SpawnProcess(SubProcess): |
131 |
"path_lookup", "pre_exec", "close_fds", "cgroup", |
132 |
"unshare_ipc", "unshare_mount", "unshare_pid", "unshare_net") |
133 |
|
134 |
- __slots__ = ("args",) + \ |
135 |
+ __slots__ = ("args", "log_filter_file") + \ |
136 |
_spawn_kwarg_names + ("_main_task", "_selinux_type",) |
137 |
|
138 |
# Max number of attempts to kill the processes listed in cgroup.procs, |
139 |
@@ -142,30 +143,45 @@ class SpawnProcess(SubProcess): |
140 |
fcntl.fcntl(stdout_fd, |
141 |
fcntl.F_GETFD) | fcntl.FD_CLOEXEC) |
142 |
|
143 |
- pipe_logger = PipeLogger(background=self.background, |
144 |
- scheduler=self.scheduler, input_fd=master_fd, |
145 |
- log_file_path=log_file_path, |
146 |
- stdout_fd=stdout_fd) |
147 |
+ build_logger = BuildLogger(env=self.env, |
148 |
+ log_path=log_file_path, |
149 |
+ log_filter_file=self.log_filter_file, |
150 |
+ scheduler=self.scheduler) |
151 |
+ |
152 |
self._registered = True |
153 |
+ pipe_logger = None |
154 |
try: |
155 |
+ yield build_logger.async_start() |
156 |
+ |
157 |
+ pipe_logger = PipeLogger(background=self.background, |
158 |
+ scheduler=self.scheduler, input_fd=master_fd, |
159 |
+ log_file_path=build_logger.stdin, |
160 |
+ stdout_fd=stdout_fd) |
161 |
+ |
162 |
yield pipe_logger.async_start() |
163 |
except asyncio.CancelledError: |
164 |
- if pipe_logger.poll() is None: |
165 |
+ if pipe_logger is not None and pipe_logger.poll() is None: |
166 |
pipe_logger.cancel() |
167 |
+ if build_logger.poll() is None: |
168 |
+ build_logger.cancel() |
169 |
raise |
170 |
|
171 |
self._main_task = asyncio.ensure_future( |
172 |
- self._main(pipe_logger), loop=self.scheduler) |
173 |
+ self._main(pipe_logger, build_logger), loop=self.scheduler) |
174 |
self._main_task.add_done_callback(self._main_exit) |
175 |
|
176 |
@coroutine |
177 |
- def _main(self, pipe_logger): |
178 |
+ def _main(self, pipe_logger, build_logger): |
179 |
try: |
180 |
if pipe_logger.poll() is None: |
181 |
yield pipe_logger.async_wait() |
182 |
+ if build_logger.poll() is None: |
183 |
+ yield build_logger.async_wait() |
184 |
except asyncio.CancelledError: |
185 |
if pipe_logger.poll() is None: |
186 |
pipe_logger.cancel() |
187 |
+ if build_logger.poll() is None: |
188 |
+ build_logger.cancel() |
189 |
raise |
190 |
|
191 |
def _main_exit(self, main_task): |
192 |
|
193 |
diff --git a/lib/portage/package/ebuild/_config/special_env_vars.py b/lib/portage/package/ebuild/_config/special_env_vars.py |
194 |
index dc01339f7..dd8105123 100644 |
195 |
--- a/lib/portage/package/ebuild/_config/special_env_vars.py |
196 |
+++ b/lib/portage/package/ebuild/_config/special_env_vars.py |
197 |
@@ -1,4 +1,4 @@ |
198 |
-# Copyright 2010-2019 Gentoo Authors |
199 |
+# Copyright 2010-2020 Gentoo Authors |
200 |
# Distributed under the terms of the GNU General Public License v2 |
201 |
|
202 |
from __future__ import unicode_literals |
203 |
@@ -175,7 +175,7 @@ environ_filter += [ |
204 |
"PORTAGE_RO_DISTDIRS", |
205 |
"PORTAGE_RSYNC_EXTRA_OPTS", "PORTAGE_RSYNC_OPTS", |
206 |
"PORTAGE_RSYNC_RETRIES", "PORTAGE_SSH_OPTS", "PORTAGE_SYNC_STALE", |
207 |
- "PORTAGE_USE", |
208 |
+ "PORTAGE_USE", "PORTAGE_LOG_FILTER_FILE", |
209 |
"PORTAGE_LOGDIR", "PORTAGE_LOGDIR_CLEAN", |
210 |
"QUICKPKG_DEFAULT_OPTS", "REPOMAN_DEFAULT_OPTS", |
211 |
"RESUMECOMMAND", "RESUMECOMMAND_FTP", |
212 |
@@ -204,7 +204,9 @@ default_globals = { |
213 |
'PORTAGE_BZIP2_COMMAND': 'bzip2', |
214 |
} |
215 |
|
216 |
-validate_commands = ('PORTAGE_BZIP2_COMMAND', 'PORTAGE_BUNZIP2_COMMAND',) |
217 |
+validate_commands = ('PORTAGE_BZIP2_COMMAND', 'PORTAGE_BUNZIP2_COMMAND', |
218 |
+ 'PORTAGE_LOG_FILTER_FILE', |
219 |
+) |
220 |
|
221 |
# To enhance usability, make some vars case insensitive |
222 |
# by forcing them to lower case. |
223 |
|
224 |
diff --git a/lib/portage/util/_async/BuildLogger.py b/lib/portage/util/_async/BuildLogger.py |
225 |
new file mode 100644 |
226 |
index 000000000..4873d9750 |
227 |
--- /dev/null |
228 |
+++ b/lib/portage/util/_async/BuildLogger.py |
229 |
@@ -0,0 +1,116 @@ |
230 |
+# Copyright 2020 Gentoo Authors |
231 |
+# Distributed under the terms of the GNU General Public License v2 |
232 |
+ |
233 |
+from portage import os |
234 |
+from portage.util import shlex_split |
235 |
+from _emerge.AsynchronousTask import AsynchronousTask |
236 |
+from portage.util._async.PipeLogger import PipeLogger |
237 |
+from portage.util.futures import asyncio |
238 |
+from portage.util.futures.compat_coroutine import coroutine |
239 |
+ |
240 |
+ |
241 |
+class BuildLogger(AsynchronousTask): |
242 |
+ """ |
243 |
+ Write to a log file, with compression support provided by PipeLogger. |
244 |
+ If the log_filter_file parameter is specified, then it is interpreted |
245 |
+ as a command to execute which filters log output (see the |
246 |
+ PORTAGE_LOG_FILTER_FILE variable in make.conf(5)). The stdin property |
247 |
+ provides access to a writable binary file stream (refers to a pipe) |
248 |
+ that log content should be written to (usually redirected from |
249 |
+ subprocess stdout and stderr streams). |
250 |
+ """ |
251 |
+ |
252 |
+ __slots__ = ('env', 'log_path', 'log_filter_file', '_main_task', '_stdin') |
253 |
+ |
254 |
+ @property |
255 |
+ def stdin(self): |
256 |
+ return self._stdin |
257 |
+ |
258 |
+ def _start(self): |
259 |
+ self.scheduler.run_until_complete(self._async_start()) |
260 |
+ |
261 |
+ @coroutine |
262 |
+ def _async_start(self): |
263 |
+ pipe_logger = None |
264 |
+ filter_proc = None |
265 |
+ try: |
266 |
+ log_input = None |
267 |
+ if self.log_path is not None: |
268 |
+ log_filter_file = self.log_filter_file |
269 |
+ if log_filter_file is not None: |
270 |
+ split_value = shlex_split(log_filter_file) |
271 |
+ log_filter_file = split_value if split_value else None |
272 |
+ if log_filter_file: |
273 |
+ filter_input, stdin = os.pipe() |
274 |
+ log_input, filter_output = os.pipe() |
275 |
+ try: |
276 |
+ filter_proc = yield asyncio.create_subprocess_exec( |
277 |
+ *log_filter_file, |
278 |
+ env=self.env, |
279 |
+ stdin=filter_input, |
280 |
+ stdout=filter_output, |
281 |
+ stderr=filter_output, |
282 |
+ loop=self.scheduler) |
283 |
+ except EnvironmentError: |
284 |
+ # Maybe the command is missing or broken somehow... |
285 |
+ os.close(filter_input) |
286 |
+ os.close(stdin) |
287 |
+ os.close(log_input) |
288 |
+ os.close(filter_output) |
289 |
+ else: |
290 |
+ self._stdin = os.fdopen(stdin, 'wb', 0) |
291 |
+ os.close(filter_input) |
292 |
+ os.close(filter_output) |
293 |
+ |
294 |
+ if self._stdin is None: |
295 |
+ # Since log_filter_file is unspecified or refers to a file |
296 |
+ # that is missing or broken somehow, create a pipe that |
297 |
+ # logs directly to pipe_logger. |
298 |
+ log_input, stdin = os.pipe() |
299 |
+ self._stdin = os.fdopen(stdin, 'wb', 0) |
300 |
+ |
301 |
+ # Set background=True so that pipe_logger does not log to stdout. |
302 |
+ pipe_logger = PipeLogger(background=True, |
303 |
+ scheduler=self.scheduler, input_fd=log_input, |
304 |
+ log_file_path=self.log_path) |
305 |
+ |
306 |
+ yield pipe_logger.async_start() |
307 |
+ except asyncio.CancelledError: |
308 |
+ if pipe_logger is not None and pipe_logger.poll() is None: |
309 |
+ pipe_logger.cancel() |
310 |
+ if filter_proc is not None and filter_proc.returncode is None: |
311 |
+ filter_proc.terminate() |
312 |
+ raise |
313 |
+ |
314 |
+ self._main_task = asyncio.ensure_future( |
315 |
+ self._main(pipe_logger, filter_proc=filter_proc), loop=self.scheduler) |
316 |
+ self._main_task.add_done_callback(self._main_exit) |
317 |
+ |
318 |
+ def _cancel(self): |
319 |
+ if self._main_task is not None: |
320 |
+ self._main_task.done() or self._main_task.cancel() |
321 |
+ if self._stdin is not None and not self._stdin.closed: |
322 |
+ self._stdin.close() |
323 |
+ |
324 |
+ @coroutine |
325 |
+ def _main(self, pipe_logger, filter_proc=None): |
326 |
+ try: |
327 |
+ if pipe_logger.poll() is None: |
328 |
+ yield pipe_logger.async_wait() |
329 |
+ if filter_proc is not None and filter_proc.returncode is None: |
330 |
+ yield filter_proc.wait() |
331 |
+ except asyncio.CancelledError: |
332 |
+ if pipe_logger.poll() is None: |
333 |
+ pipe_logger.cancel() |
334 |
+ if filter_proc is not None and filter_proc.returncode is None: |
335 |
+ filter_proc.terminate() |
336 |
+ raise |
337 |
+ |
338 |
+ def _main_exit(self, main_task): |
339 |
+ try: |
340 |
+ main_task.result() |
341 |
+ except asyncio.CancelledError: |
342 |
+ self.cancel() |
343 |
+ self._was_cancelled() |
344 |
+ self.returncode = self.returncode or 0 |
345 |
+ self._async_wait() |
346 |
|
347 |
diff --git a/lib/portage/util/_async/SchedulerInterface.py b/lib/portage/util/_async/SchedulerInterface.py |
348 |
index ec6417da1..3ff250d1d 100644 |
349 |
--- a/lib/portage/util/_async/SchedulerInterface.py |
350 |
+++ b/lib/portage/util/_async/SchedulerInterface.py |
351 |
@@ -1,4 +1,4 @@ |
352 |
-# Copyright 2012-2018 Gentoo Foundation |
353 |
+# Copyright 2012-2020 Gentoo Authors |
354 |
# Distributed under the terms of the GNU General Public License v2 |
355 |
|
356 |
import gzip |
357 |
@@ -7,6 +7,8 @@ import errno |
358 |
from portage import _encodings |
359 |
from portage import _unicode_encode |
360 |
from portage.util import writemsg_level |
361 |
+from portage.util.futures._asyncio.streams import _writer |
362 |
+from portage.util.futures.compat_coroutine import coroutine |
363 |
from ..SlotObject import SlotObject |
364 |
|
365 |
class SchedulerInterface(SlotObject): |
366 |
@@ -53,6 +55,34 @@ class SchedulerInterface(SlotObject): |
367 |
def _return_false(): |
368 |
return False |
369 |
|
370 |
+ @coroutine |
371 |
+ def async_output(self, msg, log_file=None, background=None, |
372 |
+ level=0, noiselevel=-1): |
373 |
+ """ |
374 |
+ Output a msg to stdio (if not in background) and to a log file |
375 |
+ if provided. |
376 |
+ |
377 |
+ @param msg: a message string, including newline if appropriate |
378 |
+ @type msg: str |
379 |
+ @param log_file: log file in binary mode |
380 |
+ @type log_file: file |
381 |
+ @param background: send messages only to log (not to stdio) |
382 |
+ @type background: bool |
383 |
+ @param level: a numeric logging level (see the logging module) |
384 |
+ @type level: int |
385 |
+ @param noiselevel: passed directly to writemsg |
386 |
+ @type noiselevel: int |
387 |
+ """ |
388 |
+ global_background = self._is_background() |
389 |
+ if background is None or global_background: |
390 |
+ background = global_background |
391 |
+ |
392 |
+ if not background: |
393 |
+ writemsg_level(msg, level=level, noiselevel=noiselevel) |
394 |
+ |
395 |
+ if log_file is not None: |
396 |
+ yield _writer(log_file, _unicode_encode(msg)) |
397 |
+ |
398 |
def output(self, msg, log_path=None, background=None, |
399 |
level=0, noiselevel=-1): |
400 |
""" |
401 |
|
402 |
diff --git a/man/make.conf.5 b/man/make.conf.5 |
403 |
index f82fed65a..baecd283a 100644 |
404 |
--- a/man/make.conf.5 |
405 |
+++ b/man/make.conf.5 |
406 |
@@ -1,4 +1,4 @@ |
407 |
-.TH "MAKE.CONF" "5" "Nov 2019" "Portage VERSION" "Portage" |
408 |
+.TH "MAKE.CONF" "5" "Mar 2020" "Portage VERSION" "Portage" |
409 |
.SH "NAME" |
410 |
make.conf \- custom settings for Portage |
411 |
.SH "SYNOPSIS" |
412 |
@@ -979,6 +979,11 @@ with an integer pid. For example, a value of "ionice \-c 3 \-p \\${PID}" |
413 |
will set idle io priority. For more information about ionice, see |
414 |
\fBionice\fR(1). This variable is unset by default. |
415 |
.TP |
416 |
+.B PORTAGE_LOG_FILTER_FILE |
417 |
+This variable specifies a command that filters build log output to a |
418 |
+log file. In order to filter ANSI escape codes from build logs, |
419 |
+\fBansifilter\fR(1) is a convenient setting for this variable. |
420 |
+.TP |
421 |
.B PORTAGE_LOGDIR |
422 |
This variable defines the directory in which per\-ebuild logs are kept. |
423 |
Logs are created only when this is set. They are stored as |