1 |
This variable specifies a command that filters build log output to a |
2 |
log file. The plan is to extend this to support a separate filter for |
3 |
tty output in the future. |
4 |
|
5 |
In order to enable the EbuildPhase class to write elog messages to |
6 |
the build log with PORTAGE_LOG_FILTER_FILE support, convert its _elog |
7 |
method to a coroutine, and add a SchedulerInterface async_output |
8 |
method for it to use. |
9 |
|
10 |
Bug: https://bugs.gentoo.org/709746 |
11 |
Signed-off-by: Zac Medico <zmedico@g.o> |
12 |
--- |
13 |
[PATCH v2] SpawnProcess and BuildLogger now have _run coroutine |
14 |
|
15 |
lib/_emerge/AbstractEbuildProcess.py | 2 + |
16 |
lib/_emerge/EbuildPhase.py | 21 +++- |
17 |
lib/_emerge/SpawnProcess.py | 28 ++++- |
18 |
.../ebuild/_config/special_env_vars.py | 6 +- |
19 |
lib/portage/util/_async/BuildLogger.py | 109 ++++++++++++++++++ |
20 |
lib/portage/util/_async/SchedulerInterface.py | 30 +++++ |
21 |
man/make.conf.5 | 4 + |
22 |
7 files changed, 189 insertions(+), 11 deletions(-) |
23 |
create mode 100644 lib/portage/util/_async/BuildLogger.py |
24 |
|
25 |
diff --git a/lib/_emerge/AbstractEbuildProcess.py b/lib/_emerge/AbstractEbuildProcess.py |
26 |
index d1a6d1c4e..3732f80ed 100644 |
27 |
--- a/lib/_emerge/AbstractEbuildProcess.py |
28 |
+++ b/lib/_emerge/AbstractEbuildProcess.py |
29 |
@@ -181,6 +181,8 @@ class AbstractEbuildProcess(SpawnProcess): |
30 |
null_fd = os.open('/dev/null', os.O_RDONLY) |
31 |
self.fd_pipes[0] = null_fd |
32 |
|
33 |
+ self.log_filter_file = self.settings.get('PORTAGE_LOG_FILTER_FILE') |
34 |
+ |
35 |
try: |
36 |
yield SpawnProcess._async_start(self) |
37 |
finally: |
38 |
diff --git a/lib/_emerge/EbuildPhase.py b/lib/_emerge/EbuildPhase.py |
39 |
index 63d0746da..41ab79e53 100644 |
40 |
--- a/lib/_emerge/EbuildPhase.py |
41 |
+++ b/lib/_emerge/EbuildPhase.py |
42 |
@@ -26,6 +26,7 @@ from portage.package.ebuild.prepare_build_dirs import (_prepare_workdir, |
43 |
from portage.util.futures.compat_coroutine import coroutine, coroutine_return |
44 |
from portage.util import writemsg |
45 |
from portage.util._async.AsyncTaskFuture import AsyncTaskFuture |
46 |
+from portage.util._async.BuildLogger import BuildLogger |
47 |
from portage.util.futures.executor.fork import ForkExecutor |
48 |
|
49 |
try: |
50 |
@@ -130,7 +131,7 @@ class EbuildPhase(CompositeTask): |
51 |
# Force background=True for this header since it's intended |
52 |
# for the log and it doesn't necessarily need to be visible |
53 |
# elsewhere. |
54 |
- self._elog('einfo', msg, background=True) |
55 |
+ yield self._elog('einfo', msg, background=True) |
56 |
|
57 |
if self.phase == 'package': |
58 |
if 'PORTAGE_BINPKG_TMPFILE' not in self.settings: |
59 |
@@ -392,6 +393,7 @@ class EbuildPhase(CompositeTask): |
60 |
self.returncode = 1 |
61 |
self.wait() |
62 |
|
63 |
+ @coroutine |
64 |
def _elog(self, elog_funcname, lines, background=None): |
65 |
if background is None: |
66 |
background = self.background |
67 |
@@ -408,12 +410,25 @@ class EbuildPhase(CompositeTask): |
68 |
portage.output.havecolor = global_havecolor |
69 |
msg = out.getvalue() |
70 |
if msg: |
71 |
+ build_logger = None |
72 |
+ log_file = None |
73 |
log_path = None |
74 |
if self.settings.get("PORTAGE_BACKGROUND") != "subprocess": |
75 |
log_path = self.settings.get("PORTAGE_LOG_FILE") |
76 |
- self.scheduler.output(msg, log_path=log_path, |
77 |
+ if log_path: |
78 |
+ build_logger = BuildLogger(env=self.settings.environ(), |
79 |
+ log_path=log_path, |
80 |
+ log_filter_file=self.settings.get('PORTAGE_LOG_FILTER_FILE'), |
81 |
+ scheduler=self.scheduler) |
82 |
+ yield build_logger.async_start() |
83 |
+ log_file = build_logger.stdin |
84 |
+ |
85 |
+ yield self.scheduler.async_output(msg, log_file=log_file, |
86 |
background=background) |
87 |
|
88 |
+ if build_logger is not None: |
89 |
+ build_logger.stdin.close() |
90 |
+ yield build_logger.async_wait() |
91 |
|
92 |
class _PostPhaseCommands(CompositeTask): |
93 |
|
94 |
@@ -481,4 +496,4 @@ class _PostPhaseCommands(CompositeTask): |
95 |
qa_msg.extend("\t%s: %s" % (filename, " ".join(sorted(soname_deps))) |
96 |
for filename, soname_deps in unresolved) |
97 |
qa_msg.append("") |
98 |
- self.elog("eqawarn", qa_msg) |
99 |
+ yield self.elog("eqawarn", qa_msg) |
100 |
diff --git a/lib/_emerge/SpawnProcess.py b/lib/_emerge/SpawnProcess.py |
101 |
index cda615ded..e06e2b934 100644 |
102 |
--- a/lib/_emerge/SpawnProcess.py |
103 |
+++ b/lib/_emerge/SpawnProcess.py |
104 |
@@ -19,6 +19,7 @@ from portage.const import BASH_BINARY |
105 |
from portage.localization import _ |
106 |
from portage.output import EOutput |
107 |
from portage.util import writemsg_level |
108 |
+from portage.util._async.BuildLogger import BuildLogger |
109 |
from portage.util._async.PipeLogger import PipeLogger |
110 |
from portage.util.futures import asyncio |
111 |
from portage.util.futures.compat_coroutine import coroutine |
112 |
@@ -36,7 +37,7 @@ class SpawnProcess(SubProcess): |
113 |
"path_lookup", "pre_exec", "close_fds", "cgroup", |
114 |
"unshare_ipc", "unshare_mount", "unshare_pid", "unshare_net") |
115 |
|
116 |
- __slots__ = ("args",) + \ |
117 |
+ __slots__ = ("args", "log_filter_file") + \ |
118 |
_spawn_kwarg_names + ("_main_task", "_selinux_type",) |
119 |
|
120 |
# Max number of attempts to kill the processes listed in cgroup.procs, |
121 |
@@ -142,21 +143,36 @@ class SpawnProcess(SubProcess): |
122 |
fcntl.fcntl(stdout_fd, |
123 |
fcntl.F_GETFD) | fcntl.FD_CLOEXEC) |
124 |
|
125 |
+ build_logger = BuildLogger(env=self.env, |
126 |
+ log_path=log_file_path, |
127 |
+ log_filter_file=self.log_filter_file, |
128 |
+ scheduler=self.scheduler) |
129 |
+ |
130 |
+ yield build_logger.async_start() |
131 |
+ |
132 |
pipe_logger = PipeLogger(background=self.background, |
133 |
scheduler=self.scheduler, input_fd=master_fd, |
134 |
- log_file_path=log_file_path, |
135 |
+ log_file_path=build_logger.stdin, |
136 |
stdout_fd=stdout_fd) |
137 |
self._registered = True |
138 |
yield pipe_logger.async_start() |
139 |
|
140 |
self._main_task = asyncio.ensure_future( |
141 |
- self._main(pipe_logger), loop=self.scheduler) |
142 |
+ self._main(pipe_logger, build_logger=build_logger), loop=self.scheduler) |
143 |
self._main_task.add_done_callback(self._main_exit) |
144 |
|
145 |
@coroutine |
146 |
- def _main(self, pipe_logger): |
147 |
- if pipe_logger.poll() is None: |
148 |
- yield pipe_logger.async_wait() |
149 |
+ def _main(self, pipe_logger, build_logger=None): |
150 |
+ try: |
151 |
+ if pipe_logger.poll() is None: |
152 |
+ yield pipe_logger.async_wait() |
153 |
+ except Exception: |
154 |
+ if build_logger is not None and build_logger.poll() is None: |
155 |
+ build_logger.cancel() |
156 |
+ raise |
157 |
+ |
158 |
+ if build_logger is not None and build_logger.poll() is None: |
159 |
+ yield build_logger.async_wait() |
160 |
|
161 |
def _main_exit(self, main_task): |
162 |
try: |
163 |
diff --git a/lib/portage/package/ebuild/_config/special_env_vars.py b/lib/portage/package/ebuild/_config/special_env_vars.py |
164 |
index dc01339f7..977245309 100644 |
165 |
--- a/lib/portage/package/ebuild/_config/special_env_vars.py |
166 |
+++ b/lib/portage/package/ebuild/_config/special_env_vars.py |
167 |
@@ -175,7 +175,7 @@ environ_filter += [ |
168 |
"PORTAGE_RO_DISTDIRS", |
169 |
"PORTAGE_RSYNC_EXTRA_OPTS", "PORTAGE_RSYNC_OPTS", |
170 |
"PORTAGE_RSYNC_RETRIES", "PORTAGE_SSH_OPTS", "PORTAGE_SYNC_STALE", |
171 |
- "PORTAGE_USE", |
172 |
+ "PORTAGE_USE", "PORTAGE_LOG_FILTER_FILE", |
173 |
"PORTAGE_LOGDIR", "PORTAGE_LOGDIR_CLEAN", |
174 |
"QUICKPKG_DEFAULT_OPTS", "REPOMAN_DEFAULT_OPTS", |
175 |
"RESUMECOMMAND", "RESUMECOMMAND_FTP", |
176 |
@@ -204,7 +204,9 @@ default_globals = { |
177 |
'PORTAGE_BZIP2_COMMAND': 'bzip2', |
178 |
} |
179 |
|
180 |
-validate_commands = ('PORTAGE_BZIP2_COMMAND', 'PORTAGE_BUNZIP2_COMMAND',) |
181 |
+validate_commands = ('PORTAGE_BZIP2_COMMAND', 'PORTAGE_BUNZIP2_COMMAND', |
182 |
+ 'PORTAGE_LOG_FILTER_FILE', |
183 |
+) |
184 |
|
185 |
# To enhance usability, make some vars case insensitive |
186 |
# by forcing them to lower case. |
187 |
diff --git a/lib/portage/util/_async/BuildLogger.py b/lib/portage/util/_async/BuildLogger.py |
188 |
new file mode 100644 |
189 |
index 000000000..cb1f7d3f9 |
190 |
--- /dev/null |
191 |
+++ b/lib/portage/util/_async/BuildLogger.py |
192 |
@@ -0,0 +1,109 @@ |
193 |
+# Copyright 2020 Gentoo Authors |
194 |
+# Distributed under the terms of the GNU General Public License v2 |
195 |
+ |
196 |
+from portage import os |
197 |
+from portage.util import shlex_split |
198 |
+from _emerge.AsynchronousTask import AsynchronousTask |
199 |
+from portage.util._async.PipeLogger import PipeLogger |
200 |
+from portage.util.futures import asyncio |
201 |
+from portage.util.futures.compat_coroutine import coroutine |
202 |
+ |
203 |
+ |
204 |
+class BuildLogger(AsynchronousTask): |
205 |
+ """ |
206 |
+ Write to a log file, with compression support provided by PipeLogger. |
207 |
+ If the log_filter_file parameter is specified, then it is interpreted |
208 |
+ as a command to execute which filters log output (see the |
209 |
+ PORTAGE_LOG_FILTER_FILE variable in make.conf(5)). The stdin property |
210 |
+ provides access to a binary file object (refers to a pipe) that log |
211 |
+ content should be written to (usually redirected from subprocess |
212 |
+ stdout and stderr streams). |
213 |
+ """ |
214 |
+ |
215 |
+ __slots__ = ('env', 'log_path', 'log_filter_file', '_main_task', '_stdin') |
216 |
+ |
217 |
+ @property |
218 |
+ def stdin(self): |
219 |
+ return self._stdin |
220 |
+ |
221 |
+ def _start(self): |
222 |
+ self.scheduler.run_until_complete(self._async_start()) |
223 |
+ |
224 |
+ @coroutine |
225 |
+ def _async_start(self): |
226 |
+ if self.log_path is not None: |
227 |
+ log_filter_file = self.log_filter_file |
228 |
+ if log_filter_file is not None: |
229 |
+ split_value = shlex_split(log_filter_file) |
230 |
+ log_filter_file = split_value if split_value else None |
231 |
+ if log_filter_file: |
232 |
+ filter_input, stdin = os.pipe() |
233 |
+ log_input, filter_output = os.pipe() |
234 |
+ try: |
235 |
+ filter_proc = yield asyncio.create_subprocess_exec( |
236 |
+ *log_filter_file, |
237 |
+ env=self.env, |
238 |
+ stdin=filter_input, |
239 |
+ stdout=filter_output, |
240 |
+ stderr=filter_output, |
241 |
+ loop=self.scheduler) |
242 |
+ except EnvironmentError: |
243 |
+ # Maybe the command is missing or broken somehow... |
244 |
+ os.close(filter_input) |
245 |
+ os.close(stdin) |
246 |
+ os.close(log_input) |
247 |
+ os.close(filter_output) |
248 |
+ else: |
249 |
+ self._stdin = os.fdopen(stdin, 'wb', 0) |
250 |
+ os.close(filter_input) |
251 |
+ os.close(filter_output) |
252 |
+ # Create a PipeLogger instance to read output from |
253 |
+ # filter_proc and log it. Set background=True |
254 |
+ # so that this instance does not log to stdout. |
255 |
+ pipe_logger = PipeLogger(background=True, |
256 |
+ scheduler=self.scheduler, input_fd=log_input, |
257 |
+ log_file_path=self.log_path) |
258 |
+ yield pipe_logger.async_start() |
259 |
+ self._main_task = asyncio.ensure_future( |
260 |
+ self._main(pipe_logger, filter_proc=filter_proc), loop=self.scheduler) |
261 |
+ self._main_task.add_done_callback(self._main_exit) |
262 |
+ |
263 |
+ if self._stdin is None: |
264 |
+ # Since log_filter_file is unspecified or refers to a file |
265 |
+ # that is missing or broken somehow, create a pipe that |
266 |
+ # logs directly to a PipeLogger instance. |
267 |
+ log_input, stdin = os.pipe() |
268 |
+ self._stdin = os.fdopen(stdin, 'wb', 0) |
269 |
+ pipe_logger = PipeLogger(background=True, |
270 |
+ scheduler=self.scheduler, input_fd=log_input, |
271 |
+ log_file_path=self.log_path) |
272 |
+ yield pipe_logger.async_start() |
273 |
+ self._main_task = asyncio.ensure_future( |
274 |
+ self._main(pipe_logger), loop=self.scheduler) |
275 |
+ self._main_task.add_done_callback(self._main_exit) |
276 |
+ |
277 |
+ def _cancel(self): |
278 |
+ if self._main_task is not None: |
279 |
+ self._main_task.cancel() |
280 |
+ |
281 |
+ @coroutine |
282 |
+ def _main(self, pipe_logger, filter_proc=None): |
283 |
+ try: |
284 |
+ if pipe_logger.poll() is None: |
285 |
+ yield pipe_logger.async_wait() |
286 |
+ except Exception: |
287 |
+ if filter_proc is not None and filter_proc.returncode is None: |
288 |
+ filter_proc.terminate() |
289 |
+ raise |
290 |
+ |
291 |
+ if filter_proc is not None and filter_proc.returncode is None: |
292 |
+ yield filter_proc.wait() |
293 |
+ |
294 |
+ def _main_exit(self, main_task): |
295 |
+ try: |
296 |
+ main_task.result() |
297 |
+ except asyncio.CancelledError: |
298 |
+ self.cancel() |
299 |
+ self._was_cancelled() |
300 |
+ self.returncode = self.returncode or 0 |
301 |
+ self._async_wait() |
302 |
diff --git a/lib/portage/util/_async/SchedulerInterface.py b/lib/portage/util/_async/SchedulerInterface.py |
303 |
index ec6417da1..1a9599d6f 100644 |
304 |
--- a/lib/portage/util/_async/SchedulerInterface.py |
305 |
+++ b/lib/portage/util/_async/SchedulerInterface.py |
306 |
@@ -7,6 +7,8 @@ import errno |
307 |
from portage import _encodings |
308 |
from portage import _unicode_encode |
309 |
from portage.util import writemsg_level |
310 |
+from portage.util.futures._asyncio.streams import _writer |
311 |
+from portage.util.futures.compat_coroutine import coroutine |
312 |
from ..SlotObject import SlotObject |
313 |
|
314 |
class SchedulerInterface(SlotObject): |
315 |
@@ -53,6 +55,34 @@ class SchedulerInterface(SlotObject): |
316 |
def _return_false(): |
317 |
return False |
318 |
|
319 |
+ @coroutine |
320 |
+ def async_output(self, msg, log_file=None, background=None, |
321 |
+ level=0, noiselevel=-1): |
322 |
+ """ |
323 |
+ Output a msg to stdio (if not in background) and to a log file |
324 |
+ if provided. |
325 |
+ |
326 |
+ @param msg: a message string, including newline if appropriate |
327 |
+ @type msg: str |
328 |
+ @param log_file: log file in binary mode |
329 |
+ @type log_file: file |
330 |
+ @param background: send messages only to log (not to stdio) |
331 |
+ @type background: bool |
332 |
+ @param level: a numeric logging level (see the logging module) |
333 |
+ @type level: int |
334 |
+ @param noiselevel: passed directly to writemsg |
335 |
+ @type noiselevel: int |
336 |
+ """ |
337 |
+ global_background = self._is_background() |
338 |
+ if background is None or global_background: |
339 |
+ background = global_background |
340 |
+ |
341 |
+ if not background: |
342 |
+ writemsg_level(msg, level=level, noiselevel=noiselevel) |
343 |
+ |
344 |
+ if log_file is not None: |
345 |
+ yield _writer(log_file, _unicode_encode(msg)) |
346 |
+ |
347 |
def output(self, msg, log_path=None, background=None, |
348 |
level=0, noiselevel=-1): |
349 |
""" |
350 |
diff --git a/man/make.conf.5 b/man/make.conf.5 |
351 |
index f82fed65a..69bfa6cdf 100644 |
352 |
--- a/man/make.conf.5 |
353 |
+++ b/man/make.conf.5 |
354 |
@@ -979,6 +979,10 @@ with an integer pid. For example, a value of "ionice \-c 3 \-p \\${PID}" |
355 |
will set idle io priority. For more information about ionice, see |
356 |
\fBionice\fR(1). This variable is unset by default. |
357 |
.TP |
358 |
+.B PORTAGE_LOG_FILTER_FILE |
359 |
+This variable specifies a command that filters build log output to a |
360 |
+log file. |
361 |
+.TP |
362 |
.B PORTAGE_LOGDIR |
363 |
This variable defines the directory in which per\-ebuild logs are kept. |
364 |
Logs are created only when this is set. They are stored as |
365 |
-- |
366 |
2.24.1 |