[gentoo-portage-dev] [PATCH] Support PORTAGE_LOG_FILTER_COMMAND (bug 709746)

[Zac Medico]

This variable specifies a command that filters stdout and stderr of
build logs.

Bug: https://bugs.gentoo.org/709746
Signed-off-by: Zac Medico <zmedico@g.o>
---
 lib/_emerge/AbstractEbuildProcess.py          |  8 +++-
 lib/_emerge/SpawnProcess.py                   | 43 ++++++++++++++++++-
 .../ebuild/_config/special_env_vars.py        |  2 +-
 man/make.conf.5                               |  3 ++
 4 files changed, 52 insertions(+), 4 deletions(-)

diff --git a/lib/_emerge/AbstractEbuildProcess.py b/lib/_emerge/AbstractEbuildProcess.py
index ddf04e9b3..96801f200 100644
--- a/lib/_emerge/AbstractEbuildProcess.py
+++ b/lib/_emerge/AbstractEbuildProcess.py
@@ -20,7 +20,7 @@ from portage.package.ebuild._ipc.QueryCommand import QueryCommand
 from portage import shutil, os
 from portage.util.futures import asyncio
 from portage.util._pty import _create_pty_or_pipe
-from portage.util import apply_secpass_permissions
+from portage.util import apply_secpass_permissions, shlex_split
 
 portage.proxy.lazyimport.lazyimport(globals(),
 	'portage.package.ebuild.doebuild:_global_pid_phases',
@@ -196,6 +196,12 @@ class AbstractEbuildProcess(SpawnProcess):
 			null_fd = os.open('/dev/null', os.O_RDONLY)
 			self.fd_pipes[0] = null_fd
 
+		log_filter_command = self.settings.get('PORTAGE_LOG_FILTER_COMMAND')
+		if log_filter_command is not None:
+			log_filter_command = shlex_split(log_filter_command)
+			if log_filter_command:
+				self.log_filter_command = log_filter_command
+
 		try:
 			SpawnProcess._start(self)
 		finally:
diff --git a/lib/_emerge/SpawnProcess.py b/lib/_emerge/SpawnProcess.py
index 395d66bb9..f439a5241 100644
--- a/lib/_emerge/SpawnProcess.py
+++ b/lib/_emerge/SpawnProcess.py
@@ -34,8 +34,8 @@ class SpawnProcess(SubProcess):
 		"path_lookup", "pre_exec", "close_fds", "cgroup",
 		"unshare_ipc", "unshare_mount", "unshare_pid", "unshare_net")
 
-	__slots__ = ("args",) + \
-		_spawn_kwarg_names + ("_pipe_logger", "_selinux_type",)
+	__slots__ = ("args", "log_filter_command") + \
+		_spawn_kwarg_names + ("_filter_proc", "_pipe_logger", "_selinux_type",)
 
 	# Max number of attempts to kill the processes listed in cgroup.procs,
 	# given that processes may fork before they can be killed.
@@ -137,6 +137,32 @@ class SpawnProcess(SubProcess):
 						fcntl.fcntl(stdout_fd,
 						fcntl.F_GETFD) | fcntl.FD_CLOEXEC)
 
+		if self.log_filter_command is not None:
+			pr, pw = os.pipe()
+			# stderr goes to /dev/null because the program is expected
+			# to generate an EIO error when master_fd reaches EOF.
+			with open(os.devnull, 'wb', 0) as null_fd:
+				filter_fd_pipes = {0: master_fd, 1: pw, 2: null_fd.fileno()}
+				self._filter_proc = SpawnProcess(
+					args=self.log_filter_command,
+					env=self.env,
+					fd_pipes=filter_fd_pipes,
+					scheduler=self.scheduler)
+				self._filter_proc.addExitListener(self._filter_proc_exit)
+				try:
+					self._filter_proc.start()
+				except portage.exception.CommandNotFound:
+					self._filter_proc.removeExitListener(self._filter_proc_exit)
+					self._filter_proc._unregister()
+					self._filter_proc = None
+					os.close(pw)
+					os.close(pr)
+				else:
+					os.close(pw)
+					os.close(master_fd)
+					# Send self._filter_proc output to PipeLogger
+					master_fd = pr
+
 		self._pipe_logger = PipeLogger(background=self.background,
 			scheduler=self.scheduler, input_fd=master_fd,
 			log_file_path=log_file_path,
@@ -171,11 +197,24 @@ class SpawnProcess(SubProcess):
 		self._pipe_logger = None
 		self._async_waitpid()
 
+	def _filter_proc_exit(self, filter_proc):
+		self._filter_proc = None
+
+	def _async_waitpid(self):
+		if self._filter_proc is not None:
+			# All output should have been collected by now, so kill it.
+			self._filter_proc.cancel()
+		SubProcess._async_waitpid(self)
+
 	def _unregister(self):
 		SubProcess._unregister(self)
 		if self.cgroup is not None:
 			self._cgroup_cleanup()
 			self.cgroup = None
+		if self._filter_proc is not None:
+			self._filter_proc.removeExitListener(self._filter_proc_exit)
+			self._filter_proc.cancel()
+			self._filter_proc = None
 		if self._pipe_logger is not None:
 			self._pipe_logger.cancel()
 			self._pipe_logger = None
diff --git a/lib/portage/package/ebuild/_config/special_env_vars.py b/lib/portage/package/ebuild/_config/special_env_vars.py
index dc01339f7..4f9430e58 100644
--- a/lib/portage/package/ebuild/_config/special_env_vars.py
+++ b/lib/portage/package/ebuild/_config/special_env_vars.py
@@ -175,7 +175,7 @@ environ_filter += [
 	"PORTAGE_RO_DISTDIRS",
 	"PORTAGE_RSYNC_EXTRA_OPTS", "PORTAGE_RSYNC_OPTS",
 	"PORTAGE_RSYNC_RETRIES", "PORTAGE_SSH_OPTS", "PORTAGE_SYNC_STALE",
-	"PORTAGE_USE",
+	"PORTAGE_USE", "PORTAGE_LOG_FILTER_COMMAND",
 	"PORTAGE_LOGDIR", "PORTAGE_LOGDIR_CLEAN",
 	"QUICKPKG_DEFAULT_OPTS", "REPOMAN_DEFAULT_OPTS",
 	"RESUMECOMMAND", "RESUMECOMMAND_FTP",
diff --git a/man/make.conf.5 b/man/make.conf.5
index f82fed65a..04ac75de4 100644
--- a/man/make.conf.5
+++ b/man/make.conf.5
@@ -979,6 +979,9 @@ with an integer pid. For example, a value of "ionice \-c 3 \-p \\${PID}"
 will set idle io priority. For more information about ionice, see
 \fBionice\fR(1). This variable is unset by default.
 .TP
+.B PORTAGE_LOG_FILTER_COMMAND
+This variable specifies a command that filters build log content.
+.TP
 .B PORTAGE_LOGDIR
 This variable defines the directory in which per\-ebuild logs are kept.
 Logs are created only when this is set. They are stored as
-- 
2.24.1