Gentoo Archives: gentoo-portage-dev

From: Zac Medico <zmedico@g.o>
To: gentoo-portage-dev@l.g.o
Cc: Zac Medico <zmedico@g.o>
Subject: [gentoo-portage-dev] [PATCH] MergeProcess: replace os.fork with multiprocessing.Process (bug 730192)
Date: Sun, 19 Jul 2020 06:44:46
Message-Id: 20200719064417.45582-1-zmedico@gentoo.org
1 Fix the MergeProcess _spawn method to call the superclass _spawn
2 method, in order to replace os.fork with multiprocessing.Process,
3 promoting a healthy state for the forked interpreter.
4
5 Bug: https://bugs.gentoo.org/730192
6 Signed-off-by: Zac Medico <zmedico@g.o>
7 ---
8 lib/portage/dbapi/_MergeProcess.py | 102 ++++++++---------------------
9 1 file changed, 28 insertions(+), 74 deletions(-)
10
11 diff --git a/lib/portage/dbapi/_MergeProcess.py b/lib/portage/dbapi/_MergeProcess.py
12 index 274ef586f..adf078cb2 100644
13 --- a/lib/portage/dbapi/_MergeProcess.py
14 +++ b/lib/portage/dbapi/_MergeProcess.py
15 @@ -3,9 +3,6 @@
16
17 import io
18 import platform
19 -import signal
20 -import sys
21 -import traceback
22
23 import fcntl
24 import portage
25 @@ -24,7 +21,7 @@ class MergeProcess(ForkProcess):
26 'vartree', 'blockers', 'pkgloc', 'infloc', 'myebuild',
27 'mydbapi', 'postinst_failure', 'prev_mtimes', 'unmerge',
28 '_elog_reader_fd',
29 - '_buf', '_elog_keys', '_locked_vdb')
30 + '_buf', '_counter', '_dblink', '_elog_keys', '_locked_vdb')
31
32 def _start(self):
33 # Portage should always call setcpv prior to this
34 @@ -132,57 +129,31 @@ class MergeProcess(ForkProcess):
35 # FEATURES=parallel-install skips this lock in order to
36 # improve performance, and the risk is practically negligible.
37 self._lock_vdb()
38 - counter = None
39 if not self.unmerge:
40 - counter = self.vartree.dbapi.counter_tick()
41 -
42 - parent_pid = os.getpid()
43 - pid = None
44 - try:
45 - pid = os.fork()
46 -
47 - if pid != 0:
48 - if not isinstance(pid, int):
49 - raise AssertionError(
50 - "fork returned non-integer: %s" % (repr(pid),))
51 -
52 - os.close(elog_writer_fd)
53 - self._elog_reader_fd = elog_reader_fd
54 - self._buf = ""
55 - self._elog_keys = set()
56 - # Discard messages which will be collected by the subprocess,
57 - # in order to avoid duplicates (bug #446136).
58 - portage.elog.messages.collect_messages(key=mylink.mycpv)
59 -
60 - # invalidate relevant vardbapi caches
61 - if self.vartree.dbapi._categories is not None:
62 - self.vartree.dbapi._categories = None
63 - self.vartree.dbapi._pkgs_changed = True
64 - self.vartree.dbapi._clear_pkg_cache(mylink)
65 -
66 - return [pid]
67 -
68 - os.close(elog_reader_fd)
69 -
70 - # Use default signal handlers in order to avoid problems
71 - # killing subprocesses as reported in bug #353239.
72 - signal.signal(signal.SIGINT, signal.SIG_DFL)
73 - signal.signal(signal.SIGTERM, signal.SIG_DFL)
74 -
75 - # Unregister SIGCHLD handler and wakeup_fd for the parent
76 - # process's event loop (bug 655656).
77 - signal.signal(signal.SIGCHLD, signal.SIG_DFL)
78 - try:
79 - wakeup_fd = signal.set_wakeup_fd(-1)
80 - if wakeup_fd > 0:
81 - os.close(wakeup_fd)
82 - except (ValueError, OSError):
83 - pass
84 -
85 - portage.locks._close_fds()
86 - # We don't exec, so use close_fds=False
87 - # (see _setup_pipes docstring).
88 - portage.process._setup_pipes(fd_pipes, close_fds=False)
89 + self._counter = self.vartree.dbapi.counter_tick()
90 +
91 + self._dblink = mylink
92 + self._elog_reader_fd = elog_reader_fd
93 + pids = super(MergeProcess, self)._spawn(args, fd_pipes, **kwargs)
94 + os.close(elog_writer_fd)
95 + self._buf = ""
96 + self._elog_keys = set()
97 + # Discard messages which will be collected by the subprocess,
98 + # in order to avoid duplicates (bug #446136).
99 + portage.elog.messages.collect_messages(key=mylink.mycpv)
100 +
101 + # invalidate relevant vardbapi caches
102 + if self.vartree.dbapi._categories is not None:
103 + self.vartree.dbapi._categories = None
104 + self.vartree.dbapi._pkgs_changed = True
105 + self.vartree.dbapi._clear_pkg_cache(mylink)
106 +
107 + return pids
108 +
109 + def _run(self):
110 + os.close(self._elog_reader_fd)
111 + counter = self._counter
112 + mylink = self._dblink
113
114 portage.output.havecolor = self.settings.get('NOCOLOR') \
115 not in ('yes', 'true')
116 @@ -207,8 +178,7 @@ class MergeProcess(ForkProcess):
117 self.settings.backup_changes("PORTAGE_BACKGROUND")
118
119 rval = 1
120 - try:
121 - if self.unmerge:
122 + if self.unmerge:
123 if not mylink.exists():
124 rval = os.EX_OK
125 elif mylink.unmerge(
126 @@ -219,27 +189,11 @@ class MergeProcess(ForkProcess):
127 finally:
128 mylink.unlockdb()
129 rval = os.EX_OK
130 - else:
131 + else:
132 rval = mylink.merge(self.pkgloc, self.infloc,
133 myebuild=self.myebuild, mydbapi=self.mydbapi,
134 prev_mtimes=self.prev_mtimes, counter=counter)
135 - except SystemExit:
136 - raise
137 - except:
138 - traceback.print_exc()
139 - # os._exit() skips stderr flush!
140 - sys.stderr.flush()
141 - finally:
142 - os._exit(rval)
143 -
144 - finally:
145 - if pid == 0 or (pid is None and os.getpid() != parent_pid):
146 - # Call os._exit() from a finally block in order
147 - # to suppress any finally blocks from earlier
148 - # in the call stack (see bug #345289). This
149 - # finally block has to be setup before the fork
150 - # in order to avoid a race condition.
151 - os._exit(1)
152 + return rval
153
154 def _async_waitpid_cb(self, *args, **kwargs):
155 """
156 --
157 2.25.3