Gentoo Archives: gentoo-portage-dev

From: Zac Medico <zmedico@g.o>
To: gentoo-portage-dev@l.g.o
Cc: Zac Medico <zmedico@g.o>
Subject: [gentoo-portage-dev] [PATCH] Use cached portage.getpid() function (bug 739540)
Date: Sat, 29 Aug 2020 21:30:48
Message-Id: 20200829212813.18453-1-zmedico@gentoo.org
1 Use the cached portage.getpid() function to avoid unnecessary syscalls,
2 and update the cache after each call to os.fork() where the fork may
3 invoke portage APIs.
4
5 Bug: https://bugs.gentoo.org/739540
6 Signed-off-by: Zac Medico <zmedico@g.o>
7 ---
8 bin/quickpkg | 2 +-
9 lib/_emerge/EbuildBinpkg.py | 4 +++-
10 lib/_emerge/actions.py | 2 +-
11 lib/portage/_emirrordist/FetchTask.py | 4 ++--
12 lib/portage/cache/metadata.py | 4 +++-
13 lib/portage/elog/mod_mail_summary.py | 5 ++---
14 lib/portage/elog/mod_save_summary.py | 2 +-
15 lib/portage/locks.py | 2 +-
16 lib/portage/package/ebuild/doebuild.py | 2 +-
17 lib/portage/process.py | 16 +++++++++++-----
18 lib/portage/tests/locks/test_lock_nonblock.py | 1 +
19 .../futures/asyncio/test_wakeup_fd_sigchld.py | 2 +-
20 lib/portage/util/__init__.py | 4 ++--
21 lib/portage/util/_eventloop/EventLoop.py | 6 +++---
22 .../util/_eventloop/asyncio_event_loop.py | 4 ++--
23 lib/portage/util/_eventloop/global_event_loop.py | 7 +++----
24 lib/portage/util/locale.py | 1 +
25 lib/portage/util/movefile.py | 2 +-
26 lib/portage/util/socks5.py | 2 +-
27 lib/portage/xpak.py | 2 +-
28 20 files changed, 42 insertions(+), 32 deletions(-)
29
30 diff --git a/bin/quickpkg b/bin/quickpkg
31 index be7d1d7af..a171b3bd5 100755
32 --- a/bin/quickpkg
33 +++ b/bin/quickpkg
34 @@ -111,7 +111,7 @@ def quickpkg_atom(options, infos, arg, eout):
35 vardb.aux_update(cpv, update_metadata)
36 xpdata = xpak.xpak(dblnk.dbdir)
37 binpkg_tmpfile = os.path.join(bintree.pkgdir,
38 - cpv + ".tbz2." + str(os.getpid()))
39 + cpv + ".tbz2." + str(portage.getpid()))
40 ensure_dirs(os.path.dirname(binpkg_tmpfile))
41 binpkg_compression = settings.get("BINPKG_COMPRESS", "bzip2")
42 try:
43 diff --git a/lib/_emerge/EbuildBinpkg.py b/lib/_emerge/EbuildBinpkg.py
44 index 6e098eb8a..879b3a9aa 100644
45 --- a/lib/_emerge/EbuildBinpkg.py
46 +++ b/lib/_emerge/EbuildBinpkg.py
47 @@ -3,6 +3,8 @@
48
49 from _emerge.CompositeTask import CompositeTask
50 from _emerge.EbuildPhase import EbuildPhase
51 +
52 +import portage
53 from portage import os
54
55 class EbuildBinpkg(CompositeTask):
56 @@ -17,7 +19,7 @@ class EbuildBinpkg(CompositeTask):
57 root_config = pkg.root_config
58 bintree = root_config.trees["bintree"]
59 binpkg_tmpfile = os.path.join(bintree.pkgdir,
60 - pkg.cpv + ".tbz2." + str(os.getpid()))
61 + pkg.cpv + ".tbz2." + str(portage.getpid()))
62 bintree._ensure_dir(os.path.dirname(binpkg_tmpfile))
63
64 self._binpkg_tmpfile = binpkg_tmpfile
65 diff --git a/lib/_emerge/actions.py b/lib/_emerge/actions.py
66 index 063f5d4a0..a4ecfe43d 100644
67 --- a/lib/_emerge/actions.py
68 +++ b/lib/_emerge/actions.py
69 @@ -2623,7 +2623,7 @@ def ionice(settings):
70 if not ionice_cmd:
71 return
72
73 - variables = {"PID" : str(os.getpid())}
74 + variables = {"PID" : str(portage.getpid())}
75 cmd = [varexpand(x, mydict=variables) for x in ionice_cmd]
76
77 try:
78 diff --git a/lib/portage/_emirrordist/FetchTask.py b/lib/portage/_emirrordist/FetchTask.py
79 index 457ca2ac6..41f96b962 100644
80 --- a/lib/portage/_emirrordist/FetchTask.py
81 +++ b/lib/portage/_emirrordist/FetchTask.py
82 @@ -415,7 +415,7 @@ class FetchTask(CompositeTask):
83 self._fetch_tmp_dir_info = 'distfiles'
84 distdir = self.config.options.distfiles
85
86 - tmp_basename = self.distfile + '._emirrordist_fetch_.%s' % os.getpid()
87 + tmp_basename = self.distfile + '._emirrordist_fetch_.%s' % portage.getpid()
88
89 variables = {
90 "DISTDIR": distdir,
91 @@ -622,7 +622,7 @@ class FetchTask(CompositeTask):
92
93 head, tail = os.path.split(dest)
94 hardlink_tmp = os.path.join(head, ".%s._mirrordist_hardlink_.%s" % \
95 - (tail, os.getpid()))
96 + (tail, portage.getpid()))
97
98 try:
99 try:
100 diff --git a/lib/portage/cache/metadata.py b/lib/portage/cache/metadata.py
101 index db81b8ba1..bd1b70fa0 100644
102 --- a/lib/portage/cache/metadata.py
103 +++ b/lib/portage/cache/metadata.py
104 @@ -6,6 +6,8 @@ import errno
105 import re
106 import stat
107 from operator import attrgetter
108 +
109 +import portage
110 from portage import os
111 from portage import _encodings
112 from portage import _unicode_encode
113 @@ -122,7 +124,7 @@ class database(flat_hash.database):
114
115 s = cpv.rfind("/")
116 fp = os.path.join(self.location,cpv[:s],
117 - ".update.%i.%s" % (os.getpid(), cpv[s+1:]))
118 + ".update.%i.%s" % (portage.getpid(), cpv[s+1:]))
119 try:
120 myf = open(_unicode_encode(fp,
121 encoding=_encodings['fs'], errors='strict'), 'wb')
122 diff --git a/lib/portage/elog/mod_mail_summary.py b/lib/portage/elog/mod_mail_summary.py
123 index ac260880e..789f55f4d 100644
124 --- a/lib/portage/elog/mod_mail_summary.py
125 +++ b/lib/portage/elog/mod_mail_summary.py
126 @@ -6,7 +6,6 @@ import portage
127 from portage.exception import AlarmSignal, PortageException
128 from portage.localization import _
129 from portage.util import writemsg
130 -from portage import os
131 from portage import _encodings
132 from portage import _unicode_decode
133
134 @@ -22,7 +21,7 @@ def process(mysettings, key, logentries, fulltext):
135 time.strftime("%Y%m%d-%H%M%S %Z", time.localtime(time.time())),
136 encoding=_encodings['content'], errors='replace')
137 header = _(">>> Messages generated for package %(pkg)s by process %(pid)d on %(time)s:\n\n") % \
138 - {"pkg": key, "pid": os.getpid(), "time": time_str}
139 + {"pkg": key, "pid": portage.getpid(), "time": time_str}
140 config_root = mysettings["PORTAGE_CONFIGROOT"]
141
142 # Copy needed variables from the config instance,
143 @@ -66,7 +65,7 @@ def _finalize(mysettings, items):
144 mysubject = mysubject.replace("${HOST}", socket.getfqdn())
145
146 mybody = _("elog messages for the following packages generated by "
147 - "process %(pid)d on host %(host)s:\n") % {"pid": os.getpid(), "host": socket.getfqdn()}
148 + "process %(pid)d on host %(host)s:\n") % {"pid": portage.getpid(), "host": socket.getfqdn()}
149 for key in items:
150 mybody += "- %s\n" % key
151
152 diff --git a/lib/portage/elog/mod_save_summary.py b/lib/portage/elog/mod_save_summary.py
153 index 946a1ad4c..6e24b608f 100644
154 --- a/lib/portage/elog/mod_save_summary.py
155 +++ b/lib/portage/elog/mod_save_summary.py
156 @@ -79,7 +79,7 @@ def process(mysettings, key, logentries, fulltext):
157 encoding=_encodings['content'], errors='replace')
158 elogfile.write(_(">>> Messages generated by process "
159 "%(pid)d on %(time)s for package %(pkg)s:\n\n") %
160 - {"pid": os.getpid(), "time": time_str, "pkg": key})
161 + {"pid": portage.getpid(), "time": time_str, "pkg": key})
162 elogfile.write(_unicode_decode(fulltext))
163 elogfile.write("\n")
164 elogfile.close()
165 diff --git a/lib/portage/locks.py b/lib/portage/locks.py
166 index 701093024..1073343be 100644
167 --- a/lib/portage/locks.py
168 +++ b/lib/portage/locks.py
169 @@ -501,7 +501,7 @@ def unlockfile(mytuple):
170 def hardlock_name(path):
171 base, tail = os.path.split(path)
172 return os.path.join(base, ".%s.hardlock-%s-%s" %
173 - (tail, portage._decode_argv([os.uname()[1]])[0], os.getpid()))
174 + (tail, portage._decode_argv([os.uname()[1]])[0], portage.getpid()))
175
176 def hardlink_is_mine(link, lock):
177 try:
178 diff --git a/lib/portage/package/ebuild/doebuild.py b/lib/portage/package/ebuild/doebuild.py
179 index 7bb942966..3b1991b28 100644
180 --- a/lib/portage/package/ebuild/doebuild.py
181 +++ b/lib/portage/package/ebuild/doebuild.py
182 @@ -1178,7 +1178,7 @@ def doebuild(myebuild, mydo, _unused=DeprecationWarning, settings=None, debug=0,
183 bintree = portage.db[mysettings['EROOT']]['bintree']
184 mysettings["PORTAGE_BINPKG_TMPFILE"] = \
185 bintree.getname(mysettings.mycpv) + \
186 - ".%s" % (os.getpid(),)
187 + ".%s" % (portage.getpid(),)
188 bintree._ensure_dir(os.path.dirname(
189 mysettings["PORTAGE_BINPKG_TMPFILE"]))
190 else:
191 diff --git a/lib/portage/process.py b/lib/portage/process.py
192 index 8d4cf164e..48548bacc 100644
193 --- a/lib/portage/process.py
194 +++ b/lib/portage/process.py
195 @@ -79,11 +79,11 @@ if _fd_dir is not None:
196 raise
197 return range(max_fd_limit)
198
199 -elif os.path.isdir("/proc/%s/fd" % os.getpid()):
200 +elif os.path.isdir("/proc/%s/fd" % portage.getpid()):
201 # In order for this function to work in forked subprocesses,
202 # os.getpid() must be called from inside the function.
203 def get_open_fds():
204 - return (int(fd) for fd in os.listdir("/proc/%s/fd" % os.getpid())
205 + return (int(fd) for fd in os.listdir("/proc/%s/fd" % portage.getpid())
206 if fd.isdigit())
207
208 else:
209 @@ -363,12 +363,13 @@ def spawn(mycommand, env=None, opt_name=None, fd_pipes=None, returnpid=False,
210 # fork, so that the result is cached in the main process.
211 bool(groups)
212
213 - parent_pid = os.getpid()
214 + parent_pid = portage.getpid()
215 pid = None
216 try:
217 pid = os.fork()
218
219 if pid == 0:
220 + portage._ForkWatcher.hook(portage._ForkWatcher)
221 try:
222 _exec(binary, mycommand, opt_name, fd_pipes,
223 env, gid, groups, uid, umask, cwd, pre_exec, close_fds,
224 @@ -386,7 +387,9 @@ def spawn(mycommand, env=None, opt_name=None, fd_pipes=None, returnpid=False,
225 sys.stderr.flush()
226
227 finally:
228 - if pid == 0 or (pid is None and os.getpid() != parent_pid):
229 + # Don't used portage.getpid() here, due to a race with the above
230 + # portage._ForkWatcher cache update.
231 + if pid == 0 or (pid is None and _os.getpid() != parent_pid):
232 # Call os._exit() from a finally block in order
233 # to suppress any finally blocks from earlier
234 # in the call stack (see bug #345289). This
235 @@ -603,7 +606,7 @@ def _exec(binary, mycommand, opt_name, fd_pipes,
236 # it is done before we start forking children
237 if cgroup:
238 with open(os.path.join(cgroup, 'cgroup.procs'), 'a') as f:
239 - f.write('%d\n' % os.getpid())
240 + f.write('%d\n' % portage.getpid())
241
242 # Unshare (while still uid==0)
243 if unshare_net or unshare_ipc or unshare_mount or unshare_pid:
244 @@ -640,6 +643,9 @@ def _exec(binary, mycommand, opt_name, fd_pipes,
245 if unshare_pid:
246 main_child_pid = os.fork()
247 if main_child_pid == 0:
248 + # The portage.getpid() cache may need to be updated here,
249 + # in case the pre_exec function invokes portage APIs.
250 + portage._ForkWatcher.hook(portage._ForkWatcher)
251 # pid namespace requires us to become init
252 binary, myargs = portage._python_interpreter, [
253 portage._python_interpreter,
254 diff --git a/lib/portage/tests/locks/test_lock_nonblock.py b/lib/portage/tests/locks/test_lock_nonblock.py
255 index 02ba16ad9..3448b84f6 100644
256 --- a/lib/portage/tests/locks/test_lock_nonblock.py
257 +++ b/lib/portage/tests/locks/test_lock_nonblock.py
258 @@ -19,6 +19,7 @@ class LockNonblockTestCase(TestCase):
259 lock1 = portage.locks.lockfile(path)
260 pid = os.fork()
261 if pid == 0:
262 + portage._ForkWatcher.hook(portage._ForkWatcher)
263 portage.locks._close_fds()
264 # Disable close_fds since we don't exec
265 # (see _setup_pipes docstring).
266 diff --git a/lib/portage/tests/util/futures/asyncio/test_wakeup_fd_sigchld.py b/lib/portage/tests/util/futures/asyncio/test_wakeup_fd_sigchld.py
267 index e5b104e0f..c37a6338b 100644
268 --- a/lib/portage/tests/util/futures/asyncio/test_wakeup_fd_sigchld.py
269 +++ b/lib/portage/tests/util/futures/asyncio/test_wakeup_fd_sigchld.py
270 @@ -39,7 +39,7 @@ proc = loop.run_until_complete(asyncio.create_subprocess_exec('sleep', '0', loop
271 loop.run_until_complete(proc.wait())
272
273 for i in range(8192):
274 - os.kill(os.getpid(), signal.SIGCHLD)
275 + os.kill(portage.getpid(), signal.SIGCHLD)
276
277 # Verify that the child watcher still works correctly
278 # (this will hang if it doesn't).
279 diff --git a/lib/portage/util/__init__.py b/lib/portage/util/__init__.py
280 index c14c15fe8..0412b2b59 100644
281 --- a/lib/portage/util/__init__.py
282 +++ b/lib/portage/util/__init__.py
283 @@ -1266,7 +1266,7 @@ class atomic_ofstream(ObjectProxy):
284 if follow_links:
285 canonical_path = os.path.realpath(filename)
286 object.__setattr__(self, '_real_name', canonical_path)
287 - tmp_name = "%s.%i" % (canonical_path, os.getpid())
288 + tmp_name = "%s.%i" % (canonical_path, portage.getpid())
289 try:
290 object.__setattr__(self, '_file',
291 open_func(_unicode_encode(tmp_name,
292 @@ -1281,7 +1281,7 @@ class atomic_ofstream(ObjectProxy):
293 # new error if necessary.
294
295 object.__setattr__(self, '_real_name', filename)
296 - tmp_name = "%s.%i" % (filename, os.getpid())
297 + tmp_name = "%s.%i" % (filename, portage.getpid())
298 object.__setattr__(self, '_file',
299 open_func(_unicode_encode(tmp_name,
300 encoding=_encodings['fs'], errors='strict'),
301 diff --git a/lib/portage/util/_eventloop/EventLoop.py b/lib/portage/util/_eventloop/EventLoop.py
302 index 94e637853..ff2b73255 100644
303 --- a/lib/portage/util/_eventloop/EventLoop.py
304 +++ b/lib/portage/util/_eventloop/EventLoop.py
305 @@ -188,7 +188,7 @@ class EventLoop:
306 self._sigchld_read = None
307 self._sigchld_write = None
308 self._sigchld_src_id = None
309 - self._pid = os.getpid()
310 + self._pid = portage.getpid()
311 self._asyncio_wrapper = _PortageEventLoop(loop=self)
312 self._asyncio_child_watcher = _PortageChildWatcher(self)
313
314 @@ -431,7 +431,7 @@ class EventLoop:
315 # If this signal handler was not installed by the
316 # current process then the signal doesn't belong to
317 # this EventLoop instance.
318 - if os.getpid() == self._pid:
319 + if portage.getpid() == self._pid:
320 os.write(self._sigchld_write, b'\0')
321
322 def _sigchld_io_cb(self, fd, events):
323 @@ -1026,7 +1026,7 @@ class EventLoop:
324 log_lines.append('{}: {}'.format(key, value))
325
326 logging.error('\n'.join(log_lines), exc_info=exc_info)
327 - os.kill(os.getpid(), signal.SIGTERM)
328 + os.kill(portage.getpid(), signal.SIGTERM)
329
330 def call_exception_handler(self, context):
331 """
332 diff --git a/lib/portage/util/_eventloop/asyncio_event_loop.py b/lib/portage/util/_eventloop/asyncio_event_loop.py
333 index 605e7243b..836f1c30a 100644
334 --- a/lib/portage/util/_eventloop/asyncio_event_loop.py
335 +++ b/lib/portage/util/_eventloop/asyncio_event_loop.py
336 @@ -69,7 +69,7 @@ class AsyncioEventLoop(_AbstractEventLoop):
337 # in order to ensure that emerge exits immediately (though
338 # uncleanly).
339 signal.signal(signal.SIGTERM, signal.SIG_DFL)
340 - os.kill(os.getpid(), signal.SIGTERM)
341 + os.kill(portage.getpid(), signal.SIGTERM)
342
343 def _create_future(self):
344 """
345 @@ -117,7 +117,7 @@ class AsyncioEventLoop(_AbstractEventLoop):
346 self._wakeup_fd = -1
347 # Account for any signals that may have arrived between
348 # set_wakeup_fd calls.
349 - os.kill(os.getpid(), signal.SIGCHLD)
350 + os.kill(portage.getpid(), signal.SIGCHLD)
351 try:
352 return self._loop.run_until_complete(future)
353 finally:
354 diff --git a/lib/portage/util/_eventloop/global_event_loop.py b/lib/portage/util/_eventloop/global_event_loop.py
355 index 1db958d2e..21a1d1970 100644
356 --- a/lib/portage/util/_eventloop/global_event_loop.py
357 +++ b/lib/portage/util/_eventloop/global_event_loop.py
358 @@ -1,13 +1,12 @@
359 # Copyright 2012-2020 Gentoo Authors
360 # Distributed under the terms of the GNU General Public License v2
361
362 -import os
363 -
364 +import portage
365 from .EventLoop import EventLoop
366 from portage.util._eventloop.asyncio_event_loop import AsyncioEventLoop
367
368
369 -_MAIN_PID = os.getpid()
370 +_MAIN_PID = portage.getpid()
371 _instances = {}
372
373
374 @@ -17,7 +16,7 @@ def global_event_loop():
375 belongs exclusively to the current process.
376 """
377
378 - pid = os.getpid()
379 + pid = portage.getpid()
380 instance = _instances.get(pid)
381 if instance is not None:
382 return instance
383 diff --git a/lib/portage/util/locale.py b/lib/portage/util/locale.py
384 index 99c8f7ae7..58c687139 100644
385 --- a/lib/portage/util/locale.py
386 +++ b/lib/portage/util/locale.py
387 @@ -102,6 +102,7 @@ def check_locale(silent=False, env=None):
388
389 pid = os.fork()
390 if pid == 0:
391 + portage._ForkWatcher.hook(portage._ForkWatcher)
392 try:
393 if env is not None:
394 try:
395 diff --git a/lib/portage/util/movefile.py b/lib/portage/util/movefile.py
396 index 4f8054f29..a251d369d 100644
397 --- a/lib/portage/util/movefile.py
398 +++ b/lib/portage/util/movefile.py
399 @@ -209,7 +209,7 @@ def movefile(src, dest, newmtime=None, sstat=None, mysettings=None,
400 if hardlink_candidates:
401 head, tail = os.path.split(dest)
402 hardlink_tmp = os.path.join(head, ".%s._portage_merge_.%s" % \
403 - (tail, os.getpid()))
404 + (tail, portage.getpid()))
405 try:
406 os.unlink(hardlink_tmp)
407 except OSError as e:
408 diff --git a/lib/portage/util/socks5.py b/lib/portage/util/socks5.py
409 index 9f22c1dbe..a76d1a741 100644
410 --- a/lib/portage/util/socks5.py
411 +++ b/lib/portage/util/socks5.py
412 @@ -42,7 +42,7 @@ class ProxyManager:
413 portage.util.ensure_dirs(tmpdir, **ensure_dirs_kwargs)
414
415 self.socket_path = os.path.join(tmpdir,
416 - '.portage.%d.net.sock' % os.getpid())
417 + '.portage.%d.net.sock' % portage.getpid())
418 server_bin = os.path.join(settings['PORTAGE_BIN_PATH'], 'socks5-server.py')
419 spawn_kwargs = {}
420 # The portage_uid check solves EPERM failures in Travis CI.
421 diff --git a/lib/portage/xpak.py b/lib/portage/xpak.py
422 index 2a4bcda21..9063c4c56 100644
423 --- a/lib/portage/xpak.py
424 +++ b/lib/portage/xpak.py
425 @@ -325,7 +325,7 @@ class tbz2:
426 self.scan() # Don't care about condition... We'll rewrite the data anyway.
427
428 if break_hardlinks and self.filestat and self.filestat.st_nlink > 1:
429 - tmp_fname = "%s.%d" % (self.file, os.getpid())
430 + tmp_fname = "%s.%d" % (self.file, portage.getpid())
431 copyfile(self.file, tmp_fname)
432 try:
433 portage.util.apply_stat_permissions(self.file, self.filestat)
434 --
435 2.25.3