Gentoo Archives: gentoo-commits

From: Zac Medico <zmedico@g.o>
To: gentoo-commits@l.g.o
Subject: [gentoo-commits] proj/portage:master commit in: lib/portage/, lib/portage/util/_eventloop/, lib/portage/util/, lib/portage/elog/, ...
Date: Mon, 31 Aug 2020 06:27:25
Message-Id: 1598855105.cf07a23a0ff77049476e0f4b94afbb6a844806fa.zmedico@gentoo
1 commit: cf07a23a0ff77049476e0f4b94afbb6a844806fa
2 Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
3 AuthorDate: Sat Aug 29 20:13:44 2020 +0000
4 Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
5 CommitDate: Mon Aug 31 06:25:05 2020 +0000
6 URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=cf07a23a
7
8 Use cached portage.getpid() function (bug 739540)
9
10 Use the cached portage.getpid() function to avoid unnecessary syscalls,
11 and update the cache after each call to os.fork() where the fork may
12 invoke portage APIs.
13
14 Bug: https://bugs.gentoo.org/739540
15 Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>
16
17 bin/quickpkg | 2 +-
18 lib/_emerge/EbuildBinpkg.py | 4 +++-
19 lib/_emerge/actions.py | 2 +-
20 lib/portage/_emirrordist/FetchTask.py | 4 ++--
21 lib/portage/cache/metadata.py | 4 +++-
22 lib/portage/elog/mod_mail_summary.py | 5 ++---
23 lib/portage/elog/mod_save_summary.py | 2 +-
24 lib/portage/locks.py | 2 +-
25 lib/portage/package/ebuild/doebuild.py | 2 +-
26 lib/portage/process.py | 16 +++++++++++-----
27 lib/portage/tests/locks/test_lock_nonblock.py | 1 +
28 .../tests/util/futures/asyncio/test_wakeup_fd_sigchld.py | 2 +-
29 lib/portage/util/__init__.py | 4 ++--
30 lib/portage/util/_eventloop/EventLoop.py | 6 +++---
31 lib/portage/util/_eventloop/asyncio_event_loop.py | 4 ++--
32 lib/portage/util/_eventloop/global_event_loop.py | 7 +++----
33 lib/portage/util/locale.py | 1 +
34 lib/portage/util/movefile.py | 2 +-
35 lib/portage/util/socks5.py | 2 +-
36 lib/portage/xpak.py | 2 +-
37 20 files changed, 42 insertions(+), 32 deletions(-)
38
39 diff --git a/bin/quickpkg b/bin/quickpkg
40 index be7d1d7af..a171b3bd5 100755
41 --- a/bin/quickpkg
42 +++ b/bin/quickpkg
43 @@ -111,7 +111,7 @@ def quickpkg_atom(options, infos, arg, eout):
44 vardb.aux_update(cpv, update_metadata)
45 xpdata = xpak.xpak(dblnk.dbdir)
46 binpkg_tmpfile = os.path.join(bintree.pkgdir,
47 - cpv + ".tbz2." + str(os.getpid()))
48 + cpv + ".tbz2." + str(portage.getpid()))
49 ensure_dirs(os.path.dirname(binpkg_tmpfile))
50 binpkg_compression = settings.get("BINPKG_COMPRESS", "bzip2")
51 try:
52
53 diff --git a/lib/_emerge/EbuildBinpkg.py b/lib/_emerge/EbuildBinpkg.py
54 index 6e098eb8a..879b3a9aa 100644
55 --- a/lib/_emerge/EbuildBinpkg.py
56 +++ b/lib/_emerge/EbuildBinpkg.py
57 @@ -3,6 +3,8 @@
58
59 from _emerge.CompositeTask import CompositeTask
60 from _emerge.EbuildPhase import EbuildPhase
61 +
62 +import portage
63 from portage import os
64
65 class EbuildBinpkg(CompositeTask):
66 @@ -17,7 +19,7 @@ class EbuildBinpkg(CompositeTask):
67 root_config = pkg.root_config
68 bintree = root_config.trees["bintree"]
69 binpkg_tmpfile = os.path.join(bintree.pkgdir,
70 - pkg.cpv + ".tbz2." + str(os.getpid()))
71 + pkg.cpv + ".tbz2." + str(portage.getpid()))
72 bintree._ensure_dir(os.path.dirname(binpkg_tmpfile))
73
74 self._binpkg_tmpfile = binpkg_tmpfile
75
76 diff --git a/lib/_emerge/actions.py b/lib/_emerge/actions.py
77 index 063f5d4a0..a4ecfe43d 100644
78 --- a/lib/_emerge/actions.py
79 +++ b/lib/_emerge/actions.py
80 @@ -2623,7 +2623,7 @@ def ionice(settings):
81 if not ionice_cmd:
82 return
83
84 - variables = {"PID" : str(os.getpid())}
85 + variables = {"PID" : str(portage.getpid())}
86 cmd = [varexpand(x, mydict=variables) for x in ionice_cmd]
87
88 try:
89
90 diff --git a/lib/portage/_emirrordist/FetchTask.py b/lib/portage/_emirrordist/FetchTask.py
91 index 457ca2ac6..41f96b962 100644
92 --- a/lib/portage/_emirrordist/FetchTask.py
93 +++ b/lib/portage/_emirrordist/FetchTask.py
94 @@ -415,7 +415,7 @@ class FetchTask(CompositeTask):
95 self._fetch_tmp_dir_info = 'distfiles'
96 distdir = self.config.options.distfiles
97
98 - tmp_basename = self.distfile + '._emirrordist_fetch_.%s' % os.getpid()
99 + tmp_basename = self.distfile + '._emirrordist_fetch_.%s' % portage.getpid()
100
101 variables = {
102 "DISTDIR": distdir,
103 @@ -622,7 +622,7 @@ class FetchTask(CompositeTask):
104
105 head, tail = os.path.split(dest)
106 hardlink_tmp = os.path.join(head, ".%s._mirrordist_hardlink_.%s" % \
107 - (tail, os.getpid()))
108 + (tail, portage.getpid()))
109
110 try:
111 try:
112
113 diff --git a/lib/portage/cache/metadata.py b/lib/portage/cache/metadata.py
114 index db81b8ba1..bd1b70fa0 100644
115 --- a/lib/portage/cache/metadata.py
116 +++ b/lib/portage/cache/metadata.py
117 @@ -6,6 +6,8 @@ import errno
118 import re
119 import stat
120 from operator import attrgetter
121 +
122 +import portage
123 from portage import os
124 from portage import _encodings
125 from portage import _unicode_encode
126 @@ -122,7 +124,7 @@ class database(flat_hash.database):
127
128 s = cpv.rfind("/")
129 fp = os.path.join(self.location,cpv[:s],
130 - ".update.%i.%s" % (os.getpid(), cpv[s+1:]))
131 + ".update.%i.%s" % (portage.getpid(), cpv[s+1:]))
132 try:
133 myf = open(_unicode_encode(fp,
134 encoding=_encodings['fs'], errors='strict'), 'wb')
135
136 diff --git a/lib/portage/elog/mod_mail_summary.py b/lib/portage/elog/mod_mail_summary.py
137 index ac260880e..789f55f4d 100644
138 --- a/lib/portage/elog/mod_mail_summary.py
139 +++ b/lib/portage/elog/mod_mail_summary.py
140 @@ -6,7 +6,6 @@ import portage
141 from portage.exception import AlarmSignal, PortageException
142 from portage.localization import _
143 from portage.util import writemsg
144 -from portage import os
145 from portage import _encodings
146 from portage import _unicode_decode
147
148 @@ -22,7 +21,7 @@ def process(mysettings, key, logentries, fulltext):
149 time.strftime("%Y%m%d-%H%M%S %Z", time.localtime(time.time())),
150 encoding=_encodings['content'], errors='replace')
151 header = _(">>> Messages generated for package %(pkg)s by process %(pid)d on %(time)s:\n\n") % \
152 - {"pkg": key, "pid": os.getpid(), "time": time_str}
153 + {"pkg": key, "pid": portage.getpid(), "time": time_str}
154 config_root = mysettings["PORTAGE_CONFIGROOT"]
155
156 # Copy needed variables from the config instance,
157 @@ -66,7 +65,7 @@ def _finalize(mysettings, items):
158 mysubject = mysubject.replace("${HOST}", socket.getfqdn())
159
160 mybody = _("elog messages for the following packages generated by "
161 - "process %(pid)d on host %(host)s:\n") % {"pid": os.getpid(), "host": socket.getfqdn()}
162 + "process %(pid)d on host %(host)s:\n") % {"pid": portage.getpid(), "host": socket.getfqdn()}
163 for key in items:
164 mybody += "- %s\n" % key
165
166
167 diff --git a/lib/portage/elog/mod_save_summary.py b/lib/portage/elog/mod_save_summary.py
168 index 946a1ad4c..6e24b608f 100644
169 --- a/lib/portage/elog/mod_save_summary.py
170 +++ b/lib/portage/elog/mod_save_summary.py
171 @@ -79,7 +79,7 @@ def process(mysettings, key, logentries, fulltext):
172 encoding=_encodings['content'], errors='replace')
173 elogfile.write(_(">>> Messages generated by process "
174 "%(pid)d on %(time)s for package %(pkg)s:\n\n") %
175 - {"pid": os.getpid(), "time": time_str, "pkg": key})
176 + {"pid": portage.getpid(), "time": time_str, "pkg": key})
177 elogfile.write(_unicode_decode(fulltext))
178 elogfile.write("\n")
179 elogfile.close()
180
181 diff --git a/lib/portage/locks.py b/lib/portage/locks.py
182 index 701093024..1073343be 100644
183 --- a/lib/portage/locks.py
184 +++ b/lib/portage/locks.py
185 @@ -501,7 +501,7 @@ def unlockfile(mytuple):
186 def hardlock_name(path):
187 base, tail = os.path.split(path)
188 return os.path.join(base, ".%s.hardlock-%s-%s" %
189 - (tail, portage._decode_argv([os.uname()[1]])[0], os.getpid()))
190 + (tail, portage._decode_argv([os.uname()[1]])[0], portage.getpid()))
191
192 def hardlink_is_mine(link, lock):
193 try:
194
195 diff --git a/lib/portage/package/ebuild/doebuild.py b/lib/portage/package/ebuild/doebuild.py
196 index 7bb942966..3b1991b28 100644
197 --- a/lib/portage/package/ebuild/doebuild.py
198 +++ b/lib/portage/package/ebuild/doebuild.py
199 @@ -1178,7 +1178,7 @@ def doebuild(myebuild, mydo, _unused=DeprecationWarning, settings=None, debug=0,
200 bintree = portage.db[mysettings['EROOT']]['bintree']
201 mysettings["PORTAGE_BINPKG_TMPFILE"] = \
202 bintree.getname(mysettings.mycpv) + \
203 - ".%s" % (os.getpid(),)
204 + ".%s" % (portage.getpid(),)
205 bintree._ensure_dir(os.path.dirname(
206 mysettings["PORTAGE_BINPKG_TMPFILE"]))
207 else:
208
209 diff --git a/lib/portage/process.py b/lib/portage/process.py
210 index 8d4cf164e..48548bacc 100644
211 --- a/lib/portage/process.py
212 +++ b/lib/portage/process.py
213 @@ -79,11 +79,11 @@ if _fd_dir is not None:
214 raise
215 return range(max_fd_limit)
216
217 -elif os.path.isdir("/proc/%s/fd" % os.getpid()):
218 +elif os.path.isdir("/proc/%s/fd" % portage.getpid()):
219 # In order for this function to work in forked subprocesses,
220 # os.getpid() must be called from inside the function.
221 def get_open_fds():
222 - return (int(fd) for fd in os.listdir("/proc/%s/fd" % os.getpid())
223 + return (int(fd) for fd in os.listdir("/proc/%s/fd" % portage.getpid())
224 if fd.isdigit())
225
226 else:
227 @@ -363,12 +363,13 @@ def spawn(mycommand, env=None, opt_name=None, fd_pipes=None, returnpid=False,
228 # fork, so that the result is cached in the main process.
229 bool(groups)
230
231 - parent_pid = os.getpid()
232 + parent_pid = portage.getpid()
233 pid = None
234 try:
235 pid = os.fork()
236
237 if pid == 0:
238 + portage._ForkWatcher.hook(portage._ForkWatcher)
239 try:
240 _exec(binary, mycommand, opt_name, fd_pipes,
241 env, gid, groups, uid, umask, cwd, pre_exec, close_fds,
242 @@ -386,7 +387,9 @@ def spawn(mycommand, env=None, opt_name=None, fd_pipes=None, returnpid=False,
243 sys.stderr.flush()
244
245 finally:
246 - if pid == 0 or (pid is None and os.getpid() != parent_pid):
247 + # Don't used portage.getpid() here, due to a race with the above
248 + # portage._ForkWatcher cache update.
249 + if pid == 0 or (pid is None and _os.getpid() != parent_pid):
250 # Call os._exit() from a finally block in order
251 # to suppress any finally blocks from earlier
252 # in the call stack (see bug #345289). This
253 @@ -603,7 +606,7 @@ def _exec(binary, mycommand, opt_name, fd_pipes,
254 # it is done before we start forking children
255 if cgroup:
256 with open(os.path.join(cgroup, 'cgroup.procs'), 'a') as f:
257 - f.write('%d\n' % os.getpid())
258 + f.write('%d\n' % portage.getpid())
259
260 # Unshare (while still uid==0)
261 if unshare_net or unshare_ipc or unshare_mount or unshare_pid:
262 @@ -640,6 +643,9 @@ def _exec(binary, mycommand, opt_name, fd_pipes,
263 if unshare_pid:
264 main_child_pid = os.fork()
265 if main_child_pid == 0:
266 + # The portage.getpid() cache may need to be updated here,
267 + # in case the pre_exec function invokes portage APIs.
268 + portage._ForkWatcher.hook(portage._ForkWatcher)
269 # pid namespace requires us to become init
270 binary, myargs = portage._python_interpreter, [
271 portage._python_interpreter,
272
273 diff --git a/lib/portage/tests/locks/test_lock_nonblock.py b/lib/portage/tests/locks/test_lock_nonblock.py
274 index 02ba16ad9..3448b84f6 100644
275 --- a/lib/portage/tests/locks/test_lock_nonblock.py
276 +++ b/lib/portage/tests/locks/test_lock_nonblock.py
277 @@ -19,6 +19,7 @@ class LockNonblockTestCase(TestCase):
278 lock1 = portage.locks.lockfile(path)
279 pid = os.fork()
280 if pid == 0:
281 + portage._ForkWatcher.hook(portage._ForkWatcher)
282 portage.locks._close_fds()
283 # Disable close_fds since we don't exec
284 # (see _setup_pipes docstring).
285
286 diff --git a/lib/portage/tests/util/futures/asyncio/test_wakeup_fd_sigchld.py b/lib/portage/tests/util/futures/asyncio/test_wakeup_fd_sigchld.py
287 index e5b104e0f..c37a6338b 100644
288 --- a/lib/portage/tests/util/futures/asyncio/test_wakeup_fd_sigchld.py
289 +++ b/lib/portage/tests/util/futures/asyncio/test_wakeup_fd_sigchld.py
290 @@ -39,7 +39,7 @@ proc = loop.run_until_complete(asyncio.create_subprocess_exec('sleep', '0', loop
291 loop.run_until_complete(proc.wait())
292
293 for i in range(8192):
294 - os.kill(os.getpid(), signal.SIGCHLD)
295 + os.kill(portage.getpid(), signal.SIGCHLD)
296
297 # Verify that the child watcher still works correctly
298 # (this will hang if it doesn't).
299
300 diff --git a/lib/portage/util/__init__.py b/lib/portage/util/__init__.py
301 index c14c15fe8..0412b2b59 100644
302 --- a/lib/portage/util/__init__.py
303 +++ b/lib/portage/util/__init__.py
304 @@ -1266,7 +1266,7 @@ class atomic_ofstream(ObjectProxy):
305 if follow_links:
306 canonical_path = os.path.realpath(filename)
307 object.__setattr__(self, '_real_name', canonical_path)
308 - tmp_name = "%s.%i" % (canonical_path, os.getpid())
309 + tmp_name = "%s.%i" % (canonical_path, portage.getpid())
310 try:
311 object.__setattr__(self, '_file',
312 open_func(_unicode_encode(tmp_name,
313 @@ -1281,7 +1281,7 @@ class atomic_ofstream(ObjectProxy):
314 # new error if necessary.
315
316 object.__setattr__(self, '_real_name', filename)
317 - tmp_name = "%s.%i" % (filename, os.getpid())
318 + tmp_name = "%s.%i" % (filename, portage.getpid())
319 object.__setattr__(self, '_file',
320 open_func(_unicode_encode(tmp_name,
321 encoding=_encodings['fs'], errors='strict'),
322
323 diff --git a/lib/portage/util/_eventloop/EventLoop.py b/lib/portage/util/_eventloop/EventLoop.py
324 index 94e637853..ff2b73255 100644
325 --- a/lib/portage/util/_eventloop/EventLoop.py
326 +++ b/lib/portage/util/_eventloop/EventLoop.py
327 @@ -188,7 +188,7 @@ class EventLoop:
328 self._sigchld_read = None
329 self._sigchld_write = None
330 self._sigchld_src_id = None
331 - self._pid = os.getpid()
332 + self._pid = portage.getpid()
333 self._asyncio_wrapper = _PortageEventLoop(loop=self)
334 self._asyncio_child_watcher = _PortageChildWatcher(self)
335
336 @@ -431,7 +431,7 @@ class EventLoop:
337 # If this signal handler was not installed by the
338 # current process then the signal doesn't belong to
339 # this EventLoop instance.
340 - if os.getpid() == self._pid:
341 + if portage.getpid() == self._pid:
342 os.write(self._sigchld_write, b'\0')
343
344 def _sigchld_io_cb(self, fd, events):
345 @@ -1026,7 +1026,7 @@ class EventLoop:
346 log_lines.append('{}: {}'.format(key, value))
347
348 logging.error('\n'.join(log_lines), exc_info=exc_info)
349 - os.kill(os.getpid(), signal.SIGTERM)
350 + os.kill(portage.getpid(), signal.SIGTERM)
351
352 def call_exception_handler(self, context):
353 """
354
355 diff --git a/lib/portage/util/_eventloop/asyncio_event_loop.py b/lib/portage/util/_eventloop/asyncio_event_loop.py
356 index 605e7243b..836f1c30a 100644
357 --- a/lib/portage/util/_eventloop/asyncio_event_loop.py
358 +++ b/lib/portage/util/_eventloop/asyncio_event_loop.py
359 @@ -69,7 +69,7 @@ class AsyncioEventLoop(_AbstractEventLoop):
360 # in order to ensure that emerge exits immediately (though
361 # uncleanly).
362 signal.signal(signal.SIGTERM, signal.SIG_DFL)
363 - os.kill(os.getpid(), signal.SIGTERM)
364 + os.kill(portage.getpid(), signal.SIGTERM)
365
366 def _create_future(self):
367 """
368 @@ -117,7 +117,7 @@ class AsyncioEventLoop(_AbstractEventLoop):
369 self._wakeup_fd = -1
370 # Account for any signals that may have arrived between
371 # set_wakeup_fd calls.
372 - os.kill(os.getpid(), signal.SIGCHLD)
373 + os.kill(portage.getpid(), signal.SIGCHLD)
374 try:
375 return self._loop.run_until_complete(future)
376 finally:
377
378 diff --git a/lib/portage/util/_eventloop/global_event_loop.py b/lib/portage/util/_eventloop/global_event_loop.py
379 index 1db958d2e..21a1d1970 100644
380 --- a/lib/portage/util/_eventloop/global_event_loop.py
381 +++ b/lib/portage/util/_eventloop/global_event_loop.py
382 @@ -1,13 +1,12 @@
383 # Copyright 2012-2020 Gentoo Authors
384 # Distributed under the terms of the GNU General Public License v2
385
386 -import os
387 -
388 +import portage
389 from .EventLoop import EventLoop
390 from portage.util._eventloop.asyncio_event_loop import AsyncioEventLoop
391
392
393 -_MAIN_PID = os.getpid()
394 +_MAIN_PID = portage.getpid()
395 _instances = {}
396
397
398 @@ -17,7 +16,7 @@ def global_event_loop():
399 belongs exclusively to the current process.
400 """
401
402 - pid = os.getpid()
403 + pid = portage.getpid()
404 instance = _instances.get(pid)
405 if instance is not None:
406 return instance
407
408 diff --git a/lib/portage/util/locale.py b/lib/portage/util/locale.py
409 index 99c8f7ae7..58c687139 100644
410 --- a/lib/portage/util/locale.py
411 +++ b/lib/portage/util/locale.py
412 @@ -102,6 +102,7 @@ def check_locale(silent=False, env=None):
413
414 pid = os.fork()
415 if pid == 0:
416 + portage._ForkWatcher.hook(portage._ForkWatcher)
417 try:
418 if env is not None:
419 try:
420
421 diff --git a/lib/portage/util/movefile.py b/lib/portage/util/movefile.py
422 index 4f8054f29..a251d369d 100644
423 --- a/lib/portage/util/movefile.py
424 +++ b/lib/portage/util/movefile.py
425 @@ -209,7 +209,7 @@ def movefile(src, dest, newmtime=None, sstat=None, mysettings=None,
426 if hardlink_candidates:
427 head, tail = os.path.split(dest)
428 hardlink_tmp = os.path.join(head, ".%s._portage_merge_.%s" % \
429 - (tail, os.getpid()))
430 + (tail, portage.getpid()))
431 try:
432 os.unlink(hardlink_tmp)
433 except OSError as e:
434
435 diff --git a/lib/portage/util/socks5.py b/lib/portage/util/socks5.py
436 index 9f22c1dbe..a76d1a741 100644
437 --- a/lib/portage/util/socks5.py
438 +++ b/lib/portage/util/socks5.py
439 @@ -42,7 +42,7 @@ class ProxyManager:
440 portage.util.ensure_dirs(tmpdir, **ensure_dirs_kwargs)
441
442 self.socket_path = os.path.join(tmpdir,
443 - '.portage.%d.net.sock' % os.getpid())
444 + '.portage.%d.net.sock' % portage.getpid())
445 server_bin = os.path.join(settings['PORTAGE_BIN_PATH'], 'socks5-server.py')
446 spawn_kwargs = {}
447 # The portage_uid check solves EPERM failures in Travis CI.
448
449 diff --git a/lib/portage/xpak.py b/lib/portage/xpak.py
450 index 2a4bcda21..9063c4c56 100644
451 --- a/lib/portage/xpak.py
452 +++ b/lib/portage/xpak.py
453 @@ -325,7 +325,7 @@ class tbz2:
454 self.scan() # Don't care about condition... We'll rewrite the data anyway.
455
456 if break_hardlinks and self.filestat and self.filestat.st_nlink > 1:
457 - tmp_fname = "%s.%d" % (self.file, os.getpid())
458 + tmp_fname = "%s.%d" % (self.file, portage.getpid())
459 copyfile(self.file, tmp_fname)
460 try:
461 portage.util.apply_stat_permissions(self.file, self.filestat)