1 |
For python3.4 and later, in the main process, replace portage's |
2 |
internal event loop with the standard library's asyncio event |
3 |
loop. Continue to use portage's internal event loop in subprocesses, |
4 |
since asyncio's event loop is not guaranteed to work well in |
5 |
subprocesses (see upstream python issues 22087 and 29703). |
6 |
|
7 |
An _AsyncioEventLoopPolicy class, derived from _PortageEventLoopPolicy, |
8 |
is needed for some unit tests that modify asyncio's event loop policy. |
9 |
This policy is not needed for anything other than unit testing. Portage |
10 |
uses asyncio's default event loop policy, and API consumers are free to |
11 |
use any desired event loop policy. |
12 |
|
13 |
Portage's asynchronous functions that accept a 'loop' parameter will |
14 |
work with any compatible asyncio.AbstractEventLoop implementation, since |
15 |
an internal _wrap_loop function automatically adapts the loop for |
16 |
internal use. |
17 |
|
18 |
Bug: https://bugs.gentoo.org/654390 |
19 |
--- |
20 |
pym/portage/util/_async/SchedulerInterface.py | 3 + |
21 |
pym/portage/util/_eventloop/asyncio_event_loop.py | 77 +++++++++++++++++++++++ |
22 |
pym/portage/util/_eventloop/global_event_loop.py | 5 +- |
23 |
pym/portage/util/futures/_asyncio/__init__.py | 16 +++++ |
24 |
pym/portage/util/futures/unix_events.py | 43 ++++++++++++- |
25 |
5 files changed, 142 insertions(+), 2 deletions(-) |
26 |
create mode 100644 pym/portage/util/_eventloop/asyncio_event_loop.py |
27 |
|
28 |
diff --git a/pym/portage/util/_async/SchedulerInterface.py b/pym/portage/util/_async/SchedulerInterface.py |
29 |
index f1a3e9b0b..ec6417da1 100644 |
30 |
--- a/pym/portage/util/_async/SchedulerInterface.py |
31 |
+++ b/pym/portage/util/_async/SchedulerInterface.py |
32 |
@@ -33,6 +33,9 @@ class SchedulerInterface(SlotObject): |
33 |
"time", |
34 |
|
35 |
"_asyncio_child_watcher", |
36 |
+ # This attribute it used by _wrap_loop to detect if the |
37 |
+ # loop already has a suitable wrapper. |
38 |
+ "_asyncio_wrapper", |
39 |
) |
40 |
|
41 |
__slots__ = _event_loop_attrs + ("_event_loop", "_is_background") |
42 |
diff --git a/pym/portage/util/_eventloop/asyncio_event_loop.py b/pym/portage/util/_eventloop/asyncio_event_loop.py |
43 |
new file mode 100644 |
44 |
index 000000000..b365939b0 |
45 |
--- /dev/null |
46 |
+++ b/pym/portage/util/_eventloop/asyncio_event_loop.py |
47 |
@@ -0,0 +1,77 @@ |
48 |
+# Copyright 2018 Gentoo Foundation |
49 |
+# Distributed under the terms of the GNU General Public License v2 |
50 |
+ |
51 |
+try: |
52 |
+ import asyncio as _real_asyncio |
53 |
+ from asyncio.events import AbstractEventLoop as _AbstractEventLoop |
54 |
+except ImportError: |
55 |
+ # Allow ImportModulesTestCase to succeed. |
56 |
+ _real_asyncio = None |
57 |
+ _AbstractEventLoop = object |
58 |
+ |
59 |
+ |
60 |
+class AsyncioEventLoop(_AbstractEventLoop): |
61 |
+ """ |
62 |
+ Implementation of asyncio.AbstractEventLoop which wraps asyncio's |
63 |
+ event loop and is minimally compatible with _PortageEventLoop. |
64 |
+ """ |
65 |
+ |
66 |
+ # Use portage's internal event loop in subprocesses, as a workaround |
67 |
+ # for https://bugs.python.org/issue22087, and also |
68 |
+ # https://bugs.python.org/issue29703 which affects pypy3-5.10.1. |
69 |
+ supports_multiprocessing = False |
70 |
+ |
71 |
+ def __init__(self, loop=None): |
72 |
+ loop = loop or _real_asyncio.get_event_loop() |
73 |
+ self._loop = loop |
74 |
+ self.run_until_complete = loop.run_until_complete |
75 |
+ self.call_soon = loop.call_soon |
76 |
+ self.call_soon_threadsafe = loop.call_soon_threadsafe |
77 |
+ self.call_later = loop.call_later |
78 |
+ self.call_at = loop.call_at |
79 |
+ self.is_running = loop.is_running |
80 |
+ self.is_closed = loop.is_closed |
81 |
+ self.close = loop.close |
82 |
+ self.create_future = (loop.create_future |
83 |
+ if hasattr(loop, 'create_future') else self._create_future) |
84 |
+ self.create_task = loop.create_task |
85 |
+ self.add_reader = loop.add_reader |
86 |
+ self.remove_reader = loop.remove_reader |
87 |
+ self.add_writer = loop.add_writer |
88 |
+ self.remove_writer = loop.remove_writer |
89 |
+ self.run_in_executor = loop.run_in_executor |
90 |
+ self.time = loop.time |
91 |
+ self.default_exception_handler = loop.default_exception_handler |
92 |
+ self.call_exception_handler = loop.call_exception_handler |
93 |
+ self.set_debug = loop.set_debug |
94 |
+ self.get_debug = loop.get_debug |
95 |
+ |
96 |
+ def _create_future(self): |
97 |
+ """ |
98 |
+ Provide AbstractEventLoop.create_future() for python3.4. |
99 |
+ """ |
100 |
+ return _real_asyncio.Future(loop=self._loop) |
101 |
+ |
102 |
+ @property |
103 |
+ def _asyncio_child_watcher(self): |
104 |
+ """ |
105 |
+ Portage internals use this as a layer of indirection for |
106 |
+ asyncio.get_child_watcher(), in order to support versions of |
107 |
+ python where asyncio is not available. |
108 |
+ |
109 |
+ @rtype: asyncio.AbstractChildWatcher |
110 |
+ @return: the internal event loop's AbstractChildWatcher interface |
111 |
+ """ |
112 |
+ return _real_asyncio.get_child_watcher() |
113 |
+ |
114 |
+ @property |
115 |
+ def _asyncio_wrapper(self): |
116 |
+ """ |
117 |
+ Portage internals use this as a layer of indirection in cases |
118 |
+ where a wrapper around an asyncio.AbstractEventLoop implementation |
119 |
+ is needed for purposes of compatiblity. |
120 |
+ |
121 |
+ @rtype: asyncio.AbstractEventLoop |
122 |
+ @return: the internal event loop's AbstractEventLoop interface |
123 |
+ """ |
124 |
+ return self |
125 |
diff --git a/pym/portage/util/_eventloop/global_event_loop.py b/pym/portage/util/_eventloop/global_event_loop.py |
126 |
index a3ee9248d..2f6371dc1 100644 |
127 |
--- a/pym/portage/util/_eventloop/global_event_loop.py |
128 |
+++ b/pym/portage/util/_eventloop/global_event_loop.py |
129 |
@@ -2,10 +2,13 @@ |
130 |
# Distributed under the terms of the GNU General Public License v2 |
131 |
|
132 |
import os |
133 |
+import sys |
134 |
|
135 |
from .EventLoop import EventLoop |
136 |
+from portage.util._eventloop.asyncio_event_loop import AsyncioEventLoop |
137 |
|
138 |
-_default_constructor = EventLoop |
139 |
+_asyncio_enabled = sys.version_info >= (3, 4) |
140 |
+_default_constructor = AsyncioEventLoop if _asyncio_enabled else EventLoop |
141 |
|
142 |
# If _default_constructor doesn't support multiprocessing, |
143 |
# then _multiprocessing_constructor is used in subprocesses. |
144 |
diff --git a/pym/portage/util/futures/_asyncio/__init__.py b/pym/portage/util/futures/_asyncio/__init__.py |
145 |
index e62de7a69..401a29efa 100644 |
146 |
--- a/pym/portage/util/futures/_asyncio/__init__.py |
147 |
+++ b/pym/portage/util/futures/_asyncio/__init__.py |
148 |
@@ -21,6 +21,11 @@ __all__ = ( |
149 |
) |
150 |
|
151 |
try: |
152 |
+ import asyncio as _real_asyncio |
153 |
+except ImportError: |
154 |
+ _real_asyncio = None |
155 |
+ |
156 |
+try: |
157 |
import threading |
158 |
except ImportError: |
159 |
import dummy_threading as threading |
160 |
@@ -28,6 +33,7 @@ except ImportError: |
161 |
import portage |
162 |
portage.proxy.lazyimport.lazyimport(globals(), |
163 |
'portage.util.futures.unix_events:DefaultEventLoopPolicy', |
164 |
+ 'portage.util._eventloop.asyncio_event_loop:AsyncioEventLoop@_AsyncioEventLoop', |
165 |
) |
166 |
from portage.util.futures.futures import ( |
167 |
CancelledError, |
168 |
@@ -162,3 +168,13 @@ def _wrap_loop(loop=None): |
169 |
@return: event loop |
170 |
""" |
171 |
return loop or get_event_loop() |
172 |
+ |
173 |
+ |
174 |
+if _real_asyncio is not None: |
175 |
+ get_event_loop_policy = _real_asyncio.get_event_loop_policy |
176 |
+ set_event_loop_policy = _real_asyncio.set_event_loop_policy |
177 |
+ |
178 |
+ def _wrap_loop(loop=None): |
179 |
+ loop = loop or get_event_loop() |
180 |
+ return (loop if hasattr(loop, '_asyncio_wrapper') |
181 |
+ else _AsyncioEventLoop(loop=loop)) |
182 |
diff --git a/pym/portage/util/futures/unix_events.py b/pym/portage/util/futures/unix_events.py |
183 |
index 00f522b61..ce520db00 100644 |
184 |
--- a/pym/portage/util/futures/unix_events.py |
185 |
+++ b/pym/portage/util/futures/unix_events.py |
186 |
@@ -7,6 +7,7 @@ __all__ = ( |
187 |
) |
188 |
|
189 |
try: |
190 |
+ import asyncio as _real_asyncio |
191 |
from asyncio.base_subprocess import BaseSubprocessTransport as _BaseSubprocessTransport |
192 |
from asyncio.unix_events import AbstractChildWatcher as _AbstractChildWatcher |
193 |
from asyncio.transports import ( |
194 |
@@ -14,6 +15,7 @@ try: |
195 |
WriteTransport as _WriteTransport, |
196 |
) |
197 |
except ImportError: |
198 |
+ _real_asyncio = None |
199 |
_AbstractChildWatcher = object |
200 |
_BaseSubprocessTransport = object |
201 |
_ReadTransport = object |
202 |
@@ -30,6 +32,7 @@ import subprocess |
203 |
import sys |
204 |
|
205 |
from portage.util._eventloop.global_event_loop import ( |
206 |
+ _asyncio_enabled, |
207 |
global_event_loop as _global_event_loop, |
208 |
) |
209 |
from portage.util.futures import ( |
210 |
@@ -678,4 +681,42 @@ class _PortageEventLoopPolicy(events.AbstractEventLoopPolicy): |
211 |
return _global_event_loop()._asyncio_child_watcher |
212 |
|
213 |
|
214 |
-DefaultEventLoopPolicy = _PortageEventLoopPolicy |
215 |
+class _AsyncioEventLoopPolicy(_PortageEventLoopPolicy): |
216 |
+ """ |
217 |
+ Implementation of asyncio.AbstractEventLoopPolicy based on asyncio's |
218 |
+ event loop. This supports running event loops in forks, |
219 |
+ which is not supported by the default asyncio event loop policy, |
220 |
+ see https://bugs.python.org/issue22087 and also |
221 |
+ https://bugs.python.org/issue29703 which affects pypy3-5.10.1. |
222 |
+ """ |
223 |
+ _MAIN_PID = os.getpid() |
224 |
+ |
225 |
+ def __init__(self): |
226 |
+ super(_AsyncioEventLoopPolicy, self).__init__() |
227 |
+ self._default_policy = _real_asyncio.DefaultEventLoopPolicy() |
228 |
+ |
229 |
+ def get_event_loop(self): |
230 |
+ """ |
231 |
+ Get the event loop for the current context. |
232 |
+ |
233 |
+ Returns an event loop object implementing the AbstractEventLoop |
234 |
+ interface. |
235 |
+ |
236 |
+ @rtype: asyncio.AbstractEventLoop (or compatible) |
237 |
+ @return: the current event loop policy |
238 |
+ """ |
239 |
+ if os.getpid() == self._MAIN_PID: |
240 |
+ return self._default_policy.get_event_loop() |
241 |
+ else: |
242 |
+ return super(_AsyncioEventLoopPolicy, self).get_event_loop() |
243 |
+ |
244 |
+ def get_child_watcher(self): |
245 |
+ """Get the watcher for child processes.""" |
246 |
+ if os.getpid() == self._MAIN_PID: |
247 |
+ return self._default_policy.get_child_watcher() |
248 |
+ else: |
249 |
+ return super(_AsyncioEventLoopPolicy, self).get_child_watcher() |
250 |
+ |
251 |
+ |
252 |
+DefaultEventLoopPolicy = (_AsyncioEventLoopPolicy if _asyncio_enabled |
253 |
+ else _PortageEventLoopPolicy) |
254 |
-- |
255 |
2.13.6 |