1 |
commit: 035582f0e31c071606635aac9cc4ba4b411612e7 |
2 |
Author: Zac Medico <zmedico <AT> gentoo <DOT> org> |
3 |
AuthorDate: Mon Jan 14 08:11:57 2019 +0000 |
4 |
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org> |
5 |
CommitDate: Wed Jan 16 07:48:59 2019 +0000 |
6 |
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=035582f0 |
7 |
|
8 |
tests: add unit test for portage.util.socks5 (FEATURES=network-sandbox-proxy) |
9 |
|
10 |
Bug: https://bugs.gentoo.org/604474 |
11 |
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org> |
12 |
|
13 |
lib/portage/tests/util/test_socks5.py | 211 ++++++++++++++++++++++++++++++++++ |
14 |
lib/portage/util/socks5.py | 48 +++++++- |
15 |
2 files changed, 256 insertions(+), 3 deletions(-) |
16 |
|
17 |
diff --git a/lib/portage/tests/util/test_socks5.py b/lib/portage/tests/util/test_socks5.py |
18 |
new file mode 100644 |
19 |
index 000000000..5db85b0a6 |
20 |
--- /dev/null |
21 |
+++ b/lib/portage/tests/util/test_socks5.py |
22 |
@@ -0,0 +1,211 @@ |
23 |
+# Copyright 2019 Gentoo Authors |
24 |
+# Distributed under the terms of the GNU General Public License v2 |
25 |
+ |
26 |
+import functools |
27 |
+import platform |
28 |
+import shutil |
29 |
+import socket |
30 |
+import struct |
31 |
+import sys |
32 |
+import tempfile |
33 |
+import time |
34 |
+ |
35 |
+import portage |
36 |
+from portage.tests import TestCase |
37 |
+from portage.util._eventloop.global_event_loop import global_event_loop |
38 |
+from portage.util import socks5 |
39 |
+from portage.const import PORTAGE_BIN_PATH |
40 |
+ |
41 |
+try: |
42 |
+ from http.server import BaseHTTPRequestHandler, HTTPServer |
43 |
+except ImportError: |
44 |
+ from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer |
45 |
+ |
46 |
+try: |
47 |
+ from urllib.request import urlopen |
48 |
+except ImportError: |
49 |
+ from urllib import urlopen |
50 |
+ |
51 |
+ |
52 |
+class _Handler(BaseHTTPRequestHandler): |
53 |
+ |
54 |
+ def __init__(self, content, *args, **kwargs): |
55 |
+ self.content = content |
56 |
+ BaseHTTPRequestHandler.__init__(self, *args, **kwargs) |
57 |
+ |
58 |
+ def do_GET(self): |
59 |
+ doc = self.send_head() |
60 |
+ if doc is not None: |
61 |
+ self.wfile.write(doc) |
62 |
+ |
63 |
+ def do_HEAD(self): |
64 |
+ self.send_head() |
65 |
+ |
66 |
+ def send_head(self): |
67 |
+ doc = self.content.get(self.path) |
68 |
+ if doc is None: |
69 |
+ self.send_error(404, "File not found") |
70 |
+ return None |
71 |
+ |
72 |
+ self.send_response(200) |
73 |
+ self.send_header("Content-type", "text/plain") |
74 |
+ self.send_header("Content-Length", len(doc)) |
75 |
+ self.send_header("Last-Modified", self.date_time_string(time.time())) |
76 |
+ self.end_headers() |
77 |
+ return doc |
78 |
+ |
79 |
+ def log_message(self, fmt, *args): |
80 |
+ pass |
81 |
+ |
82 |
+ |
83 |
+class AsyncHTTPServer(object): |
84 |
+ def __init__(self, host, content, loop): |
85 |
+ self._host = host |
86 |
+ self._content = content |
87 |
+ self._loop = loop |
88 |
+ self.server_port = None |
89 |
+ self._httpd = None |
90 |
+ |
91 |
+ def __enter__(self): |
92 |
+ httpd = self._httpd = HTTPServer((self._host, 0), functools.partial(_Handler, self._content)) |
93 |
+ self.server_port = httpd.server_port |
94 |
+ self._loop.add_reader(httpd.socket.fileno(), self._httpd._handle_request_noblock) |
95 |
+ return self |
96 |
+ |
97 |
+ def __exit__(self, exc_type, exc_value, exc_traceback): |
98 |
+ if self._httpd is not None: |
99 |
+ self._loop.remove_reader(self._httpd.socket.fileno()) |
100 |
+ self._httpd.socket.close() |
101 |
+ self._httpd = None |
102 |
+ |
103 |
+ |
104 |
+class AsyncHTTPServerTestCase(TestCase): |
105 |
+ |
106 |
+ @staticmethod |
107 |
+ def _fetch_directly(host, port, path): |
108 |
+ # NOTE: python2.7 does not have context manager support here |
109 |
+ try: |
110 |
+ f = urlopen('http://{host}:{port}{path}'.format( # nosec |
111 |
+ host=host, port=port, path=path)) |
112 |
+ return f.read() |
113 |
+ finally: |
114 |
+ if f is not None: |
115 |
+ f.close() |
116 |
+ |
117 |
+ def test_http_server(self): |
118 |
+ host = '127.0.0.1' |
119 |
+ content = b'Hello World!\n' |
120 |
+ path = '/index.html' |
121 |
+ loop = global_event_loop() |
122 |
+ for i in range(2): |
123 |
+ with AsyncHTTPServer(host, {path: content}, loop) as server: |
124 |
+ for j in range(2): |
125 |
+ result = loop.run_until_complete(loop.run_in_executor(None, |
126 |
+ self._fetch_directly, host, server.server_port, path)) |
127 |
+ self.assertEqual(result, content) |
128 |
+ |
129 |
+ |
130 |
+class _socket_file_wrapper(portage.proxy.objectproxy.ObjectProxy): |
131 |
+ """ |
132 |
+ A file-like object that wraps a socket and closes the socket when |
133 |
+ closed. Since python2.7 does not support socket.detach(), this is a |
134 |
+ convenient way to have a file attached to a socket that closes |
135 |
+ automatically (without resource warnings about unclosed sockets). |
136 |
+ """ |
137 |
+ |
138 |
+ __slots__ = ('_file', '_socket') |
139 |
+ |
140 |
+ def __init__(self, socket, f): |
141 |
+ object.__setattr__(self, '_socket', socket) |
142 |
+ object.__setattr__(self, '_file', f) |
143 |
+ |
144 |
+ def _get_target(self): |
145 |
+ return object.__getattribute__(self, '_file') |
146 |
+ |
147 |
+ def __getattribute__(self, attr): |
148 |
+ if attr == 'close': |
149 |
+ return object.__getattribute__(self, 'close') |
150 |
+ return super(_socket_file_wrapper, self).__getattribute__(attr) |
151 |
+ |
152 |
+ def __enter__(self): |
153 |
+ return self |
154 |
+ |
155 |
+ def close(self): |
156 |
+ object.__getattribute__(self, '_file').close() |
157 |
+ object.__getattribute__(self, '_socket').close() |
158 |
+ |
159 |
+ def __exit__(self, exc_type, exc_value, traceback): |
160 |
+ self.close() |
161 |
+ |
162 |
+ |
163 |
+def socks5_http_get_ipv4(proxy, host, port, path): |
164 |
+ """ |
165 |
+ Open http GET request via socks5 proxy listening on a unix socket, |
166 |
+ and return a file to read the response body from. |
167 |
+ """ |
168 |
+ s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) |
169 |
+ f = _socket_file_wrapper(s, s.makefile('rb', 1024)) |
170 |
+ try: |
171 |
+ s.connect(proxy) |
172 |
+ s.send(struct.pack('!BBB', 0x05, 0x01, 0x00)) |
173 |
+ vers, method = struct.unpack('!BB', s.recv(2)) |
174 |
+ s.send(struct.pack('!BBBB', 0x05, 0x01, 0x00, 0x01)) |
175 |
+ s.send(socket.inet_pton(socket.AF_INET, host)) |
176 |
+ s.send(struct.pack('!H', port)) |
177 |
+ reply = struct.unpack('!BBB', s.recv(3)) |
178 |
+ if reply != (0x05, 0x00, 0x00): |
179 |
+ raise AssertionError(repr(reply)) |
180 |
+ struct.unpack('!B4sH', s.recv(7)) # contains proxied address info |
181 |
+ s.send("GET {} HTTP/1.1\r\nHost: {}:{}\r\nAccept: */*\r\nConnection: close\r\n\r\n".format( |
182 |
+ path, host, port).encode()) |
183 |
+ headers = [] |
184 |
+ while True: |
185 |
+ headers.append(f.readline()) |
186 |
+ if headers[-1] == b'\r\n': |
187 |
+ return f |
188 |
+ except Exception: |
189 |
+ f.close() |
190 |
+ raise |
191 |
+ |
192 |
+ |
193 |
+class Socks5ServerTestCase(TestCase): |
194 |
+ |
195 |
+ @staticmethod |
196 |
+ def _fetch_via_proxy(proxy, host, port, path): |
197 |
+ with socks5_http_get_ipv4(proxy, host, port, path) as f: |
198 |
+ return f.read() |
199 |
+ |
200 |
+ def test_socks5_proxy(self): |
201 |
+ |
202 |
+ loop = global_event_loop() |
203 |
+ |
204 |
+ host = '127.0.0.1' |
205 |
+ content = b'Hello World!' |
206 |
+ path = '/index.html' |
207 |
+ proxy = None |
208 |
+ tempdir = tempfile.mkdtemp() |
209 |
+ |
210 |
+ try: |
211 |
+ with AsyncHTTPServer(host, {path: content}, loop) as server: |
212 |
+ |
213 |
+ settings = { |
214 |
+ 'PORTAGE_TMPDIR': tempdir, |
215 |
+ 'PORTAGE_BIN_PATH': PORTAGE_BIN_PATH, |
216 |
+ } |
217 |
+ |
218 |
+ try: |
219 |
+ proxy = socks5.get_socks5_proxy(settings) |
220 |
+ except NotImplementedError: |
221 |
+ # bug 658172 for python2.7 |
222 |
+ self.skipTest('get_socks5_proxy not implemented for {} {}.{}'.format( |
223 |
+ platform.python_implementation(), *sys.version_info[:2])) |
224 |
+ else: |
225 |
+ loop.run_until_complete(socks5.proxy.ready()) |
226 |
+ |
227 |
+ result = loop.run_until_complete(loop.run_in_executor(None, |
228 |
+ self._fetch_via_proxy, proxy, host, server.server_port, path)) |
229 |
+ |
230 |
+ self.assertEqual(result, content) |
231 |
+ finally: |
232 |
+ socks5.proxy.stop() |
233 |
+ shutil.rmtree(tempdir) |
234 |
|
235 |
diff --git a/lib/portage/util/socks5.py b/lib/portage/util/socks5.py |
236 |
index 74b0714eb..59e6699ec 100644 |
237 |
--- a/lib/portage/util/socks5.py |
238 |
+++ b/lib/portage/util/socks5.py |
239 |
@@ -1,13 +1,18 @@ |
240 |
# SOCKSv5 proxy manager for network-sandbox |
241 |
-# Copyright 2015 Gentoo Foundation |
242 |
+# Copyright 2015-2019 Gentoo Authors |
243 |
# Distributed under the terms of the GNU General Public License v2 |
244 |
|
245 |
+import errno |
246 |
import os |
247 |
import signal |
248 |
+import socket |
249 |
|
250 |
+import portage.data |
251 |
from portage import _python_interpreter |
252 |
from portage.data import portage_gid, portage_uid, userpriv_groups |
253 |
from portage.process import atexit_register, spawn |
254 |
+from portage.util.futures.compat_coroutine import coroutine |
255 |
+from portage.util.futures import asyncio |
256 |
|
257 |
|
258 |
class ProxyManager(object): |
259 |
@@ -36,9 +41,16 @@ class ProxyManager(object): |
260 |
self.socket_path = os.path.join(settings['PORTAGE_TMPDIR'], |
261 |
'.portage.%d.net.sock' % os.getpid()) |
262 |
server_bin = os.path.join(settings['PORTAGE_BIN_PATH'], 'socks5-server.py') |
263 |
+ spawn_kwargs = {} |
264 |
+ # The portage_uid check solves EPERM failures in Travis CI. |
265 |
+ if portage.data.secpass > 1 and os.geteuid() != portage_uid: |
266 |
+ spawn_kwargs.update( |
267 |
+ uid=portage_uid, |
268 |
+ gid=portage_gid, |
269 |
+ groups=userpriv_groups, |
270 |
+ umask=0o077) |
271 |
self._pids = spawn([_python_interpreter, server_bin, self.socket_path], |
272 |
- returnpid=True, uid=portage_uid, gid=portage_gid, |
273 |
- groups=userpriv_groups, umask=0o077) |
274 |
+ returnpid=True, **spawn_kwargs) |
275 |
|
276 |
def stop(self): |
277 |
""" |
278 |
@@ -60,6 +72,36 @@ class ProxyManager(object): |
279 |
return self.socket_path is not None |
280 |
|
281 |
|
282 |
+ @coroutine |
283 |
+ def ready(self): |
284 |
+ """ |
285 |
+ Wait for the proxy socket to become ready. This method is a coroutine. |
286 |
+ """ |
287 |
+ |
288 |
+ while True: |
289 |
+ try: |
290 |
+ wait_retval = os.waitpid(self._pids[0], os.WNOHANG) |
291 |
+ except OSError as e: |
292 |
+ if e.errno == errno.EINTR: |
293 |
+ continue |
294 |
+ raise |
295 |
+ |
296 |
+ if wait_retval is not None and wait_retval != (0, 0): |
297 |
+ raise OSError(3, 'No such process') |
298 |
+ |
299 |
+ try: |
300 |
+ s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) |
301 |
+ s.connect(self.socket_path) |
302 |
+ except EnvironmentError as e: |
303 |
+ if e.errno != errno.ENOENT: |
304 |
+ raise |
305 |
+ yield asyncio.sleep(0.2) |
306 |
+ else: |
307 |
+ break |
308 |
+ finally: |
309 |
+ s.close() |
310 |
+ |
311 |
+ |
312 |
proxy = ProxyManager() |