Gentoo Archives: gentoo-commits

From: Brian Dolbec <brian.dolbec@×××××.com>
To: gentoo-commits@l.g.o
Subject: [gentoo-commits] proj/mirrorselect:ssl commit in: /, mirrorselect/
Date: Fri, 31 Jan 2014 15:44:56
Message-Id: 1391182749.d872348a8172b17261cfc5d13ee87e0ea6c23fc1.dol-sen@gentoo
1 commit: d872348a8172b17261cfc5d13ee87e0ea6c23fc1
2 Author: Brian Dolbec <dolsen <AT> gentoo <DOT> org>
3 AuthorDate: Fri Jan 31 15:35:11 2014 +0000
4 Commit: Brian Dolbec <brian.dolbec <AT> gmail <DOT> com>
5 CommitDate: Fri Jan 31 15:39:09 2014 +0000
6 URL: http://git.overlays.gentoo.org/gitweb/?p=proj/mirrorselect.git;a=commit;h=d872348a
7
8 Fix some long lines.
9
10 ---
11 mirrorselect/extractor.py | 4 +-
12 mirrorselect/selectors.py | 134 +++++++++++++++++++++++++---------------------
13 setup.py | 3 +-
14 3 files changed, 78 insertions(+), 63 deletions(-)
15
16 diff --git a/mirrorselect/extractor.py b/mirrorselect/extractor.py
17 index a949c75..217d1e1 100644
18 --- a/mirrorselect/extractor.py
19 +++ b/mirrorselect/extractor.py
20 @@ -37,8 +37,8 @@ USERAGENT = "Mirrorselect-" + version
21
22 class Extractor(object):
23 """The Extractor employs a MirrorParser3 object to get a list of valid
24 - mirrors, and then filters them. Only the mirrors that should be tested, based on
25 - user input are saved. They will be in the hosts attribute."""
26 + mirrors, and then filters them. Only the mirrors that should be tested,
27 + based on user input are saved. They will be in the hosts attribute."""
28
29 def __init__(self, list_url, options, output):
30 self.output = output
31
32 diff --git a/mirrorselect/selectors.py b/mirrorselect/selectors.py
33 index 1544937..5d30695 100644
34 --- a/mirrorselect/selectors.py
35 +++ b/mirrorselect/selectors.py
36 @@ -82,13 +82,13 @@ class Shallow(object):
37 top_hosts = []
38
39 if not quiet:
40 - self.output.print_info('Using netselect to choose the top %d mirrors...' \
41 - % number)
42 + self.output.print_info('Using netselect to choose the top '
43 + '%d mirrors...' % number)
44
45 host_string = ' '.join(hosts)
46
47 - self.output.write('\nnetselect(): running "netselect -s%d %s"\n' % (int(number),
48 - host_string), 2)
49 + self.output.write('\nnetselect(): running "netselect -s%d %s"\n'
50 + % (int(number), host_string), 2)
51
52 proc = subprocess.Popen( ['netselect', '-s%d' % (number,)] + hosts,
53 stdout=subprocess.PIPE, stderr=subprocess.PIPE)
54 @@ -119,7 +119,8 @@ class Shallow(object):
55
56 def netselect_split(self, hosts, number, block_size):
57 """
58 - This uses netselect to test mirrors in chunks, each at most block_size in length.
59 + This uses netselect to test mirrors in chunks,
60 + each at most block_size in length.
61 This is done in a tournament style.
62 """
63 hosts = [host[0] for host in hosts]
64 @@ -141,8 +142,8 @@ class Shallow(object):
65
66 host_dict = self.netselect(block, len(block), quiet=True)[1]
67
68 - self.output.write('ran netselect(%s, %s), and got %s\n' % (block, len(block),
69 - host_dict), 2)
70 + self.output.write('ran netselect(%s, %s), and got %s\n'
71 + % (block, len(block), host_dict), 2)
72
73 for key in list(host_dict.keys()):
74 ret_hosts[key] = host_dict[key]
75 @@ -165,7 +166,8 @@ class Shallow(object):
76
77 def host_blocks(self, hosts, block_size):
78 """
79 - Takes a list of hosts and a block size, and returns an list of lists of URLs.
80 + Takes a list of hosts and a block size,
81 + and returns an list of lists of URLs.
82 Each of the sublists is at most block_size in length.
83 """
84 host_array = []
85 @@ -178,8 +180,9 @@ class Shallow(object):
86 mylist = []
87 host_array.append(hosts)
88
89 - self.output.write('\n_host_blocks(): returns %s blocks, each about %s in size\n'
90 - % (len(host_array), len(host_array[0])), 2)
91 + self.output.write('\n_host_blocks(): returns '
92 + '%s blocks, each about %s in size\n'
93 + % (len(host_array), len(host_array[0])), 2)
94
95 return host_array
96
97 @@ -235,11 +238,13 @@ class Deep(object):
98
99 prog += 1
100 if self.test_file is not 'mirrorselect-test':
101 - self.output.print_info('Downloading %s files from each mirror... [%s of %s]'\
102 - % (self.test_file, prog, num_hosts) )
103 + self.output.print_info(
104 + 'Downloading %s files from each mirror... [%s of %s]'
105 + % (self.test_file, prog, num_hosts) )
106 else:
107 - self.output.print_info('Downloading 100k files from each mirror... [%s of %s]'\
108 - % (prog, num_hosts) )
109 + self.output.print_info(
110 + 'Downloading 100k files from each mirror... [%s of %s]'
111 + % (prog, num_hosts) )
112
113 mytime, ignore = self.deeptime(host, maxtime)
114
115 @@ -249,18 +254,20 @@ class Deep(object):
116 else:
117 continue
118
119 - self.output.write('deeptest(): got %s hosts, and returned %s\n' % (num_hosts, \
120 - str(list(top_hosts.values()))), 2)
121 + self.output.write('deeptest(): got %s hosts, and returned %s\n'
122 + % (num_hosts, str(list(top_hosts.values()))), 2)
123
124 self.output.write('\n') #this just makes output nicer
125
126 - #can't just return the dict.values, because we want the fastest mirror first...
127 + #can't just return the dict.values,
128 + #because we want the fastest mirror first...
129 keys = list(top_hosts.keys())
130 keys.sort()
131
132 rethosts = []
133 for key in keys:
134 - #self.output.write('deeptest(): adding rethost %s, %s' % (key, top_hosts[key]), 2)
135 + #self.output.write('deeptest(): adding rethost '
136 + #'%s, %s' % (key, top_hosts[key]), 2)
137 rethosts.append(top_hosts[key])
138
139 self.output.write('deeptest(): final rethost %s\n' % (rethosts), 2)
140 @@ -298,17 +305,21 @@ class Deep(object):
141 ip = "[%s]" % ip
142 ips.append(ip)
143 except socket.error as e:
144 - self.output.write('deeptime(): dns error for host %s: %s\n' % (url_parts.hostname, e), 2)
145 + self.output.write('deeptime(): dns error for host %s: %s\n'
146 + % (url_parts.hostname, e), 2)
147 except TimeoutException:
148 - self.output.write('deeptime(): dns timeout for host %s\n' % url_parts.hostname, 2)
149 + self.output.write('deeptime(): dns timeout for host %s\n'
150 + % url_parts.hostname, 2)
151 finally:
152 signal.alarm(0)
153
154 if not ips:
155 - self.output.write('deeptime(): unable to resolve ip for host %s\n' % url_parts.hostname, 2)
156 + self.output.write('deeptime(): unable to resolve ip for host %s\n'
157 + % url_parts.hostname, 2)
158 return (None, True)
159
160 - self.output.write("deeptime(): ip's for host %s: %s\n" % (url_parts.hostname, str(ips)), 2)
161 + self.output.write("deeptime(): ip's for host %s: %s\n"
162 + % (url_parts.hostname, str(ips)), 2)
163 delta = 0
164 f = None
165
166 @@ -336,13 +347,11 @@ class Deep(object):
167 finally:
168 signal.alarm(0)
169 except EnvironmentError as e:
170 - self.output.write(('deeptime(): closing connection to host %s ' + \
171 - 'failed for ip %s: %s\n') % \
172 - (url_parts.hostname, ip, e), 2)
173 + self.output.write(('deeptime(): closing connection to host %s '
174 + 'failed for ip %s: %s\n') % (url_parts.hostname, ip, e), 2)
175 except TimeoutException:
176 - self.output.write(('deeptime(): closing connection to host %s ' + \
177 - 'timed out for ip %s\n') % \
178 - (url_parts.hostname, ip), 2)
179 + self.output.write(('deeptime(): closing connection to host %s '
180 + 'timed out for ip %s\n') % (url_parts.hostname, ip), 2)
181
182 self.output.write('deeptime(): timing url: %s\n' % test_url, 2)
183 try:
184 @@ -360,10 +369,12 @@ class Deep(object):
185 f.close()
186 if md5 != self.test_md5:
187 self.output.write(
188 - "deeptime(): md5sum error for file: %s\n" % self.test_file +
189 + "deeptime(): md5sum error for file: %s\n"
190 + % self.test_file +
191 " expected: %s\n" % self.test_md5 +
192 " got.....: %s\n" % md5 +
193 - " host....: %s, %s\n" % (url_parts.hostname, ip))
194 + " host....: %s, %s\n"
195 + % (url_parts.hostname, ip))
196 self.dl_failures += 1
197 return (None, True)
198
199 @@ -371,20 +382,19 @@ class Deep(object):
200 signal.alarm(0)
201
202 except EnvironmentError as e:
203 - self.output.write(('deeptime(): download from host %s ' + \
204 - 'failed for ip %s: %s\n') % \
205 - (url_parts.hostname, ip, e), 2)
206 + self.output.write(('deeptime(): download from host %s '
207 + 'failed for ip %s: %s\n') % (url_parts.hostname, ip, e), 2)
208 return (None, True)
209 except TimeoutException:
210 - self.output.write(('deeptime(): download from host %s ' + \
211 - 'timed out for ip %s\n') % \
212 - (url_parts.hostname, ip), 2)
213 + self.output.write(('deeptime(): download from host %s '
214 + 'timed out for ip %s\n') % (url_parts.hostname, ip), 2)
215 return (None, True)
216
217 signal.signal(signal.SIGALRM, signal.SIG_DFL)
218
219 self.output.write('deeptime(): download completed.\n', 2)
220 - self.output.write('deeptime(): %s seconds for host %s\n' % (delta, url), 2)
221 + self.output.write('deeptime(): %s seconds for host %s\n'
222 + % (delta, url), 2)
223 return (delta, False)
224
225
226 @@ -403,21 +413,20 @@ class Deep(object):
227 finally:
228 signal.alarm(0)
229 except HTTPError as e:
230 - self.output.write(('deeptime(): connection to host %s\n' + \
231 - ' returned HTTPError: %s for ip %s\n' \
232 - ' Switching back to original url\n') % \
233 - (url_parts.hostname, e, ip), 2)
234 + self.output.write('deeptime(): connection to host %s\n'
235 + ' returned HTTPError: %s for ip %s\n'
236 + ' Switching back to original url\n'
237 + % (url_parts.hostname, e, ip), 2)
238 if len(ips) == 1:
239 test_url = url_unparse(url_parts)
240 return self._test_connection(test_url, url_parts, ip, [])
241 except EnvironmentError as e:
242 - self.output.write(('deeptime(): connection to host %s ' + \
243 - 'failed for ip %s:\n %s\n') % \
244 - (url_parts.hostname, ip, e), 2)
245 + self.output.write('deeptime(): connection to host %s '
246 + 'failed for ip %s:\n %s\n'
247 + % (url_parts.hostname, ip, e), 2)
248 except TimeoutException:
249 - self.output.write(('deeptime(): connection to host %s ' + \
250 - 'timed out for ip %s\n') % \
251 - (url_parts.hostname, ip), 2)
252 + self.output.write(('deeptime(): connection to host %s '
253 + 'timed out for ip %s\n') % (url_parts.hostname, ip), 2)
254 return f, test_url, early_out
255
256
257 @@ -426,24 +435,25 @@ class Deep(object):
258 Takes argumets ((time, host), maxtime, host_dict, maxlen)
259 Adds a new time:host pair to the dictionary of top hosts.
260 If the dictionary is full, the slowest host is removed to make space.
261 - Returns the new maxtime, be it the specified timeout, or the slowest host.
262 + Returns the new maxtime, be it the specified timeout,
263 + or the slowest host.
264 """
265 if len(host_dict) < maxlen: #still have room, and host is fast. add it.
266
267 - self.output.write('_list_add(): added host %s. with a time of %s\n' %
268 - (time_host[1], time_host[0]), 2)
269 + self.output.write('_list_add(): added host %s. with a time of %s\n'
270 + % (time_host[1], time_host[0]), 2)
271
272 host_dict.update(dict([time_host]))
273 times = list(host_dict.keys())
274 times.sort()
275
276 else: #We need to make room in the dict before we add. Kill the slowest.
277 - self.output.write('_list_add(): Adding host %s with a time of %s\n' %
278 - (time_host[1], time_host[0]), 2)
279 + self.output.write('_list_add(): Adding host %s with a time of %s\n'
280 + % (time_host[1], time_host[0]), 2)
281 times = list(host_dict.keys())
282 times.sort()
283 - self.output.write('_list_add(): removing %s\n' % host_dict[times[-1]],
284 - 2)
285 + self.output.write('_list_add(): removing %s\n'
286 + % host_dict[times[-1]], 2)
287 del host_dict[times[-1]]
288 host_dict.update(dict([time_host]))
289 #done adding. now return the appropriate time
290 @@ -455,8 +465,8 @@ class Deep(object):
291 ' reusing timeout of %s sec.\n' % maxtime, 2)
292 retval = maxtime
293 else:
294 - self.output.write('_list_add(): host_dict is full. Selecting the best'
295 - ' timeout\n', 2)
296 + self.output.write('_list_add(): host_dict is full. '
297 + 'Selecting the best timeout\n', 2)
298 if times[-1] < maxtime:
299 retval = times[-1]
300 else:
301 @@ -476,7 +486,8 @@ class Interactive(object):
302 self.urls = []
303
304 self.interactive(hosts, options)
305 - self.output.write('Interactive.interactive(): self.urls = %s\n' % self.urls, 2)
306 + self.output.write('Interactive.interactive(): self.urls = %s\n'
307 + % self.urls, 2)
308
309 if not self.urls or len(self.urls[0]) == 0:
310 sys.exit(1)
311 @@ -499,7 +510,8 @@ class Interactive(object):
312
313 dialog.extend(['20', '110', '14'])
314
315 - for (url, args) in sorted(hosts, key = lambda x: (x[1]['country'].lower(), x[1]['name'].lower()) ):
316 + for (url, args) in sorted(hosts, key = lambda x:
317 + (x[1]['country'].lower(), x[1]['name'].lower()) ):
318 marker = ""
319 if options.rsync and not url.endswith("/gentoo-portage"):
320 url+="/gentoo-portage"
321 @@ -508,7 +520,8 @@ class Interactive(object):
322 if options.ipv6 and ( args['ipv6'] == 'n' ): continue
323 if options.ipv4 and ( args['ipv4'] == 'n' ): continue
324
325 - #dialog.append('"%s" "%s%s: %s" "OFF"' % ( url, marker, args['country'], args['name']))
326 + #dialog.append('"%s" "%s%s: %s" "OFF"'
327 + #% ( url, marker, args['country'], args['name']))
328 dialog.extend(["%s" %url,
329 "%s%s: %s" %(marker, args['country'], args['name']),
330 "OFF"])
331 @@ -522,7 +535,8 @@ class Interactive(object):
332
333 if self.urls:
334 if hasattr(self.urls[0], 'decode'):
335 - self.urls = decode_selection([x.decode('utf-8').rstrip() for x in self.urls])
336 + self.urls = decode_selection(
337 + [x.decode('utf-8').rstrip() for x in self.urls])
338 else:
339 self.urls = decode_selection([x.rstrip() for x in self.urls])
340
341
342 diff --git a/setup.py b/setup.py
343 index 32c7e1d..df7b851 100755
344 --- a/setup.py
345 +++ b/setup.py
346 @@ -108,7 +108,8 @@ core.setup(
347 #package_data = test_data,
348 scripts=(['bin/mirrorselect']),
349 data_files=(
350 - (os.path.join(os.sep, EPREFIX.lstrip(os.sep), 'usr/share/man/man8'), ['mirrorselect.8']),
351 + (os.path.join(os.sep, EPREFIX.lstrip(os.sep), 'usr/share/man/man8'),
352 + ['mirrorselect.8']),
353 ),
354 cmdclass={
355 'test': load_test(),