1 |
commit: d872348a8172b17261cfc5d13ee87e0ea6c23fc1 |
2 |
Author: Brian Dolbec <dolsen <AT> gentoo <DOT> org> |
3 |
AuthorDate: Fri Jan 31 15:35:11 2014 +0000 |
4 |
Commit: Brian Dolbec <brian.dolbec <AT> gmail <DOT> com> |
5 |
CommitDate: Fri Jan 31 15:39:09 2014 +0000 |
6 |
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/mirrorselect.git;a=commit;h=d872348a |
7 |
|
8 |
Fix some long lines. |
9 |
|
10 |
--- |
11 |
mirrorselect/extractor.py | 4 +- |
12 |
mirrorselect/selectors.py | 134 +++++++++++++++++++++++++--------------------- |
13 |
setup.py | 3 +- |
14 |
3 files changed, 78 insertions(+), 63 deletions(-) |
15 |
|
16 |
diff --git a/mirrorselect/extractor.py b/mirrorselect/extractor.py |
17 |
index a949c75..217d1e1 100644 |
18 |
--- a/mirrorselect/extractor.py |
19 |
+++ b/mirrorselect/extractor.py |
20 |
@@ -37,8 +37,8 @@ USERAGENT = "Mirrorselect-" + version |
21 |
|
22 |
class Extractor(object): |
23 |
"""The Extractor employs a MirrorParser3 object to get a list of valid |
24 |
- mirrors, and then filters them. Only the mirrors that should be tested, based on |
25 |
- user input are saved. They will be in the hosts attribute.""" |
26 |
+ mirrors, and then filters them. Only the mirrors that should be tested, |
27 |
+ based on user input are saved. They will be in the hosts attribute.""" |
28 |
|
29 |
def __init__(self, list_url, options, output): |
30 |
self.output = output |
31 |
|
32 |
diff --git a/mirrorselect/selectors.py b/mirrorselect/selectors.py |
33 |
index 1544937..5d30695 100644 |
34 |
--- a/mirrorselect/selectors.py |
35 |
+++ b/mirrorselect/selectors.py |
36 |
@@ -82,13 +82,13 @@ class Shallow(object): |
37 |
top_hosts = [] |
38 |
|
39 |
if not quiet: |
40 |
- self.output.print_info('Using netselect to choose the top %d mirrors...' \ |
41 |
- % number) |
42 |
+ self.output.print_info('Using netselect to choose the top ' |
43 |
+ '%d mirrors...' % number) |
44 |
|
45 |
host_string = ' '.join(hosts) |
46 |
|
47 |
- self.output.write('\nnetselect(): running "netselect -s%d %s"\n' % (int(number), |
48 |
- host_string), 2) |
49 |
+ self.output.write('\nnetselect(): running "netselect -s%d %s"\n' |
50 |
+ % (int(number), host_string), 2) |
51 |
|
52 |
proc = subprocess.Popen( ['netselect', '-s%d' % (number,)] + hosts, |
53 |
stdout=subprocess.PIPE, stderr=subprocess.PIPE) |
54 |
@@ -119,7 +119,8 @@ class Shallow(object): |
55 |
|
56 |
def netselect_split(self, hosts, number, block_size): |
57 |
""" |
58 |
- This uses netselect to test mirrors in chunks, each at most block_size in length. |
59 |
+ This uses netselect to test mirrors in chunks, |
60 |
+ each at most block_size in length. |
61 |
This is done in a tournament style. |
62 |
""" |
63 |
hosts = [host[0] for host in hosts] |
64 |
@@ -141,8 +142,8 @@ class Shallow(object): |
65 |
|
66 |
host_dict = self.netselect(block, len(block), quiet=True)[1] |
67 |
|
68 |
- self.output.write('ran netselect(%s, %s), and got %s\n' % (block, len(block), |
69 |
- host_dict), 2) |
70 |
+ self.output.write('ran netselect(%s, %s), and got %s\n' |
71 |
+ % (block, len(block), host_dict), 2) |
72 |
|
73 |
for key in list(host_dict.keys()): |
74 |
ret_hosts[key] = host_dict[key] |
75 |
@@ -165,7 +166,8 @@ class Shallow(object): |
76 |
|
77 |
def host_blocks(self, hosts, block_size): |
78 |
""" |
79 |
- Takes a list of hosts and a block size, and returns an list of lists of URLs. |
80 |
+ Takes a list of hosts and a block size, |
81 |
+ and returns an list of lists of URLs. |
82 |
Each of the sublists is at most block_size in length. |
83 |
""" |
84 |
host_array = [] |
85 |
@@ -178,8 +180,9 @@ class Shallow(object): |
86 |
mylist = [] |
87 |
host_array.append(hosts) |
88 |
|
89 |
- self.output.write('\n_host_blocks(): returns %s blocks, each about %s in size\n' |
90 |
- % (len(host_array), len(host_array[0])), 2) |
91 |
+ self.output.write('\n_host_blocks(): returns ' |
92 |
+ '%s blocks, each about %s in size\n' |
93 |
+ % (len(host_array), len(host_array[0])), 2) |
94 |
|
95 |
return host_array |
96 |
|
97 |
@@ -235,11 +238,13 @@ class Deep(object): |
98 |
|
99 |
prog += 1 |
100 |
if self.test_file is not 'mirrorselect-test': |
101 |
- self.output.print_info('Downloading %s files from each mirror... [%s of %s]'\ |
102 |
- % (self.test_file, prog, num_hosts) ) |
103 |
+ self.output.print_info( |
104 |
+ 'Downloading %s files from each mirror... [%s of %s]' |
105 |
+ % (self.test_file, prog, num_hosts) ) |
106 |
else: |
107 |
- self.output.print_info('Downloading 100k files from each mirror... [%s of %s]'\ |
108 |
- % (prog, num_hosts) ) |
109 |
+ self.output.print_info( |
110 |
+ 'Downloading 100k files from each mirror... [%s of %s]' |
111 |
+ % (prog, num_hosts) ) |
112 |
|
113 |
mytime, ignore = self.deeptime(host, maxtime) |
114 |
|
115 |
@@ -249,18 +254,20 @@ class Deep(object): |
116 |
else: |
117 |
continue |
118 |
|
119 |
- self.output.write('deeptest(): got %s hosts, and returned %s\n' % (num_hosts, \ |
120 |
- str(list(top_hosts.values()))), 2) |
121 |
+ self.output.write('deeptest(): got %s hosts, and returned %s\n' |
122 |
+ % (num_hosts, str(list(top_hosts.values()))), 2) |
123 |
|
124 |
self.output.write('\n') #this just makes output nicer |
125 |
|
126 |
- #can't just return the dict.values, because we want the fastest mirror first... |
127 |
+ #can't just return the dict.values, |
128 |
+ #because we want the fastest mirror first... |
129 |
keys = list(top_hosts.keys()) |
130 |
keys.sort() |
131 |
|
132 |
rethosts = [] |
133 |
for key in keys: |
134 |
- #self.output.write('deeptest(): adding rethost %s, %s' % (key, top_hosts[key]), 2) |
135 |
+ #self.output.write('deeptest(): adding rethost ' |
136 |
+ #'%s, %s' % (key, top_hosts[key]), 2) |
137 |
rethosts.append(top_hosts[key]) |
138 |
|
139 |
self.output.write('deeptest(): final rethost %s\n' % (rethosts), 2) |
140 |
@@ -298,17 +305,21 @@ class Deep(object): |
141 |
ip = "[%s]" % ip |
142 |
ips.append(ip) |
143 |
except socket.error as e: |
144 |
- self.output.write('deeptime(): dns error for host %s: %s\n' % (url_parts.hostname, e), 2) |
145 |
+ self.output.write('deeptime(): dns error for host %s: %s\n' |
146 |
+ % (url_parts.hostname, e), 2) |
147 |
except TimeoutException: |
148 |
- self.output.write('deeptime(): dns timeout for host %s\n' % url_parts.hostname, 2) |
149 |
+ self.output.write('deeptime(): dns timeout for host %s\n' |
150 |
+ % url_parts.hostname, 2) |
151 |
finally: |
152 |
signal.alarm(0) |
153 |
|
154 |
if not ips: |
155 |
- self.output.write('deeptime(): unable to resolve ip for host %s\n' % url_parts.hostname, 2) |
156 |
+ self.output.write('deeptime(): unable to resolve ip for host %s\n' |
157 |
+ % url_parts.hostname, 2) |
158 |
return (None, True) |
159 |
|
160 |
- self.output.write("deeptime(): ip's for host %s: %s\n" % (url_parts.hostname, str(ips)), 2) |
161 |
+ self.output.write("deeptime(): ip's for host %s: %s\n" |
162 |
+ % (url_parts.hostname, str(ips)), 2) |
163 |
delta = 0 |
164 |
f = None |
165 |
|
166 |
@@ -336,13 +347,11 @@ class Deep(object): |
167 |
finally: |
168 |
signal.alarm(0) |
169 |
except EnvironmentError as e: |
170 |
- self.output.write(('deeptime(): closing connection to host %s ' + \ |
171 |
- 'failed for ip %s: %s\n') % \ |
172 |
- (url_parts.hostname, ip, e), 2) |
173 |
+ self.output.write(('deeptime(): closing connection to host %s ' |
174 |
+ 'failed for ip %s: %s\n') % (url_parts.hostname, ip, e), 2) |
175 |
except TimeoutException: |
176 |
- self.output.write(('deeptime(): closing connection to host %s ' + \ |
177 |
- 'timed out for ip %s\n') % \ |
178 |
- (url_parts.hostname, ip), 2) |
179 |
+ self.output.write(('deeptime(): closing connection to host %s ' |
180 |
+ 'timed out for ip %s\n') % (url_parts.hostname, ip), 2) |
181 |
|
182 |
self.output.write('deeptime(): timing url: %s\n' % test_url, 2) |
183 |
try: |
184 |
@@ -360,10 +369,12 @@ class Deep(object): |
185 |
f.close() |
186 |
if md5 != self.test_md5: |
187 |
self.output.write( |
188 |
- "deeptime(): md5sum error for file: %s\n" % self.test_file + |
189 |
+ "deeptime(): md5sum error for file: %s\n" |
190 |
+ % self.test_file + |
191 |
" expected: %s\n" % self.test_md5 + |
192 |
" got.....: %s\n" % md5 + |
193 |
- " host....: %s, %s\n" % (url_parts.hostname, ip)) |
194 |
+ " host....: %s, %s\n" |
195 |
+ % (url_parts.hostname, ip)) |
196 |
self.dl_failures += 1 |
197 |
return (None, True) |
198 |
|
199 |
@@ -371,20 +382,19 @@ class Deep(object): |
200 |
signal.alarm(0) |
201 |
|
202 |
except EnvironmentError as e: |
203 |
- self.output.write(('deeptime(): download from host %s ' + \ |
204 |
- 'failed for ip %s: %s\n') % \ |
205 |
- (url_parts.hostname, ip, e), 2) |
206 |
+ self.output.write(('deeptime(): download from host %s ' |
207 |
+ 'failed for ip %s: %s\n') % (url_parts.hostname, ip, e), 2) |
208 |
return (None, True) |
209 |
except TimeoutException: |
210 |
- self.output.write(('deeptime(): download from host %s ' + \ |
211 |
- 'timed out for ip %s\n') % \ |
212 |
- (url_parts.hostname, ip), 2) |
213 |
+ self.output.write(('deeptime(): download from host %s ' |
214 |
+ 'timed out for ip %s\n') % (url_parts.hostname, ip), 2) |
215 |
return (None, True) |
216 |
|
217 |
signal.signal(signal.SIGALRM, signal.SIG_DFL) |
218 |
|
219 |
self.output.write('deeptime(): download completed.\n', 2) |
220 |
- self.output.write('deeptime(): %s seconds for host %s\n' % (delta, url), 2) |
221 |
+ self.output.write('deeptime(): %s seconds for host %s\n' |
222 |
+ % (delta, url), 2) |
223 |
return (delta, False) |
224 |
|
225 |
|
226 |
@@ -403,21 +413,20 @@ class Deep(object): |
227 |
finally: |
228 |
signal.alarm(0) |
229 |
except HTTPError as e: |
230 |
- self.output.write(('deeptime(): connection to host %s\n' + \ |
231 |
- ' returned HTTPError: %s for ip %s\n' \ |
232 |
- ' Switching back to original url\n') % \ |
233 |
- (url_parts.hostname, e, ip), 2) |
234 |
+ self.output.write('deeptime(): connection to host %s\n' |
235 |
+ ' returned HTTPError: %s for ip %s\n' |
236 |
+ ' Switching back to original url\n' |
237 |
+ % (url_parts.hostname, e, ip), 2) |
238 |
if len(ips) == 1: |
239 |
test_url = url_unparse(url_parts) |
240 |
return self._test_connection(test_url, url_parts, ip, []) |
241 |
except EnvironmentError as e: |
242 |
- self.output.write(('deeptime(): connection to host %s ' + \ |
243 |
- 'failed for ip %s:\n %s\n') % \ |
244 |
- (url_parts.hostname, ip, e), 2) |
245 |
+ self.output.write('deeptime(): connection to host %s ' |
246 |
+ 'failed for ip %s:\n %s\n' |
247 |
+ % (url_parts.hostname, ip, e), 2) |
248 |
except TimeoutException: |
249 |
- self.output.write(('deeptime(): connection to host %s ' + \ |
250 |
- 'timed out for ip %s\n') % \ |
251 |
- (url_parts.hostname, ip), 2) |
252 |
+ self.output.write(('deeptime(): connection to host %s ' |
253 |
+ 'timed out for ip %s\n') % (url_parts.hostname, ip), 2) |
254 |
return f, test_url, early_out |
255 |
|
256 |
|
257 |
@@ -426,24 +435,25 @@ class Deep(object): |
258 |
Takes argumets ((time, host), maxtime, host_dict, maxlen) |
259 |
Adds a new time:host pair to the dictionary of top hosts. |
260 |
If the dictionary is full, the slowest host is removed to make space. |
261 |
- Returns the new maxtime, be it the specified timeout, or the slowest host. |
262 |
+ Returns the new maxtime, be it the specified timeout, |
263 |
+ or the slowest host. |
264 |
""" |
265 |
if len(host_dict) < maxlen: #still have room, and host is fast. add it. |
266 |
|
267 |
- self.output.write('_list_add(): added host %s. with a time of %s\n' % |
268 |
- (time_host[1], time_host[0]), 2) |
269 |
+ self.output.write('_list_add(): added host %s. with a time of %s\n' |
270 |
+ % (time_host[1], time_host[0]), 2) |
271 |
|
272 |
host_dict.update(dict([time_host])) |
273 |
times = list(host_dict.keys()) |
274 |
times.sort() |
275 |
|
276 |
else: #We need to make room in the dict before we add. Kill the slowest. |
277 |
- self.output.write('_list_add(): Adding host %s with a time of %s\n' % |
278 |
- (time_host[1], time_host[0]), 2) |
279 |
+ self.output.write('_list_add(): Adding host %s with a time of %s\n' |
280 |
+ % (time_host[1], time_host[0]), 2) |
281 |
times = list(host_dict.keys()) |
282 |
times.sort() |
283 |
- self.output.write('_list_add(): removing %s\n' % host_dict[times[-1]], |
284 |
- 2) |
285 |
+ self.output.write('_list_add(): removing %s\n' |
286 |
+ % host_dict[times[-1]], 2) |
287 |
del host_dict[times[-1]] |
288 |
host_dict.update(dict([time_host])) |
289 |
#done adding. now return the appropriate time |
290 |
@@ -455,8 +465,8 @@ class Deep(object): |
291 |
' reusing timeout of %s sec.\n' % maxtime, 2) |
292 |
retval = maxtime |
293 |
else: |
294 |
- self.output.write('_list_add(): host_dict is full. Selecting the best' |
295 |
- ' timeout\n', 2) |
296 |
+ self.output.write('_list_add(): host_dict is full. ' |
297 |
+ 'Selecting the best timeout\n', 2) |
298 |
if times[-1] < maxtime: |
299 |
retval = times[-1] |
300 |
else: |
301 |
@@ -476,7 +486,8 @@ class Interactive(object): |
302 |
self.urls = [] |
303 |
|
304 |
self.interactive(hosts, options) |
305 |
- self.output.write('Interactive.interactive(): self.urls = %s\n' % self.urls, 2) |
306 |
+ self.output.write('Interactive.interactive(): self.urls = %s\n' |
307 |
+ % self.urls, 2) |
308 |
|
309 |
if not self.urls or len(self.urls[0]) == 0: |
310 |
sys.exit(1) |
311 |
@@ -499,7 +510,8 @@ class Interactive(object): |
312 |
|
313 |
dialog.extend(['20', '110', '14']) |
314 |
|
315 |
- for (url, args) in sorted(hosts, key = lambda x: (x[1]['country'].lower(), x[1]['name'].lower()) ): |
316 |
+ for (url, args) in sorted(hosts, key = lambda x: |
317 |
+ (x[1]['country'].lower(), x[1]['name'].lower()) ): |
318 |
marker = "" |
319 |
if options.rsync and not url.endswith("/gentoo-portage"): |
320 |
url+="/gentoo-portage" |
321 |
@@ -508,7 +520,8 @@ class Interactive(object): |
322 |
if options.ipv6 and ( args['ipv6'] == 'n' ): continue |
323 |
if options.ipv4 and ( args['ipv4'] == 'n' ): continue |
324 |
|
325 |
- #dialog.append('"%s" "%s%s: %s" "OFF"' % ( url, marker, args['country'], args['name'])) |
326 |
+ #dialog.append('"%s" "%s%s: %s" "OFF"' |
327 |
+ #% ( url, marker, args['country'], args['name'])) |
328 |
dialog.extend(["%s" %url, |
329 |
"%s%s: %s" %(marker, args['country'], args['name']), |
330 |
"OFF"]) |
331 |
@@ -522,7 +535,8 @@ class Interactive(object): |
332 |
|
333 |
if self.urls: |
334 |
if hasattr(self.urls[0], 'decode'): |
335 |
- self.urls = decode_selection([x.decode('utf-8').rstrip() for x in self.urls]) |
336 |
+ self.urls = decode_selection( |
337 |
+ [x.decode('utf-8').rstrip() for x in self.urls]) |
338 |
else: |
339 |
self.urls = decode_selection([x.rstrip() for x in self.urls]) |
340 |
|
341 |
|
342 |
diff --git a/setup.py b/setup.py |
343 |
index 32c7e1d..df7b851 100755 |
344 |
--- a/setup.py |
345 |
+++ b/setup.py |
346 |
@@ -108,7 +108,8 @@ core.setup( |
347 |
#package_data = test_data, |
348 |
scripts=(['bin/mirrorselect']), |
349 |
data_files=( |
350 |
- (os.path.join(os.sep, EPREFIX.lstrip(os.sep), 'usr/share/man/man8'), ['mirrorselect.8']), |
351 |
+ (os.path.join(os.sep, EPREFIX.lstrip(os.sep), 'usr/share/man/man8'), |
352 |
+ ['mirrorselect.8']), |
353 |
), |
354 |
cmdclass={ |
355 |
'test': load_test(), |