1 |
--- |
2 |
pym/portage/sync/modules/rsync/rsync.py | 467 ++++++++++++++++---------------- |
3 |
1 file changed, 235 insertions(+), 232 deletions(-) |
4 |
|
5 |
diff --git a/pym/portage/sync/modules/rsync/rsync.py b/pym/portage/sync/modules/rsync/rsync.py |
6 |
index e6e218868..5c0b53f9e 100644 |
7 |
--- a/pym/portage/sync/modules/rsync/rsync.py |
8 |
+++ b/pym/portage/sync/modules/rsync/rsync.py |
9 |
@@ -110,247 +110,250 @@ class RsyncSync(NewBase): |
10 |
level=logging.WARNING, noiselevel=-1) |
11 |
self.verify_jobs = None |
12 |
|
13 |
- # Real local timestamp file. |
14 |
- self.servertimestampfile = os.path.join( |
15 |
- self.repo.location, "metadata", "timestamp.chk") |
16 |
- |
17 |
- content = portage.util.grabfile(self.servertimestampfile) |
18 |
- timestamp = 0 |
19 |
- if content: |
20 |
- try: |
21 |
- timestamp = time.mktime(time.strptime(content[0], |
22 |
- TIMESTAMP_FORMAT)) |
23 |
- except (OverflowError, ValueError): |
24 |
- pass |
25 |
- del content |
26 |
- |
27 |
- try: |
28 |
- self.rsync_initial_timeout = \ |
29 |
- int(self.settings.get("PORTAGE_RSYNC_INITIAL_TIMEOUT", "15")) |
30 |
- except ValueError: |
31 |
- self.rsync_initial_timeout = 15 |
32 |
- |
33 |
- try: |
34 |
- maxretries=int(self.settings["PORTAGE_RSYNC_RETRIES"]) |
35 |
- except SystemExit as e: |
36 |
- raise # Needed else can't exit |
37 |
- except: |
38 |
- maxretries = -1 #default number of retries |
39 |
- |
40 |
- if syncuri.startswith("file://"): |
41 |
- self.proto = "file" |
42 |
- dosyncuri = syncuri[7:] |
43 |
- unchanged, is_synced, exitcode, updatecache_flg = self._do_rsync( |
44 |
- dosyncuri, timestamp, opts) |
45 |
- self._process_exitcode(exitcode, dosyncuri, out, 1) |
46 |
- return (exitcode, updatecache_flg) |
47 |
- |
48 |
- retries=0 |
49 |
try: |
50 |
- self.proto, user_name, hostname, port = re.split( |
51 |
- r"(rsync|ssh)://([^:/]+@)?(\[[:\da-fA-F]*\]|[^:/]*)(:[0-9]+)?", |
52 |
- syncuri, maxsplit=4)[1:5] |
53 |
- except ValueError: |
54 |
- writemsg_level("!!! sync-uri is invalid: %s\n" % syncuri, |
55 |
- noiselevel=-1, level=logging.ERROR) |
56 |
- return (1, False) |
57 |
+ # Real local timestamp file. |
58 |
+ self.servertimestampfile = os.path.join( |
59 |
+ self.repo.location, "metadata", "timestamp.chk") |
60 |
|
61 |
- self.ssh_opts = self.settings.get("PORTAGE_SSH_OPTS") |
62 |
+ content = portage.util.grabfile(self.servertimestampfile) |
63 |
+ timestamp = 0 |
64 |
+ if content: |
65 |
+ try: |
66 |
+ timestamp = time.mktime(time.strptime(content[0], |
67 |
+ TIMESTAMP_FORMAT)) |
68 |
+ except (OverflowError, ValueError): |
69 |
+ pass |
70 |
+ del content |
71 |
|
72 |
- if port is None: |
73 |
- port="" |
74 |
- if user_name is None: |
75 |
- user_name="" |
76 |
- if re.match(r"^\[[:\da-fA-F]*\]$", hostname) is None: |
77 |
- getaddrinfo_host = hostname |
78 |
- else: |
79 |
- # getaddrinfo needs the brackets stripped |
80 |
- getaddrinfo_host = hostname[1:-1] |
81 |
- updatecache_flg = False |
82 |
- all_rsync_opts = set(self.rsync_opts) |
83 |
- all_rsync_opts.update(self.extra_rsync_opts) |
84 |
+ try: |
85 |
+ self.rsync_initial_timeout = \ |
86 |
+ int(self.settings.get("PORTAGE_RSYNC_INITIAL_TIMEOUT", "15")) |
87 |
+ except ValueError: |
88 |
+ self.rsync_initial_timeout = 15 |
89 |
|
90 |
- family = socket.AF_UNSPEC |
91 |
- if "-4" in all_rsync_opts or "--ipv4" in all_rsync_opts: |
92 |
- family = socket.AF_INET |
93 |
- elif socket.has_ipv6 and \ |
94 |
- ("-6" in all_rsync_opts or "--ipv6" in all_rsync_opts): |
95 |
- family = socket.AF_INET6 |
96 |
+ try: |
97 |
+ maxretries=int(self.settings["PORTAGE_RSYNC_RETRIES"]) |
98 |
+ except SystemExit as e: |
99 |
+ raise # Needed else can't exit |
100 |
+ except: |
101 |
+ maxretries = -1 #default number of retries |
102 |
+ |
103 |
+ if syncuri.startswith("file://"): |
104 |
+ self.proto = "file" |
105 |
+ dosyncuri = syncuri[7:] |
106 |
+ unchanged, is_synced, exitcode, updatecache_flg = self._do_rsync( |
107 |
+ dosyncuri, timestamp, opts) |
108 |
+ self._process_exitcode(exitcode, dosyncuri, out, 1) |
109 |
+ return (exitcode, updatecache_flg) |
110 |
+ |
111 |
+ retries=0 |
112 |
+ try: |
113 |
+ self.proto, user_name, hostname, port = re.split( |
114 |
+ r"(rsync|ssh)://([^:/]+@)?(\[[:\da-fA-F]*\]|[^:/]*)(:[0-9]+)?", |
115 |
+ syncuri, maxsplit=4)[1:5] |
116 |
+ except ValueError: |
117 |
+ writemsg_level("!!! sync-uri is invalid: %s\n" % syncuri, |
118 |
+ noiselevel=-1, level=logging.ERROR) |
119 |
+ return (1, False) |
120 |
|
121 |
- addrinfos = None |
122 |
- uris = [] |
123 |
+ self.ssh_opts = self.settings.get("PORTAGE_SSH_OPTS") |
124 |
|
125 |
- try: |
126 |
- addrinfos = getaddrinfo_validate( |
127 |
- socket.getaddrinfo(getaddrinfo_host, None, |
128 |
- family, socket.SOCK_STREAM)) |
129 |
- except socket.error as e: |
130 |
- writemsg_level( |
131 |
- "!!! getaddrinfo failed for '%s': %s\n" |
132 |
- % (_unicode_decode(hostname), _unicode(e)), |
133 |
- noiselevel=-1, level=logging.ERROR) |
134 |
- |
135 |
- if addrinfos: |
136 |
- |
137 |
- AF_INET = socket.AF_INET |
138 |
- AF_INET6 = None |
139 |
- if socket.has_ipv6: |
140 |
- AF_INET6 = socket.AF_INET6 |
141 |
- |
142 |
- ips_v4 = [] |
143 |
- ips_v6 = [] |
144 |
- |
145 |
- for addrinfo in addrinfos: |
146 |
- if addrinfo[0] == AF_INET: |
147 |
- ips_v4.append("%s" % addrinfo[4][0]) |
148 |
- elif AF_INET6 is not None and addrinfo[0] == AF_INET6: |
149 |
- # IPv6 addresses need to be enclosed in square brackets |
150 |
- ips_v6.append("[%s]" % addrinfo[4][0]) |
151 |
- |
152 |
- random.shuffle(ips_v4) |
153 |
- random.shuffle(ips_v6) |
154 |
- |
155 |
- # Give priority to the address family that |
156 |
- # getaddrinfo() returned first. |
157 |
- if AF_INET6 is not None and addrinfos and \ |
158 |
- addrinfos[0][0] == AF_INET6: |
159 |
- ips = ips_v6 + ips_v4 |
160 |
- else: |
161 |
- ips = ips_v4 + ips_v6 |
162 |
- |
163 |
- for ip in ips: |
164 |
- uris.append(syncuri.replace( |
165 |
- "//" + user_name + hostname + port + "/", |
166 |
- "//" + user_name + ip + port + "/", 1)) |
167 |
- |
168 |
- if not uris: |
169 |
- # With some configurations we need to use the plain hostname |
170 |
- # rather than try to resolve the ip addresses (bug #340817). |
171 |
- uris.append(syncuri) |
172 |
- |
173 |
- # reverse, for use with pop() |
174 |
- uris.reverse() |
175 |
- uris_orig = uris[:] |
176 |
- |
177 |
- effective_maxretries = maxretries |
178 |
- if effective_maxretries < 0: |
179 |
- effective_maxretries = len(uris) - 1 |
180 |
- |
181 |
- local_state_unchanged = True |
182 |
- while (1): |
183 |
- if uris: |
184 |
- dosyncuri = uris.pop() |
185 |
- elif maxretries < 0 or retries > maxretries: |
186 |
- writemsg("!!! Exhausted addresses for %s\n" |
187 |
- % _unicode_decode(hostname), noiselevel=-1) |
188 |
- return (1, False) |
189 |
- else: |
190 |
- uris.extend(uris_orig) |
191 |
- dosyncuri = uris.pop() |
192 |
- |
193 |
- if (retries==0): |
194 |
- if "--ask" in opts: |
195 |
- uq = UserQuery(opts) |
196 |
- if uq.query("Do you want to sync your Portage tree " + \ |
197 |
- "with the mirror at\n" + blue(dosyncuri) + bold("?"), |
198 |
- enter_invalid) == "No": |
199 |
- print() |
200 |
- print("Quitting.") |
201 |
- print() |
202 |
- sys.exit(128 + signal.SIGINT) |
203 |
- self.logger(self.xterm_titles, |
204 |
- ">>> Starting rsync with " + dosyncuri) |
205 |
- if "--quiet" not in opts: |
206 |
- print(">>> Starting rsync with "+dosyncuri+"...") |
207 |
- else: |
208 |
- self.logger(self.xterm_titles, |
209 |
- ">>> Starting retry %d of %d with %s" % \ |
210 |
- (retries, effective_maxretries, dosyncuri)) |
211 |
- writemsg_stdout( |
212 |
- "\n\n>>> Starting retry %d of %d with %s\n" % \ |
213 |
- (retries, effective_maxretries, dosyncuri), noiselevel=-1) |
214 |
- |
215 |
- if dosyncuri.startswith('ssh://'): |
216 |
- dosyncuri = dosyncuri[6:].replace('/', ':/', 1) |
217 |
- |
218 |
- unchanged, is_synced, exitcode, updatecache_flg = self._do_rsync( |
219 |
- dosyncuri, timestamp, opts) |
220 |
- if not unchanged: |
221 |
- local_state_unchanged = False |
222 |
- if is_synced: |
223 |
- break |
224 |
- |
225 |
- retries=retries+1 |
226 |
- |
227 |
- if maxretries < 0 or retries <= maxretries: |
228 |
- print(">>> Retrying...") |
229 |
- else: |
230 |
- # over retries |
231 |
- # exit loop |
232 |
- exitcode = EXCEEDED_MAX_RETRIES |
233 |
- break |
234 |
- self._process_exitcode(exitcode, dosyncuri, out, maxretries) |
235 |
- |
236 |
- # if synced successfully, verify now |
237 |
- if exitcode == 0 and self.verify_metamanifest: |
238 |
- if gemato is None: |
239 |
- writemsg_level("!!! Unable to verify: gemato-11.0+ is required\n", |
240 |
- level=logging.ERROR, noiselevel=-1) |
241 |
- exitcode = 127 |
242 |
+ if port is None: |
243 |
+ port="" |
244 |
+ if user_name is None: |
245 |
+ user_name="" |
246 |
+ if re.match(r"^\[[:\da-fA-F]*\]$", hostname) is None: |
247 |
+ getaddrinfo_host = hostname |
248 |
else: |
249 |
- # Use isolated environment if key is specified, |
250 |
- # system environment otherwise |
251 |
- if self.repo.sync_openpgp_key_path is not None: |
252 |
- openpgp_env_cls = gemato.openpgp.OpenPGPEnvironment |
253 |
+ # getaddrinfo needs the brackets stripped |
254 |
+ getaddrinfo_host = hostname[1:-1] |
255 |
+ updatecache_flg = False |
256 |
+ all_rsync_opts = set(self.rsync_opts) |
257 |
+ all_rsync_opts.update(self.extra_rsync_opts) |
258 |
+ |
259 |
+ family = socket.AF_UNSPEC |
260 |
+ if "-4" in all_rsync_opts or "--ipv4" in all_rsync_opts: |
261 |
+ family = socket.AF_INET |
262 |
+ elif socket.has_ipv6 and \ |
263 |
+ ("-6" in all_rsync_opts or "--ipv6" in all_rsync_opts): |
264 |
+ family = socket.AF_INET6 |
265 |
+ |
266 |
+ addrinfos = None |
267 |
+ uris = [] |
268 |
+ |
269 |
+ try: |
270 |
+ addrinfos = getaddrinfo_validate( |
271 |
+ socket.getaddrinfo(getaddrinfo_host, None, |
272 |
+ family, socket.SOCK_STREAM)) |
273 |
+ except socket.error as e: |
274 |
+ writemsg_level( |
275 |
+ "!!! getaddrinfo failed for '%s': %s\n" |
276 |
+ % (_unicode_decode(hostname), _unicode(e)), |
277 |
+ noiselevel=-1, level=logging.ERROR) |
278 |
+ |
279 |
+ if addrinfos: |
280 |
+ |
281 |
+ AF_INET = socket.AF_INET |
282 |
+ AF_INET6 = None |
283 |
+ if socket.has_ipv6: |
284 |
+ AF_INET6 = socket.AF_INET6 |
285 |
+ |
286 |
+ ips_v4 = [] |
287 |
+ ips_v6 = [] |
288 |
+ |
289 |
+ for addrinfo in addrinfos: |
290 |
+ if addrinfo[0] == AF_INET: |
291 |
+ ips_v4.append("%s" % addrinfo[4][0]) |
292 |
+ elif AF_INET6 is not None and addrinfo[0] == AF_INET6: |
293 |
+ # IPv6 addresses need to be enclosed in square brackets |
294 |
+ ips_v6.append("[%s]" % addrinfo[4][0]) |
295 |
+ |
296 |
+ random.shuffle(ips_v4) |
297 |
+ random.shuffle(ips_v6) |
298 |
+ |
299 |
+ # Give priority to the address family that |
300 |
+ # getaddrinfo() returned first. |
301 |
+ if AF_INET6 is not None and addrinfos and \ |
302 |
+ addrinfos[0][0] == AF_INET6: |
303 |
+ ips = ips_v6 + ips_v4 |
304 |
+ else: |
305 |
+ ips = ips_v4 + ips_v6 |
306 |
+ |
307 |
+ for ip in ips: |
308 |
+ uris.append(syncuri.replace( |
309 |
+ "//" + user_name + hostname + port + "/", |
310 |
+ "//" + user_name + ip + port + "/", 1)) |
311 |
+ |
312 |
+ if not uris: |
313 |
+ # With some configurations we need to use the plain hostname |
314 |
+ # rather than try to resolve the ip addresses (bug #340817). |
315 |
+ uris.append(syncuri) |
316 |
+ |
317 |
+ # reverse, for use with pop() |
318 |
+ uris.reverse() |
319 |
+ uris_orig = uris[:] |
320 |
+ |
321 |
+ effective_maxretries = maxretries |
322 |
+ if effective_maxretries < 0: |
323 |
+ effective_maxretries = len(uris) - 1 |
324 |
+ |
325 |
+ local_state_unchanged = True |
326 |
+ while (1): |
327 |
+ if uris: |
328 |
+ dosyncuri = uris.pop() |
329 |
+ elif maxretries < 0 or retries > maxretries: |
330 |
+ writemsg("!!! Exhausted addresses for %s\n" |
331 |
+ % _unicode_decode(hostname), noiselevel=-1) |
332 |
+ return (1, False) |
333 |
+ else: |
334 |
+ uris.extend(uris_orig) |
335 |
+ dosyncuri = uris.pop() |
336 |
+ |
337 |
+ if (retries==0): |
338 |
+ if "--ask" in opts: |
339 |
+ uq = UserQuery(opts) |
340 |
+ if uq.query("Do you want to sync your Portage tree " + \ |
341 |
+ "with the mirror at\n" + blue(dosyncuri) + bold("?"), |
342 |
+ enter_invalid) == "No": |
343 |
+ print() |
344 |
+ print("Quitting.") |
345 |
+ print() |
346 |
+ sys.exit(128 + signal.SIGINT) |
347 |
+ self.logger(self.xterm_titles, |
348 |
+ ">>> Starting rsync with " + dosyncuri) |
349 |
+ if "--quiet" not in opts: |
350 |
+ print(">>> Starting rsync with "+dosyncuri+"...") |
351 |
+ else: |
352 |
+ self.logger(self.xterm_titles, |
353 |
+ ">>> Starting retry %d of %d with %s" % \ |
354 |
+ (retries, effective_maxretries, dosyncuri)) |
355 |
+ writemsg_stdout( |
356 |
+ "\n\n>>> Starting retry %d of %d with %s\n" % \ |
357 |
+ (retries, effective_maxretries, dosyncuri), noiselevel=-1) |
358 |
+ |
359 |
+ if dosyncuri.startswith('ssh://'): |
360 |
+ dosyncuri = dosyncuri[6:].replace('/', ':/', 1) |
361 |
+ |
362 |
+ unchanged, is_synced, exitcode, updatecache_flg = self._do_rsync( |
363 |
+ dosyncuri, timestamp, opts) |
364 |
+ if not unchanged: |
365 |
+ local_state_unchanged = False |
366 |
+ if is_synced: |
367 |
+ break |
368 |
+ |
369 |
+ retries=retries+1 |
370 |
+ |
371 |
+ if maxretries < 0 or retries <= maxretries: |
372 |
+ print(">>> Retrying...") |
373 |
else: |
374 |
- openpgp_env_cls = gemato.openpgp.OpenPGPSystemEnvironment |
375 |
+ # over retries |
376 |
+ # exit loop |
377 |
+ exitcode = EXCEEDED_MAX_RETRIES |
378 |
+ break |
379 |
+ self._process_exitcode(exitcode, dosyncuri, out, maxretries) |
380 |
+ |
381 |
+ # if synced successfully, verify now |
382 |
+ if exitcode == 0 and self.verify_metamanifest: |
383 |
+ if gemato is None: |
384 |
+ writemsg_level("!!! Unable to verify: gemato-11.0+ is required\n", |
385 |
+ level=logging.ERROR, noiselevel=-1) |
386 |
+ exitcode = 127 |
387 |
+ else: |
388 |
+ # Use isolated environment if key is specified, |
389 |
+ # system environment otherwise |
390 |
+ if self.repo.sync_openpgp_key_path is not None: |
391 |
+ openpgp_env_cls = gemato.openpgp.OpenPGPEnvironment |
392 |
+ else: |
393 |
+ openpgp_env_cls = gemato.openpgp.OpenPGPSystemEnvironment |
394 |
+ |
395 |
+ try: |
396 |
+ with openpgp_env_cls() as openpgp_env: |
397 |
+ if self.repo.sync_openpgp_key_path is not None: |
398 |
+ out.einfo('Using keys from %s' % (self.repo.sync_openpgp_key_path,)) |
399 |
+ with io.open(self.repo.sync_openpgp_key_path, 'rb') as f: |
400 |
+ openpgp_env.import_key(f) |
401 |
+ out.ebegin('Refreshing keys from keyserver') |
402 |
+ openpgp_env.refresh_keys() |
403 |
+ out.eend(0) |
404 |
+ |
405 |
+ # we always verify the Manifest signature, in case |
406 |
+ # we had to deal with key revocation case |
407 |
+ m = gemato.recursiveloader.ManifestRecursiveLoader( |
408 |
+ os.path.join(self.repo.location, 'Manifest'), |
409 |
+ verify_openpgp=True, |
410 |
+ openpgp_env=openpgp_env, |
411 |
+ max_jobs=self.verify_jobs) |
412 |
+ if not m.openpgp_signed: |
413 |
+ raise RuntimeError('OpenPGP signature not found on Manifest') |
414 |
+ |
415 |
+ ts = m.find_timestamp() |
416 |
+ if ts is None: |
417 |
+ raise RuntimeError('Timestamp not found in Manifest') |
418 |
+ |
419 |
+ out.einfo('Manifest timestamp: %s UTC' % (ts.ts,)) |
420 |
+ out.einfo('Valid OpenPGP signature found:') |
421 |
+ out.einfo('- primary key: %s' % ( |
422 |
+ m.openpgp_signature.primary_key_fingerprint)) |
423 |
+ out.einfo('- subkey: %s' % ( |
424 |
+ m.openpgp_signature.fingerprint)) |
425 |
+ out.einfo('- timestamp: %s UTC' % ( |
426 |
+ m.openpgp_signature.timestamp)) |
427 |
+ |
428 |
+ # if nothing has changed, skip the actual Manifest |
429 |
+ # verification |
430 |
+ if not local_state_unchanged: |
431 |
+ out.ebegin('Verifying %s' % (self.repo.location,)) |
432 |
+ m.assert_directory_verifies() |
433 |
+ out.eend(0) |
434 |
+ except GematoException as e: |
435 |
+ writemsg_level("!!! Manifest verification failed:\n%s\n" |
436 |
+ % (e,), |
437 |
+ level=logging.ERROR, noiselevel=-1) |
438 |
+ exitcode = 1 |
439 |
|
440 |
- try: |
441 |
- with openpgp_env_cls() as openpgp_env: |
442 |
- if self.repo.sync_openpgp_key_path is not None: |
443 |
- out.einfo('Using keys from %s' % (self.repo.sync_openpgp_key_path,)) |
444 |
- with io.open(self.repo.sync_openpgp_key_path, 'rb') as f: |
445 |
- openpgp_env.import_key(f) |
446 |
- out.ebegin('Refreshing keys from keyserver') |
447 |
- openpgp_env.refresh_keys() |
448 |
- out.eend(0) |
449 |
- |
450 |
- # we always verify the Manifest signature, in case |
451 |
- # we had to deal with key revocation case |
452 |
- m = gemato.recursiveloader.ManifestRecursiveLoader( |
453 |
- os.path.join(self.repo.location, 'Manifest'), |
454 |
- verify_openpgp=True, |
455 |
- openpgp_env=openpgp_env, |
456 |
- max_jobs=self.verify_jobs) |
457 |
- if not m.openpgp_signed: |
458 |
- raise RuntimeError('OpenPGP signature not found on Manifest') |
459 |
- |
460 |
- ts = m.find_timestamp() |
461 |
- if ts is None: |
462 |
- raise RuntimeError('Timestamp not found in Manifest') |
463 |
- |
464 |
- out.einfo('Manifest timestamp: %s UTC' % (ts.ts,)) |
465 |
- out.einfo('Valid OpenPGP signature found:') |
466 |
- out.einfo('- primary key: %s' % ( |
467 |
- m.openpgp_signature.primary_key_fingerprint)) |
468 |
- out.einfo('- subkey: %s' % ( |
469 |
- m.openpgp_signature.fingerprint)) |
470 |
- out.einfo('- timestamp: %s UTC' % ( |
471 |
- m.openpgp_signature.timestamp)) |
472 |
- |
473 |
- # if nothing has changed, skip the actual Manifest |
474 |
- # verification |
475 |
- if not local_state_unchanged: |
476 |
- out.ebegin('Verifying %s' % (self.repo.location,)) |
477 |
- m.assert_directory_verifies() |
478 |
- out.eend(0) |
479 |
- except GematoException as e: |
480 |
- writemsg_level("!!! Manifest verification failed:\n%s\n" |
481 |
- % (e,), |
482 |
- level=logging.ERROR, noiselevel=-1) |
483 |
- exitcode = 1 |
484 |
- |
485 |
- return (exitcode, updatecache_flg) |
486 |
+ return (exitcode, updatecache_flg) |
487 |
+ finally: |
488 |
+ pass |
489 |
|
490 |
|
491 |
def _process_exitcode(self, exitcode, syncuri, out, maxretries): |
492 |
-- |
493 |
2.16.1 |