1 |
commit: a6bc4fb46ac15d213cc5f5591086ccf03c38db06 |
2 |
Author: Brian Dolbec <dolsen <AT> gentoo <DOT> org> |
3 |
AuthorDate: Fri May 2 17:04:39 2014 +0000 |
4 |
Commit: Brian Dolbec <brian.dolbec <AT> gmail <DOT> com> |
5 |
CommitDate: Mon Sep 29 17:20:20 2014 +0000 |
6 |
URL: http://sources.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=a6bc4fb4 |
7 |
|
8 |
sync/modules/rsync: Refactor legacy code |
9 |
|
10 |
Split up the _sync() into logical chunks. |
11 |
Replace out my* variables. |
12 |
Move constants out of code, to top of the file. |
13 |
|
14 |
--- |
15 |
pym/portage/sync/modules/rsync/rsync.py | 508 +++++++++++++++++--------------- |
16 |
1 file changed, 267 insertions(+), 241 deletions(-) |
17 |
|
18 |
diff --git a/pym/portage/sync/modules/rsync/rsync.py b/pym/portage/sync/modules/rsync/rsync.py |
19 |
index d0ee886..76d83f2 100644 |
20 |
--- a/pym/portage/sync/modules/rsync/rsync.py |
21 |
+++ b/pym/portage/sync/modules/rsync/rsync.py |
22 |
@@ -24,6 +24,10 @@ from _emerge.UserQuery import UserQuery |
23 |
from portage.sync.syncbase import SyncBase |
24 |
|
25 |
|
26 |
+SERVER_OUT_OF_DATE = -1 |
27 |
+EXCEEDED_MAX_RETRIES = -2 |
28 |
+ |
29 |
+ |
30 |
class RsyncSync(SyncBase): |
31 |
'''Rsync sync module''' |
32 |
|
33 |
@@ -40,112 +44,47 @@ class RsyncSync(SyncBase): |
34 |
|
35 |
def _sync(self): |
36 |
'''Internal sync function which performs only the sync''' |
37 |
- myopts = self.options.get('emerge_config').opts |
38 |
- usersync_uid = self.options.get('usersync_uid', None) |
39 |
- enter_invalid = '--ask-enter-invalid' in myopts |
40 |
+ opts = self.options.get('emerge_config').opts |
41 |
+ self.usersync_uid = self.options.get('usersync_uid', None) |
42 |
+ enter_invalid = '--ask-enter-invalid' in opts |
43 |
out = portage.output.EOutput() |
44 |
syncuri = self.repo.sync_uri |
45 |
- dosyncuri = syncuri |
46 |
vcs_dirs = frozenset(VCS_DIRS) |
47 |
vcs_dirs = vcs_dirs.intersection(os.listdir(self.repo.location)) |
48 |
|
49 |
- |
50 |
for vcs_dir in vcs_dirs: |
51 |
writemsg_level(("!!! %s appears to be under revision " + \ |
52 |
"control (contains %s).\n!!! Aborting rsync sync.\n") % \ |
53 |
(self.repo.location, vcs_dir), level=logging.ERROR, noiselevel=-1) |
54 |
return (1, False) |
55 |
- mytimeout=180 |
56 |
+ self.timeout=180 |
57 |
|
58 |
rsync_opts = [] |
59 |
if self.settings["PORTAGE_RSYNC_OPTS"] == "": |
60 |
- portage.writemsg("PORTAGE_RSYNC_OPTS empty or unset, using hardcoded defaults\n") |
61 |
- rsync_opts.extend([ |
62 |
- "--recursive", # Recurse directories |
63 |
- "--links", # Consider symlinks |
64 |
- "--safe-links", # Ignore links outside of tree |
65 |
- "--perms", # Preserve permissions |
66 |
- "--times", # Preserive mod times |
67 |
- "--omit-dir-times", |
68 |
- "--compress", # Compress the data transmitted |
69 |
- "--force", # Force deletion on non-empty dirs |
70 |
- "--whole-file", # Don't do block transfers, only entire files |
71 |
- "--delete", # Delete files that aren't in the master tree |
72 |
- "--stats", # Show final statistics about what was transfered |
73 |
- "--human-readable", |
74 |
- "--timeout="+str(mytimeout), # IO timeout if not done in X seconds |
75 |
- "--exclude=/distfiles", # Exclude distfiles from consideration |
76 |
- "--exclude=/local", # Exclude local from consideration |
77 |
- "--exclude=/packages", # Exclude packages from consideration |
78 |
- ]) |
79 |
- |
80 |
- else: |
81 |
- # The below validation is not needed when using the above hardcoded |
82 |
- # defaults. |
83 |
- |
84 |
- portage.writemsg("Using PORTAGE_RSYNC_OPTS instead of hardcoded defaults\n", 1) |
85 |
- rsync_opts.extend(portage.util.shlex_split( |
86 |
- self.settings.get("PORTAGE_RSYNC_OPTS", ""))) |
87 |
- for opt in ("--recursive", "--times"): |
88 |
- if opt not in rsync_opts: |
89 |
- portage.writemsg(yellow("WARNING:") + " adding required option " + \ |
90 |
- "%s not included in PORTAGE_RSYNC_OPTS\n" % opt) |
91 |
- rsync_opts.append(opt) |
92 |
- |
93 |
- for exclude in ("distfiles", "local", "packages"): |
94 |
- opt = "--exclude=/%s" % exclude |
95 |
- if opt not in rsync_opts: |
96 |
- portage.writemsg(yellow("WARNING:") + \ |
97 |
- " adding required option %s not included in " % opt + \ |
98 |
- "PORTAGE_RSYNC_OPTS (can be overridden with --exclude='!')\n") |
99 |
- rsync_opts.append(opt) |
100 |
- |
101 |
- if syncuri.rstrip("/").endswith(".gentoo.org/gentoo-portage"): |
102 |
- def rsync_opt_startswith(opt_prefix): |
103 |
- for x in rsync_opts: |
104 |
- if x.startswith(opt_prefix): |
105 |
- return (1, False) |
106 |
- return (0, False) |
107 |
- |
108 |
- if not rsync_opt_startswith("--timeout="): |
109 |
- rsync_opts.append("--timeout=%d" % mytimeout) |
110 |
- |
111 |
- for opt in ("--compress", "--whole-file"): |
112 |
- if opt not in rsync_opts: |
113 |
- portage.writemsg(yellow("WARNING:") + " adding required option " + \ |
114 |
- "%s not included in PORTAGE_RSYNC_OPTS\n" % opt) |
115 |
- rsync_opts.append(opt) |
116 |
- |
117 |
- if "--quiet" in myopts: |
118 |
- rsync_opts.append("--quiet") # Shut up a lot |
119 |
+ rsync_opts = self._set_rsync_defaults() |
120 |
else: |
121 |
- rsync_opts.append("--verbose") # Print filelist |
122 |
- |
123 |
- if "--verbose" in myopts: |
124 |
- rsync_opts.append("--progress") # Progress meter for each file |
125 |
- |
126 |
- if "--debug" in myopts: |
127 |
- rsync_opts.append("--checksum") # Force checksum on all files |
128 |
+ rsync_opts = self._validate_rsync_opts(rsync_opts, syncuri) |
129 |
+ self.rsync_opts = self._rsync_opts_extend(opts, rsync_opts) |
130 |
|
131 |
# Real local timestamp file. |
132 |
- servertimestampfile = os.path.join( |
133 |
+ self.servertimestampfile = os.path.join( |
134 |
self.repo.location, "metadata", "timestamp.chk") |
135 |
|
136 |
- content = portage.util.grabfile(servertimestampfile) |
137 |
- mytimestamp = 0 |
138 |
+ content = portage.util.grabfile(self.servertimestampfile) |
139 |
+ timestamp = 0 |
140 |
if content: |
141 |
try: |
142 |
- mytimestamp = time.mktime(time.strptime(content[0], |
143 |
+ timestamp = time.mktime(time.strptime(content[0], |
144 |
TIMESTAMP_FORMAT)) |
145 |
except (OverflowError, ValueError): |
146 |
pass |
147 |
del content |
148 |
|
149 |
try: |
150 |
- rsync_initial_timeout = \ |
151 |
+ self.rsync_initial_timeout = \ |
152 |
int(self.settings.get("PORTAGE_RSYNC_INITIAL_TIMEOUT", "15")) |
153 |
except ValueError: |
154 |
- rsync_initial_timeout = 15 |
155 |
+ self.rsync_initial_timeout = 15 |
156 |
|
157 |
try: |
158 |
maxretries=int(self.settings["PORTAGE_RSYNC_RETRIES"]) |
159 |
@@ -156,7 +95,7 @@ class RsyncSync(SyncBase): |
160 |
|
161 |
retries=0 |
162 |
try: |
163 |
- proto, user_name, hostname, port = re.split( |
164 |
+ self.proto, user_name, hostname, port = re.split( |
165 |
r"(rsync|ssh)://([^:/]+@)?(\[[:\da-fA-F]*\]|[^:/]*)(:[0-9]+)?", |
166 |
syncuri, maxsplit=4)[1:5] |
167 |
except ValueError: |
168 |
@@ -164,7 +103,7 @@ class RsyncSync(SyncBase): |
169 |
noiselevel=-1, level=logging.ERROR) |
170 |
return (1, False) |
171 |
|
172 |
- ssh_opts = self.settings.get("PORTAGE_SSH_OPTS") |
173 |
+ self.ssh_opts = self.settings.get("PORTAGE_SSH_OPTS") |
174 |
|
175 |
if port is None: |
176 |
port="" |
177 |
@@ -176,10 +115,10 @@ class RsyncSync(SyncBase): |
178 |
# getaddrinfo needs the brackets stripped |
179 |
getaddrinfo_host = hostname[1:-1] |
180 |
updatecache_flg=True |
181 |
- all_rsync_opts = set(rsync_opts) |
182 |
- extra_rsync_opts = portage.util.shlex_split( |
183 |
+ all_rsync_opts = set(self.rsync_opts) |
184 |
+ self.extra_rsync_opts = portage.util.shlex_split( |
185 |
self.settings.get("PORTAGE_RSYNC_EXTRA_OPTS","")) |
186 |
- all_rsync_opts.update(extra_rsync_opts) |
187 |
+ all_rsync_opts.update(self.extra_rsync_opts) |
188 |
|
189 |
family = socket.AF_UNSPEC |
190 |
if "-4" in all_rsync_opts or "--ipv4" in all_rsync_opts: |
191 |
@@ -245,8 +184,6 @@ class RsyncSync(SyncBase): |
192 |
if effective_maxretries < 0: |
193 |
effective_maxretries = len(uris) - 1 |
194 |
|
195 |
- SERVER_OUT_OF_DATE = -1 |
196 |
- EXCEEDED_MAX_RETRIES = -2 |
197 |
while (1): |
198 |
if uris: |
199 |
dosyncuri = uris.pop() |
200 |
@@ -267,7 +204,7 @@ class RsyncSync(SyncBase): |
201 |
sys.exit(128 + signal.SIGINT) |
202 |
self.logger(self.xterm_titles, |
203 |
">>> Starting rsync with " + dosyncuri) |
204 |
- if "--quiet" not in myopts: |
205 |
+ if "--quiet" not in opts: |
206 |
print(">>> Starting rsync with "+dosyncuri+"...") |
207 |
else: |
208 |
self.logger(self.xterm_titles, |
209 |
@@ -280,160 +217,9 @@ class RsyncSync(SyncBase): |
210 |
if dosyncuri.startswith('ssh://'): |
211 |
dosyncuri = dosyncuri[6:].replace('/', ':/', 1) |
212 |
|
213 |
- if mytimestamp != 0 and "--quiet" not in myopts: |
214 |
- print(">>> Checking server timestamp ...") |
215 |
- |
216 |
- rsynccommand = [self.bin_command] + rsync_opts + extra_rsync_opts |
217 |
- |
218 |
- if proto == 'ssh' and ssh_opts: |
219 |
- rsynccommand.append("--rsh=ssh " + ssh_opts) |
220 |
- |
221 |
- if "--debug" in myopts: |
222 |
- print(rsynccommand) |
223 |
- |
224 |
- exitcode = os.EX_OK |
225 |
- servertimestamp = 0 |
226 |
- # Even if there's no timestamp available locally, fetch the |
227 |
- # timestamp anyway as an initial probe to verify that the server is |
228 |
- # responsive. This protects us from hanging indefinitely on a |
229 |
- # connection attempt to an unresponsive server which rsync's |
230 |
- # --timeout option does not prevent. |
231 |
- if True: |
232 |
- # Temporary file for remote server timestamp comparison. |
233 |
- # NOTE: If FEATURES=usersync is enabled then the tempfile |
234 |
- # needs to be in a directory that's readable by the usersync |
235 |
- # user. We assume that PORTAGE_TMPDIR will satisfy this |
236 |
- # requirement, since that's not necessarily true for the |
237 |
- # default directory used by the tempfile module. |
238 |
- if usersync_uid is not None: |
239 |
- tmpdir = self.settings['PORTAGE_TMPDIR'] |
240 |
- else: |
241 |
- # use default dir from tempfile module |
242 |
- tmpdir = None |
243 |
- fd, tmpservertimestampfile = \ |
244 |
- tempfile.mkstemp(dir=tmpdir) |
245 |
- os.close(fd) |
246 |
- if usersync_uid is not None: |
247 |
- portage.util.apply_permissions(tmpservertimestampfile, |
248 |
- uid=usersync_uid) |
249 |
- mycommand = rsynccommand[:] |
250 |
- mycommand.append(dosyncuri.rstrip("/") + \ |
251 |
- "/metadata/timestamp.chk") |
252 |
- mycommand.append(tmpservertimestampfile) |
253 |
- content = None |
254 |
- mypids = [] |
255 |
- try: |
256 |
- # Timeout here in case the server is unresponsive. The |
257 |
- # --timeout rsync option doesn't apply to the initial |
258 |
- # connection attempt. |
259 |
- try: |
260 |
- if rsync_initial_timeout: |
261 |
- portage.exception.AlarmSignal.register( |
262 |
- rsync_initial_timeout) |
263 |
- |
264 |
- mypids.extend(portage.process.spawn( |
265 |
- mycommand, returnpid=True, |
266 |
- **portage._native_kwargs(self.spawn_kwargs))) |
267 |
- exitcode = os.waitpid(mypids[0], 0)[1] |
268 |
- if usersync_uid is not None: |
269 |
- portage.util.apply_permissions(tmpservertimestampfile, |
270 |
- uid=os.getuid()) |
271 |
- content = portage.grabfile(tmpservertimestampfile) |
272 |
- finally: |
273 |
- if rsync_initial_timeout: |
274 |
- portage.exception.AlarmSignal.unregister() |
275 |
- try: |
276 |
- os.unlink(tmpservertimestampfile) |
277 |
- except OSError: |
278 |
- pass |
279 |
- except portage.exception.AlarmSignal: |
280 |
- # timed out |
281 |
- print('timed out') |
282 |
- # With waitpid and WNOHANG, only check the |
283 |
- # first element of the tuple since the second |
284 |
- # element may vary (bug #337465). |
285 |
- if mypids and os.waitpid(mypids[0], os.WNOHANG)[0] == 0: |
286 |
- os.kill(mypids[0], signal.SIGTERM) |
287 |
- os.waitpid(mypids[0], 0) |
288 |
- # This is the same code rsync uses for timeout. |
289 |
- exitcode = 30 |
290 |
- else: |
291 |
- if exitcode != os.EX_OK: |
292 |
- if exitcode & 0xff: |
293 |
- exitcode = (exitcode & 0xff) << 8 |
294 |
- else: |
295 |
- exitcode = exitcode >> 8 |
296 |
- |
297 |
- if content: |
298 |
- try: |
299 |
- servertimestamp = time.mktime(time.strptime( |
300 |
- content[0], TIMESTAMP_FORMAT)) |
301 |
- except (OverflowError, ValueError): |
302 |
- pass |
303 |
- del mycommand, mypids, content |
304 |
- if exitcode == os.EX_OK: |
305 |
- if (servertimestamp != 0) and (servertimestamp == mytimestamp): |
306 |
- self.logger(self.xterm_titles, |
307 |
- ">>> Cancelling sync -- Already current.") |
308 |
- print() |
309 |
- print(">>>") |
310 |
- print(">>> Timestamps on the server and in the local repository are the same.") |
311 |
- print(">>> Cancelling all further sync action. You are already up to date.") |
312 |
- print(">>>") |
313 |
- print(">>> In order to force sync, remove '%s'." % servertimestampfile) |
314 |
- print(">>>") |
315 |
- print() |
316 |
- return (exitcode, updatecache_flg) |
317 |
- elif (servertimestamp != 0) and (servertimestamp < mytimestamp): |
318 |
- self.logger(self.xterm_titles, |
319 |
- ">>> Server out of date: %s" % dosyncuri) |
320 |
- print() |
321 |
- print(">>>") |
322 |
- print(">>> SERVER OUT OF DATE: %s" % dosyncuri) |
323 |
- print(">>>") |
324 |
- print(">>> In order to force sync, remove '%s'." % servertimestampfile) |
325 |
- print(">>>") |
326 |
- print() |
327 |
- exitcode = SERVER_OUT_OF_DATE |
328 |
- elif (servertimestamp == 0) or (servertimestamp > mytimestamp): |
329 |
- # actual sync |
330 |
- mycommand = rsynccommand + [dosyncuri+"/", self.repo.location] |
331 |
- exitcode = None |
332 |
- try: |
333 |
- exitcode = portage.process.spawn(mycommand, |
334 |
- **portage._native_kwargs(self.spawn_kwargs)) |
335 |
- finally: |
336 |
- if exitcode is None: |
337 |
- # interrupted |
338 |
- exitcode = 128 + signal.SIGINT |
339 |
- |
340 |
- # 0 Success |
341 |
- # 1 Syntax or usage error |
342 |
- # 2 Protocol incompatibility |
343 |
- # 5 Error starting client-server protocol |
344 |
- # 35 Timeout waiting for daemon connection |
345 |
- if exitcode not in (0, 1, 2, 5, 35): |
346 |
- # If the exit code is not among those listed above, |
347 |
- # then we may have a partial/inconsistent sync |
348 |
- # state, so our previously read timestamp as well |
349 |
- # as the corresponding file can no longer be |
350 |
- # trusted. |
351 |
- mytimestamp = 0 |
352 |
- try: |
353 |
- os.unlink(servertimestampfile) |
354 |
- except OSError: |
355 |
- pass |
356 |
- |
357 |
- if exitcode in [0,1,3,4,11,14,20,21]: |
358 |
- break |
359 |
- elif exitcode in [1,3,4,11,14,20,21]: |
360 |
+ is_synced, exitcode = self._do_rsync(dosyncuri, timestamp, opts) |
361 |
+ if is_synced: |
362 |
break |
363 |
- else: |
364 |
- # Code 2 indicates protocol incompatibility, which is expected |
365 |
- # for servers with protocol < 29 that don't support |
366 |
- # --prune-empty-directories. Retry for a server that supports |
367 |
- # at least rsync protocol version 29 (>=rsync-2.6.4). |
368 |
- pass |
369 |
|
370 |
retries=retries+1 |
371 |
|
372 |
@@ -445,9 +231,13 @@ class RsyncSync(SyncBase): |
373 |
updatecache_flg=False |
374 |
exitcode = EXCEEDED_MAX_RETRIES |
375 |
break |
376 |
+ self._process_exitcode(exitcode, dosyncuri, out, maxretries) |
377 |
+ return (exitcode, updatecache_flg) |
378 |
+ |
379 |
|
380 |
+ def _process_exitcode(self, exitcode, syncuri, out, maxretries): |
381 |
if (exitcode==0): |
382 |
- self.logger(self.xterm_titles, "=== Sync completed with %s" % dosyncuri) |
383 |
+ self.logger(self.xterm_titles, "=== Sync completed with %s" % syncuri) |
384 |
elif exitcode == SERVER_OUT_OF_DATE: |
385 |
exitcode = 1 |
386 |
elif exitcode == EXCEEDED_MAX_RETRIES: |
387 |
@@ -476,7 +266,6 @@ class RsyncSync(SyncBase): |
388 |
msg.append("(and possibly your system's filesystem) configuration.") |
389 |
for line in msg: |
390 |
out.eerror(line) |
391 |
- return (exitcode, updatecache_flg) |
392 |
|
393 |
|
394 |
def new(self, **kwargs): |
395 |
@@ -491,3 +280,240 @@ class RsyncSync(SyncBase): |
396 |
return (1, False) |
397 |
return self._sync() |
398 |
|
399 |
+ |
400 |
+ def _set_rsync_defaults(self): |
401 |
+ portage.writemsg("PORTAGE_RSYNC_OPTS empty or unset, using hardcoded defaults\n") |
402 |
+ rsync_opts = [ |
403 |
+ "--recursive", # Recurse directories |
404 |
+ "--links", # Consider symlinks |
405 |
+ "--safe-links", # Ignore links outside of tree |
406 |
+ "--perms", # Preserve permissions |
407 |
+ "--times", # Preserive mod times |
408 |
+ "--omit-dir-times", |
409 |
+ "--compress", # Compress the data transmitted |
410 |
+ "--force", # Force deletion on non-empty dirs |
411 |
+ "--whole-file", # Don't do block transfers, only entire files |
412 |
+ "--delete", # Delete files that aren't in the master tree |
413 |
+ "--stats", # Show final statistics about what was transfered |
414 |
+ "--human-readable", |
415 |
+ "--timeout="+str(self.timeout), # IO timeout if not done in X seconds |
416 |
+ "--exclude=/distfiles", # Exclude distfiles from consideration |
417 |
+ "--exclude=/local", # Exclude local from consideration |
418 |
+ "--exclude=/packages", # Exclude packages from consideration |
419 |
+ ] |
420 |
+ return rsync_opts |
421 |
+ |
422 |
+ |
423 |
+ def _validate_rsync_opts(self, rsync_opts, syncuri): |
424 |
+ # The below validation is not needed when using the above hardcoded |
425 |
+ # defaults. |
426 |
+ |
427 |
+ portage.writemsg("Using PORTAGE_RSYNC_OPTS instead of hardcoded defaults\n", 1) |
428 |
+ rsync_opts.extend(portage.util.shlex_split( |
429 |
+ self.settings.get("PORTAGE_RSYNC_OPTS", ""))) |
430 |
+ for opt in ("--recursive", "--times"): |
431 |
+ if opt not in rsync_opts: |
432 |
+ portage.writemsg(yellow("WARNING:") + " adding required option " + \ |
433 |
+ "%s not included in PORTAGE_RSYNC_OPTS\n" % opt) |
434 |
+ rsync_opts.append(opt) |
435 |
+ |
436 |
+ for exclude in ("distfiles", "local", "packages"): |
437 |
+ opt = "--exclude=/%s" % exclude |
438 |
+ if opt not in rsync_opts: |
439 |
+ portage.writemsg(yellow("WARNING:") + \ |
440 |
+ " adding required option %s not included in " % opt + \ |
441 |
+ "PORTAGE_RSYNC_OPTS (can be overridden with --exclude='!')\n") |
442 |
+ rsync_opts.append(opt) |
443 |
+ |
444 |
+ if syncuri.rstrip("/").endswith(".gentoo.org/gentoo-portage"): |
445 |
+ def rsync_opt_startswith(opt_prefix): |
446 |
+ for x in rsync_opts: |
447 |
+ if x.startswith(opt_prefix): |
448 |
+ return (1, False) |
449 |
+ return (0, False) |
450 |
+ |
451 |
+ if not rsync_opt_startswith("--timeout="): |
452 |
+ rsync_opts.append("--timeout=%d" % self.timeout) |
453 |
+ |
454 |
+ for opt in ("--compress", "--whole-file"): |
455 |
+ if opt not in rsync_opts: |
456 |
+ portage.writemsg(yellow("WARNING:") + " adding required option " + \ |
457 |
+ "%s not included in PORTAGE_RSYNC_OPTS\n" % opt) |
458 |
+ rsync_opts.append(opt) |
459 |
+ return rsync_opts |
460 |
+ |
461 |
+ |
462 |
+ @staticmethod |
463 |
+ def _rsync_opts_extend(opts, rsync_opts): |
464 |
+ if "--quiet" in opts: |
465 |
+ rsync_opts.append("--quiet") # Shut up a lot |
466 |
+ else: |
467 |
+ rsync_opts.append("--verbose") # Print filelist |
468 |
+ |
469 |
+ if "--verbose" in opts: |
470 |
+ rsync_opts.append("--progress") # Progress meter for each file |
471 |
+ |
472 |
+ if "--debug" in opts: |
473 |
+ rsync_opts.append("--checksum") # Force checksum on all files |
474 |
+ return rsync_opts |
475 |
+ |
476 |
+ |
477 |
+ def _do_rsync(self, syncuri, timestamp, opts): |
478 |
+ is_synced = False |
479 |
+ if timestamp != 0 and "--quiet" not in opts: |
480 |
+ print(">>> Checking server timestamp ...") |
481 |
+ |
482 |
+ rsynccommand = [self.bin_command] + self.rsync_opts + self.extra_rsync_opts |
483 |
+ |
484 |
+ if self.proto == 'ssh' and self.ssh_opts: |
485 |
+ rsynccommand.append("--rsh=ssh " + self.ssh_opts) |
486 |
+ |
487 |
+ if "--debug" in opts: |
488 |
+ print(rsynccommand) |
489 |
+ |
490 |
+ exitcode = os.EX_OK |
491 |
+ servertimestamp = 0 |
492 |
+ # Even if there's no timestamp available locally, fetch the |
493 |
+ # timestamp anyway as an initial probe to verify that the server is |
494 |
+ # responsive. This protects us from hanging indefinitely on a |
495 |
+ # connection attempt to an unresponsive server which rsync's |
496 |
+ # --timeout option does not prevent. |
497 |
+ |
498 |
+ #if True: |
499 |
+ # Temporary file for remote server timestamp comparison. |
500 |
+ # NOTE: If FEATURES=usersync is enabled then the tempfile |
501 |
+ # needs to be in a directory that's readable by the usersync |
502 |
+ # user. We assume that PORTAGE_TMPDIR will satisfy this |
503 |
+ # requirement, since that's not necessarily true for the |
504 |
+ # default directory used by the tempfile module. |
505 |
+ if self.usersync_uid is not None: |
506 |
+ tmpdir = self.settings['PORTAGE_TMPDIR'] |
507 |
+ else: |
508 |
+ # use default dir from tempfile module |
509 |
+ tmpdir = None |
510 |
+ fd, tmpservertimestampfile = \ |
511 |
+ tempfile.mkstemp(dir=tmpdir) |
512 |
+ os.close(fd) |
513 |
+ if self.usersync_uid is not None: |
514 |
+ portage.util.apply_permissions(tmpservertimestampfile, |
515 |
+ uid=self.usersync_uid) |
516 |
+ command = rsynccommand[:] |
517 |
+ command.append(syncuri.rstrip("/") + \ |
518 |
+ "/metadata/timestamp.chk") |
519 |
+ command.append(tmpservertimestampfile) |
520 |
+ content = None |
521 |
+ pids = [] |
522 |
+ try: |
523 |
+ # Timeout here in case the server is unresponsive. The |
524 |
+ # --timeout rsync option doesn't apply to the initial |
525 |
+ # connection attempt. |
526 |
+ try: |
527 |
+ if self.rsync_initial_timeout: |
528 |
+ portage.exception.AlarmSignal.register( |
529 |
+ self.rsync_initial_timeout) |
530 |
+ |
531 |
+ pids.extend(portage.process.spawn( |
532 |
+ command, returnpid=True, |
533 |
+ **portage._native_kwargs(self.spawn_kwargs))) |
534 |
+ exitcode = os.waitpid(pids[0], 0)[1] |
535 |
+ if self.usersync_uid is not None: |
536 |
+ portage.util.apply_permissions(tmpservertimestampfile, |
537 |
+ uid=os.getuid()) |
538 |
+ content = portage.grabfile(tmpservertimestampfile) |
539 |
+ finally: |
540 |
+ if self.rsync_initial_timeout: |
541 |
+ portage.exception.AlarmSignal.unregister() |
542 |
+ try: |
543 |
+ os.unlink(tmpservertimestampfile) |
544 |
+ except OSError: |
545 |
+ pass |
546 |
+ except portage.exception.AlarmSignal: |
547 |
+ # timed out |
548 |
+ print('timed out') |
549 |
+ # With waitpid and WNOHANG, only check the |
550 |
+ # first element of the tuple since the second |
551 |
+ # element may vary (bug #337465). |
552 |
+ if pids and os.waitpid(pids[0], os.WNOHANG)[0] == 0: |
553 |
+ os.kill(pids[0], signal.SIGTERM) |
554 |
+ os.waitpid(pids[0], 0) |
555 |
+ # This is the same code rsync uses for timeout. |
556 |
+ exitcode = 30 |
557 |
+ else: |
558 |
+ if exitcode != os.EX_OK: |
559 |
+ if exitcode & 0xff: |
560 |
+ exitcode = (exitcode & 0xff) << 8 |
561 |
+ else: |
562 |
+ exitcode = exitcode >> 8 |
563 |
+ |
564 |
+ if content: |
565 |
+ try: |
566 |
+ servertimestamp = time.mktime(time.strptime( |
567 |
+ content[0], TIMESTAMP_FORMAT)) |
568 |
+ except (OverflowError, ValueError): |
569 |
+ pass |
570 |
+ del command, pids, content |
571 |
+ |
572 |
+ if exitcode == os.EX_OK: |
573 |
+ if (servertimestamp != 0) and (servertimestamp == timestamp): |
574 |
+ self.logger(self.xterm_titles, |
575 |
+ ">>> Cancelling sync -- Already current.") |
576 |
+ print() |
577 |
+ print(">>>") |
578 |
+ print(">>> Timestamps on the server and in the local repository are the same.") |
579 |
+ print(">>> Cancelling all further sync action. You are already up to date.") |
580 |
+ print(">>>") |
581 |
+ print(">>> In order to force sync, remove '%s'." % self.servertimestampfile) |
582 |
+ print(">>>") |
583 |
+ print() |
584 |
+ return is_synced, exitcode |
585 |
+ elif (servertimestamp != 0) and (servertimestamp < timestamp): |
586 |
+ self.logger(self.xterm_titles, |
587 |
+ ">>> Server out of date: %s" % syncuri) |
588 |
+ print() |
589 |
+ print(">>>") |
590 |
+ print(">>> SERVER OUT OF DATE: %s" % syncuri) |
591 |
+ print(">>>") |
592 |
+ print(">>> In order to force sync, remove '%s'." % self.servertimestampfile) |
593 |
+ print(">>>") |
594 |
+ print() |
595 |
+ exitcode = SERVER_OUT_OF_DATE |
596 |
+ elif (servertimestamp == 0) or (servertimestamp > timestamp): |
597 |
+ # actual sync |
598 |
+ command = rsynccommand + [syncuri+"/", self.repo.location] |
599 |
+ exitcode = None |
600 |
+ try: |
601 |
+ exitcode = portage.process.spawn(command, |
602 |
+ **portage._native_kwargs(self.spawn_kwargs)) |
603 |
+ finally: |
604 |
+ if exitcode is None: |
605 |
+ # interrupted |
606 |
+ exitcode = 128 + signal.SIGINT |
607 |
+ |
608 |
+ # 0 Success |
609 |
+ # 1 Syntax or usage error |
610 |
+ # 2 Protocol incompatibility |
611 |
+ # 5 Error starting client-server protocol |
612 |
+ # 35 Timeout waiting for daemon connection |
613 |
+ if exitcode not in (0, 1, 2, 5, 35): |
614 |
+ # If the exit code is not among those listed above, |
615 |
+ # then we may have a partial/inconsistent sync |
616 |
+ # state, so our previously read timestamp as well |
617 |
+ # as the corresponding file can no longer be |
618 |
+ # trusted. |
619 |
+ timestamp = 0 |
620 |
+ try: |
621 |
+ os.unlink(self.servertimestampfile) |
622 |
+ except OSError: |
623 |
+ pass |
624 |
+ |
625 |
+ if exitcode in [0,1,3,4,11,14,20,21]: |
626 |
+ is_synced = True |
627 |
+ elif exitcode in [1,3,4,11,14,20,21]: |
628 |
+ is_synced = True |
629 |
+ else: |
630 |
+ # Code 2 indicates protocol incompatibility, which is expected |
631 |
+ # for servers with protocol < 29 that don't support |
632 |
+ # --prune-empty-directories. Retry for a server that supports |
633 |
+ # at least rsync protocol version 29 (>=rsync-2.6.4). |
634 |
+ pass |
635 |
+ return is_synced, exitcode |