1 |
commit: 1e2afc89d0e9ab3060936df8260eb64c5ab5e829 |
2 |
Author: Brian Dolbec <dolsen <AT> gentoo <DOT> org> |
3 |
AuthorDate: Fri May 2 17:04:39 2014 +0000 |
4 |
Commit: Brian Dolbec <brian.dolbec <AT> gmail <DOT> com> |
5 |
CommitDate: Fri May 2 23:12:15 2014 +0000 |
6 |
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=1e2afc89 |
7 |
|
8 |
sync/modules/rsync: Refactor legacy code |
9 |
|
10 |
Split up the _sync() into logical chunks. |
11 |
Replace out my* variables. |
12 |
Move constants out of code, to top of the file. |
13 |
|
14 |
--- |
15 |
pym/portage/sync/modules/rsync/rsync.py | 510 +++++++++++++++++--------------- |
16 |
1 file changed, 269 insertions(+), 241 deletions(-) |
17 |
|
18 |
diff --git a/pym/portage/sync/modules/rsync/rsync.py b/pym/portage/sync/modules/rsync/rsync.py |
19 |
index 0cdf414..e954363 100644 |
20 |
--- a/pym/portage/sync/modules/rsync/rsync.py |
21 |
+++ b/pym/portage/sync/modules/rsync/rsync.py |
22 |
@@ -24,6 +24,10 @@ from _emerge.userquery import userquery |
23 |
from portage.sync.syncbase import SyncBase |
24 |
|
25 |
|
26 |
+SERVER_OUT_OF_DATE = -1 |
27 |
+EXCEEDED_MAX_RETRIES = -2 |
28 |
+ |
29 |
+ |
30 |
class RsyncSync(SyncBase): |
31 |
'''Rsync sync module''' |
32 |
|
33 |
@@ -40,12 +44,11 @@ class RsyncSync(SyncBase): |
34 |
|
35 |
def _sync(self): |
36 |
'''Internal sync function which performs only the sync''' |
37 |
- myopts = self.options.get('emerge_config').opts |
38 |
- usersync_uid = self.options.get('usersync_uid', None) |
39 |
- enter_invalid = '--ask-enter-invalid' in myopts |
40 |
+ opts = self.options.get('emerge_config').opts |
41 |
+ self.usersync_uid = self.options.get('usersync_uid', None) |
42 |
+ enter_invalid = '--ask-enter-invalid' in opts |
43 |
out = portage.output.EOutput() |
44 |
syncuri = self.repo.sync_uri |
45 |
- dosyncuri = syncuri |
46 |
vcs_dirs = frozenset(VCS_DIRS) |
47 |
vcs_dirs = vcs_dirs.intersection(os.listdir(self.repo.location)) |
48 |
|
49 |
@@ -55,97 +58,34 @@ class RsyncSync(SyncBase): |
50 |
"control (contains %s).\n!!! Aborting rsync sync.\n") % \ |
51 |
(self.repo.location, vcs_dir), level=logging.ERROR, noiselevel=-1) |
52 |
return (1, False) |
53 |
- mytimeout=180 |
54 |
+ self.timeout=180 |
55 |
|
56 |
rsync_opts = [] |
57 |
if self.settings["PORTAGE_RSYNC_OPTS"] == "": |
58 |
- portage.writemsg("PORTAGE_RSYNC_OPTS empty or unset, using hardcoded defaults\n") |
59 |
- rsync_opts.extend([ |
60 |
- "--recursive", # Recurse directories |
61 |
- "--links", # Consider symlinks |
62 |
- "--safe-links", # Ignore links outside of tree |
63 |
- "--perms", # Preserve permissions |
64 |
- "--times", # Preserive mod times |
65 |
- "--omit-dir-times", |
66 |
- "--compress", # Compress the data transmitted |
67 |
- "--force", # Force deletion on non-empty dirs |
68 |
- "--whole-file", # Don't do block transfers, only entire files |
69 |
- "--delete", # Delete files that aren't in the master tree |
70 |
- "--stats", # Show final statistics about what was transfered |
71 |
- "--human-readable", |
72 |
- "--timeout="+str(mytimeout), # IO timeout if not done in X seconds |
73 |
- "--exclude=/distfiles", # Exclude distfiles from consideration |
74 |
- "--exclude=/local", # Exclude local from consideration |
75 |
- "--exclude=/packages", # Exclude packages from consideration |
76 |
- ]) |
77 |
- |
78 |
- else: |
79 |
- # The below validation is not needed when using the above hardcoded |
80 |
- # defaults. |
81 |
- |
82 |
- portage.writemsg("Using PORTAGE_RSYNC_OPTS instead of hardcoded defaults\n", 1) |
83 |
- rsync_opts.extend(portage.util.shlex_split( |
84 |
- self.settings.get("PORTAGE_RSYNC_OPTS", ""))) |
85 |
- for opt in ("--recursive", "--times"): |
86 |
- if opt not in rsync_opts: |
87 |
- portage.writemsg(yellow("WARNING:") + " adding required option " + \ |
88 |
- "%s not included in PORTAGE_RSYNC_OPTS\n" % opt) |
89 |
- rsync_opts.append(opt) |
90 |
- |
91 |
- for exclude in ("distfiles", "local", "packages"): |
92 |
- opt = "--exclude=/%s" % exclude |
93 |
- if opt not in rsync_opts: |
94 |
- portage.writemsg(yellow("WARNING:") + \ |
95 |
- " adding required option %s not included in " % opt + \ |
96 |
- "PORTAGE_RSYNC_OPTS (can be overridden with --exclude='!')\n") |
97 |
- rsync_opts.append(opt) |
98 |
- |
99 |
- if syncuri.rstrip("/").endswith(".gentoo.org/gentoo-portage"): |
100 |
- def rsync_opt_startswith(opt_prefix): |
101 |
- for x in rsync_opts: |
102 |
- if x.startswith(opt_prefix): |
103 |
- return (1, False) |
104 |
- return (0, False) |
105 |
- |
106 |
- if not rsync_opt_startswith("--timeout="): |
107 |
- rsync_opts.append("--timeout=%d" % mytimeout) |
108 |
- |
109 |
- for opt in ("--compress", "--whole-file"): |
110 |
- if opt not in rsync_opts: |
111 |
- portage.writemsg(yellow("WARNING:") + " adding required option " + \ |
112 |
- "%s not included in PORTAGE_RSYNC_OPTS\n" % opt) |
113 |
- rsync_opts.append(opt) |
114 |
- |
115 |
- if "--quiet" in myopts: |
116 |
- rsync_opts.append("--quiet") # Shut up a lot |
117 |
+ rsync_opts = self._set_rsync_defaults() |
118 |
else: |
119 |
- rsync_opts.append("--verbose") # Print filelist |
120 |
- |
121 |
- if "--verbose" in myopts: |
122 |
- rsync_opts.append("--progress") # Progress meter for each file |
123 |
- |
124 |
- if "--debug" in myopts: |
125 |
- rsync_opts.append("--checksum") # Force checksum on all files |
126 |
+ rsync_opts = self._validate_rsync_opts(rsync_opts, syncuri) |
127 |
+ self.rsync_opts = self._rsync_opts_extend(opts, rsync_opts) |
128 |
|
129 |
# Real local timestamp file. |
130 |
- servertimestampfile = os.path.join( |
131 |
+ self.servertimestampfile = os.path.join( |
132 |
self.repo.location, "metadata", "timestamp.chk") |
133 |
|
134 |
- content = portage.util.grabfile(servertimestampfile) |
135 |
- mytimestamp = 0 |
136 |
+ content = portage.util.grabfile(self.servertimestampfile) |
137 |
+ timestamp = 0 |
138 |
if content: |
139 |
try: |
140 |
- mytimestamp = time.mktime(time.strptime(content[0], |
141 |
+ timestamp = time.mktime(time.strptime(content[0], |
142 |
TIMESTAMP_FORMAT)) |
143 |
except (OverflowError, ValueError): |
144 |
pass |
145 |
del content |
146 |
|
147 |
try: |
148 |
- rsync_initial_timeout = \ |
149 |
+ self.rsync_initial_timeout = \ |
150 |
int(self.settings.get("PORTAGE_RSYNC_INITIAL_TIMEOUT", "15")) |
151 |
except ValueError: |
152 |
- rsync_initial_timeout = 15 |
153 |
+ self.rsync_initial_timeout = 15 |
154 |
|
155 |
try: |
156 |
maxretries=int(self.settings["PORTAGE_RSYNC_RETRIES"]) |
157 |
@@ -156,7 +96,7 @@ class RsyncSync(SyncBase): |
158 |
|
159 |
retries=0 |
160 |
try: |
161 |
- proto, user_name, hostname, port = re.split( |
162 |
+ self.proto, user_name, hostname, port = re.split( |
163 |
r"(rsync|ssh)://([^:/]+@)?(\[[:\da-fA-F]*\]|[^:/]*)(:[0-9]+)?", |
164 |
syncuri, maxsplit=4)[1:5] |
165 |
except ValueError: |
166 |
@@ -164,7 +104,7 @@ class RsyncSync(SyncBase): |
167 |
noiselevel=-1, level=logging.ERROR) |
168 |
return (1, False) |
169 |
|
170 |
- ssh_opts = self.settings.get("PORTAGE_SSH_OPTS") |
171 |
+ self.ssh_opts = self.settings.get("PORTAGE_SSH_OPTS") |
172 |
|
173 |
if port is None: |
174 |
port="" |
175 |
@@ -176,10 +116,10 @@ class RsyncSync(SyncBase): |
176 |
# getaddrinfo needs the brackets stripped |
177 |
getaddrinfo_host = hostname[1:-1] |
178 |
updatecache_flg=True |
179 |
- all_rsync_opts = set(rsync_opts) |
180 |
- extra_rsync_opts = portage.util.shlex_split( |
181 |
+ all_rsync_opts = set(self.rsync_opts) |
182 |
+ self.extra_rsync_opts = portage.util.shlex_split( |
183 |
self.settings.get("PORTAGE_RSYNC_EXTRA_OPTS","")) |
184 |
- all_rsync_opts.update(extra_rsync_opts) |
185 |
+ all_rsync_opts.update(self.extra_rsync_opts) |
186 |
|
187 |
family = socket.AF_UNSPEC |
188 |
if "-4" in all_rsync_opts or "--ipv4" in all_rsync_opts: |
189 |
@@ -245,8 +185,6 @@ class RsyncSync(SyncBase): |
190 |
if effective_maxretries < 0: |
191 |
effective_maxretries = len(uris) - 1 |
192 |
|
193 |
- SERVER_OUT_OF_DATE = -1 |
194 |
- EXCEEDED_MAX_RETRIES = -2 |
195 |
while (1): |
196 |
if uris: |
197 |
dosyncuri = uris.pop() |
198 |
@@ -256,7 +194,7 @@ class RsyncSync(SyncBase): |
199 |
return (1, False) |
200 |
|
201 |
if (retries==0): |
202 |
- if "--ask" in myopts: |
203 |
+ if "--ask" in opts: |
204 |
if userquery("Do you want to sync your Portage tree " + \ |
205 |
"with the mirror at\n" + blue(dosyncuri) + bold("?"), |
206 |
enter_invalid) == "No": |
207 |
@@ -266,7 +204,7 @@ class RsyncSync(SyncBase): |
208 |
sys.exit(128 + signal.SIGINT) |
209 |
self.logger(self.xterm_titles, |
210 |
">>> Starting rsync with " + dosyncuri) |
211 |
- if "--quiet" not in myopts: |
212 |
+ if "--quiet" not in opts: |
213 |
print(">>> Starting rsync with "+dosyncuri+"...") |
214 |
else: |
215 |
self.logger(self.xterm_titles, |
216 |
@@ -279,160 +217,10 @@ class RsyncSync(SyncBase): |
217 |
if dosyncuri.startswith('ssh://'): |
218 |
dosyncuri = dosyncuri[6:].replace('/', ':/', 1) |
219 |
|
220 |
- if mytimestamp != 0 and "--quiet" not in myopts: |
221 |
- print(">>> Checking server timestamp ...") |
222 |
- |
223 |
- rsynccommand = [self.bin_command] + rsync_opts + extra_rsync_opts |
224 |
- |
225 |
- if proto == 'ssh' and ssh_opts: |
226 |
- rsynccommand.append("--rsh=ssh " + ssh_opts) |
227 |
- |
228 |
- if "--debug" in myopts: |
229 |
- print(rsynccommand) |
230 |
- |
231 |
- exitcode = os.EX_OK |
232 |
- servertimestamp = 0 |
233 |
- # Even if there's no timestamp available locally, fetch the |
234 |
- # timestamp anyway as an initial probe to verify that the server is |
235 |
- # responsive. This protects us from hanging indefinitely on a |
236 |
- # connection attempt to an unresponsive server which rsync's |
237 |
- # --timeout option does not prevent. |
238 |
- if True: |
239 |
- # Temporary file for remote server timestamp comparison. |
240 |
- # NOTE: If FEATURES=usersync is enabled then the tempfile |
241 |
- # needs to be in a directory that's readable by the usersync |
242 |
- # user. We assume that PORTAGE_TMPDIR will satisfy this |
243 |
- # requirement, since that's not necessarily true for the |
244 |
- # default directory used by the tempfile module. |
245 |
- if usersync_uid is not None: |
246 |
- tmpdir = self.settings['PORTAGE_TMPDIR'] |
247 |
- else: |
248 |
- # use default dir from tempfile module |
249 |
- tmpdir = None |
250 |
- fd, tmpservertimestampfile = \ |
251 |
- tempfile.mkstemp(dir=tmpdir) |
252 |
- os.close(fd) |
253 |
- if usersync_uid is not None: |
254 |
- portage.util.apply_permissions(tmpservertimestampfile, |
255 |
- uid=usersync_uid) |
256 |
- mycommand = rsynccommand[:] |
257 |
- mycommand.append(dosyncuri.rstrip("/") + \ |
258 |
- "/metadata/timestamp.chk") |
259 |
- mycommand.append(tmpservertimestampfile) |
260 |
- content = None |
261 |
- mypids = [] |
262 |
- try: |
263 |
- # Timeout here in case the server is unresponsive. The |
264 |
- # --timeout rsync option doesn't apply to the initial |
265 |
- # connection attempt. |
266 |
- try: |
267 |
- if rsync_initial_timeout: |
268 |
- portage.exception.AlarmSignal.register( |
269 |
- rsync_initial_timeout) |
270 |
- |
271 |
- mypids.extend(portage.process.spawn( |
272 |
- mycommand, returnpid=True, |
273 |
- **portage._native_kwargs(self.spawn_kwargs))) |
274 |
- exitcode = os.waitpid(mypids[0], 0)[1] |
275 |
- if usersync_uid is not None: |
276 |
- portage.util.apply_permissions(tmpservertimestampfile, |
277 |
- uid=os.getuid()) |
278 |
- content = portage.grabfile(tmpservertimestampfile) |
279 |
- finally: |
280 |
- if rsync_initial_timeout: |
281 |
- portage.exception.AlarmSignal.unregister() |
282 |
- try: |
283 |
- os.unlink(tmpservertimestampfile) |
284 |
- except OSError: |
285 |
- pass |
286 |
- except portage.exception.AlarmSignal: |
287 |
- # timed out |
288 |
- print('timed out') |
289 |
- # With waitpid and WNOHANG, only check the |
290 |
- # first element of the tuple since the second |
291 |
- # element may vary (bug #337465). |
292 |
- if mypids and os.waitpid(mypids[0], os.WNOHANG)[0] == 0: |
293 |
- os.kill(mypids[0], signal.SIGTERM) |
294 |
- os.waitpid(mypids[0], 0) |
295 |
- # This is the same code rsync uses for timeout. |
296 |
- exitcode = 30 |
297 |
- else: |
298 |
- if exitcode != os.EX_OK: |
299 |
- if exitcode & 0xff: |
300 |
- exitcode = (exitcode & 0xff) << 8 |
301 |
- else: |
302 |
- exitcode = exitcode >> 8 |
303 |
- |
304 |
- if content: |
305 |
- try: |
306 |
- servertimestamp = time.mktime(time.strptime( |
307 |
- content[0], TIMESTAMP_FORMAT)) |
308 |
- except (OverflowError, ValueError): |
309 |
- pass |
310 |
- del mycommand, mypids, content |
311 |
- if exitcode == os.EX_OK: |
312 |
- if (servertimestamp != 0) and (servertimestamp == mytimestamp): |
313 |
- self.logger(self.xterm_titles, |
314 |
- ">>> Cancelling sync -- Already current.") |
315 |
- print() |
316 |
- print(">>>") |
317 |
- print(">>> Timestamps on the server and in the local repository are the same.") |
318 |
- print(">>> Cancelling all further sync action. You are already up to date.") |
319 |
- print(">>>") |
320 |
- print(">>> In order to force sync, remove '%s'." % servertimestampfile) |
321 |
- print(">>>") |
322 |
- print() |
323 |
- return (exitcode, updatecache_flg) |
324 |
- elif (servertimestamp != 0) and (servertimestamp < mytimestamp): |
325 |
- self.logger(self.xterm_titles, |
326 |
- ">>> Server out of date: %s" % dosyncuri) |
327 |
- print() |
328 |
- print(">>>") |
329 |
- print(">>> SERVER OUT OF DATE: %s" % dosyncuri) |
330 |
- print(">>>") |
331 |
- print(">>> In order to force sync, remove '%s'." % servertimestampfile) |
332 |
- print(">>>") |
333 |
- print() |
334 |
- exitcode = SERVER_OUT_OF_DATE |
335 |
- elif (servertimestamp == 0) or (servertimestamp > mytimestamp): |
336 |
- # actual sync |
337 |
- mycommand = rsynccommand + [dosyncuri+"/", self.repo.location] |
338 |
- exitcode = None |
339 |
- try: |
340 |
- exitcode = portage.process.spawn(mycommand, |
341 |
- **portage._native_kwargs(self.spawn_kwargs)) |
342 |
- finally: |
343 |
- if exitcode is None: |
344 |
- # interrupted |
345 |
- exitcode = 128 + signal.SIGINT |
346 |
- |
347 |
- # 0 Success |
348 |
- # 1 Syntax or usage error |
349 |
- # 2 Protocol incompatibility |
350 |
- # 5 Error starting client-server protocol |
351 |
- # 35 Timeout waiting for daemon connection |
352 |
- if exitcode not in (0, 1, 2, 5, 35): |
353 |
- # If the exit code is not among those listed above, |
354 |
- # then we may have a partial/inconsistent sync |
355 |
- # state, so our previously read timestamp as well |
356 |
- # as the corresponding file can no longer be |
357 |
- # trusted. |
358 |
- mytimestamp = 0 |
359 |
- try: |
360 |
- os.unlink(servertimestampfile) |
361 |
- except OSError: |
362 |
- pass |
363 |
- |
364 |
- if exitcode in [0,1,3,4,11,14,20,21]: |
365 |
- break |
366 |
- elif exitcode in [1,3,4,11,14,20,21]: |
367 |
+ |
368 |
+ is_synced, exitcode = self._do_rsync(dosyncuri, timestamp, opts) |
369 |
+ if is_synced: |
370 |
break |
371 |
- else: |
372 |
- # Code 2 indicates protocol incompatibility, which is expected |
373 |
- # for servers with protocol < 29 that don't support |
374 |
- # --prune-empty-directories. Retry for a server that supports |
375 |
- # at least rsync protocol version 29 (>=rsync-2.6.4). |
376 |
- pass |
377 |
|
378 |
retries=retries+1 |
379 |
|
380 |
@@ -444,9 +232,13 @@ class RsyncSync(SyncBase): |
381 |
updatecache_flg=False |
382 |
exitcode = EXCEEDED_MAX_RETRIES |
383 |
break |
384 |
+ self._process_exitcode(exitcode, dosyncuri, out, maxretries) |
385 |
+ return (exitcode, updatecache_flg) |
386 |
+ |
387 |
|
388 |
+ def _process_exitcode(self, exitcode, syncuri, out, maxretries): |
389 |
if (exitcode==0): |
390 |
- self.logger(self.xterm_titles, "=== Sync completed with %s" % dosyncuri) |
391 |
+ self.logger(self.xterm_titles, "=== Sync completed with %s" % syncuri) |
392 |
elif exitcode == SERVER_OUT_OF_DATE: |
393 |
exitcode = 1 |
394 |
elif exitcode == EXCEEDED_MAX_RETRIES: |
395 |
@@ -475,7 +267,6 @@ class RsyncSync(SyncBase): |
396 |
msg.append("(and possibly your system's filesystem) configuration.") |
397 |
for line in msg: |
398 |
out.eerror(line) |
399 |
- return (exitcode, updatecache_flg) |
400 |
|
401 |
|
402 |
def new(self, **kwargs): |
403 |
@@ -490,3 +281,240 @@ class RsyncSync(SyncBase): |
404 |
return (1, False) |
405 |
return self._sync() |
406 |
|
407 |
+ |
408 |
+ def _set_rsync_defaults(self): |
409 |
+ portage.writemsg("PORTAGE_RSYNC_OPTS empty or unset, using hardcoded defaults\n") |
410 |
+ rsync_opts = [ |
411 |
+ "--recursive", # Recurse directories |
412 |
+ "--links", # Consider symlinks |
413 |
+ "--safe-links", # Ignore links outside of tree |
414 |
+ "--perms", # Preserve permissions |
415 |
+ "--times", # Preserive mod times |
416 |
+ "--omit-dir-times", |
417 |
+ "--compress", # Compress the data transmitted |
418 |
+ "--force", # Force deletion on non-empty dirs |
419 |
+ "--whole-file", # Don't do block transfers, only entire files |
420 |
+ "--delete", # Delete files that aren't in the master tree |
421 |
+ "--stats", # Show final statistics about what was transfered |
422 |
+ "--human-readable", |
423 |
+ "--timeout="+str(self.timeout), # IO timeout if not done in X seconds |
424 |
+ "--exclude=/distfiles", # Exclude distfiles from consideration |
425 |
+ "--exclude=/local", # Exclude local from consideration |
426 |
+ "--exclude=/packages", # Exclude packages from consideration |
427 |
+ ] |
428 |
+ return rsync_opts |
429 |
+ |
430 |
+ |
431 |
+ def _validate_rsync_opts(self, rsync_opts, syncuri): |
432 |
+ # The below validation is not needed when using the above hardcoded |
433 |
+ # defaults. |
434 |
+ |
435 |
+ portage.writemsg("Using PORTAGE_RSYNC_OPTS instead of hardcoded defaults\n", 1) |
436 |
+ rsync_opts.extend(portage.util.shlex_split( |
437 |
+ self.settings.get("PORTAGE_RSYNC_OPTS", ""))) |
438 |
+ for opt in ("--recursive", "--times"): |
439 |
+ if opt not in rsync_opts: |
440 |
+ portage.writemsg(yellow("WARNING:") + " adding required option " + \ |
441 |
+ "%s not included in PORTAGE_RSYNC_OPTS\n" % opt) |
442 |
+ rsync_opts.append(opt) |
443 |
+ |
444 |
+ for exclude in ("distfiles", "local", "packages"): |
445 |
+ opt = "--exclude=/%s" % exclude |
446 |
+ if opt not in rsync_opts: |
447 |
+ portage.writemsg(yellow("WARNING:") + \ |
448 |
+ " adding required option %s not included in " % opt + \ |
449 |
+ "PORTAGE_RSYNC_OPTS (can be overridden with --exclude='!')\n") |
450 |
+ rsync_opts.append(opt) |
451 |
+ |
452 |
+ if syncuri.rstrip("/").endswith(".gentoo.org/gentoo-portage"): |
453 |
+ def rsync_opt_startswith(opt_prefix): |
454 |
+ for x in rsync_opts: |
455 |
+ if x.startswith(opt_prefix): |
456 |
+ return (1, False) |
457 |
+ return (0, False) |
458 |
+ |
459 |
+ if not rsync_opt_startswith("--timeout="): |
460 |
+ rsync_opts.append("--timeout=%d" % self.timeout) |
461 |
+ |
462 |
+ for opt in ("--compress", "--whole-file"): |
463 |
+ if opt not in rsync_opts: |
464 |
+ portage.writemsg(yellow("WARNING:") + " adding required option " + \ |
465 |
+ "%s not included in PORTAGE_RSYNC_OPTS\n" % opt) |
466 |
+ rsync_opts.append(opt) |
467 |
+ return rsync_opts |
468 |
+ |
469 |
+ |
470 |
+ @staticmethod |
471 |
+ def _rsync_opts_extend(opts, rsync_opts): |
472 |
+ if "--quiet" in opts: |
473 |
+ rsync_opts.append("--quiet") # Shut up a lot |
474 |
+ else: |
475 |
+ rsync_opts.append("--verbose") # Print filelist |
476 |
+ |
477 |
+ if "--verbose" in opts: |
478 |
+ rsync_opts.append("--progress") # Progress meter for each file |
479 |
+ |
480 |
+ if "--debug" in opts: |
481 |
+ rsync_opts.append("--checksum") # Force checksum on all files |
482 |
+ return rsync_opts |
483 |
+ |
484 |
+ |
485 |
+ def _do_rsync(self, syncuri, timestamp, opts): |
486 |
+ is_synced = False |
487 |
+ if timestamp != 0 and "--quiet" not in opts: |
488 |
+ print(">>> Checking server timestamp ...") |
489 |
+ |
490 |
+ rsynccommand = [self.bin_command] + self.rsync_opts + self.extra_rsync_opts |
491 |
+ |
492 |
+ if self.proto == 'ssh' and self.ssh_opts: |
493 |
+ rsynccommand.append("--rsh=ssh " + self.ssh_opts) |
494 |
+ |
495 |
+ if "--debug" in opts: |
496 |
+ print(rsynccommand) |
497 |
+ |
498 |
+ exitcode = os.EX_OK |
499 |
+ servertimestamp = 0 |
500 |
+ # Even if there's no timestamp available locally, fetch the |
501 |
+ # timestamp anyway as an initial probe to verify that the server is |
502 |
+ # responsive. This protects us from hanging indefinitely on a |
503 |
+ # connection attempt to an unresponsive server which rsync's |
504 |
+ # --timeout option does not prevent. |
505 |
+ |
506 |
+ #if True: |
507 |
+ # Temporary file for remote server timestamp comparison. |
508 |
+ # NOTE: If FEATURES=usersync is enabled then the tempfile |
509 |
+ # needs to be in a directory that's readable by the usersync |
510 |
+ # user. We assume that PORTAGE_TMPDIR will satisfy this |
511 |
+ # requirement, since that's not necessarily true for the |
512 |
+ # default directory used by the tempfile module. |
513 |
+ if self.usersync_uid is not None: |
514 |
+ tmpdir = self.settings['PORTAGE_TMPDIR'] |
515 |
+ else: |
516 |
+ # use default dir from tempfile module |
517 |
+ tmpdir = None |
518 |
+ fd, tmpservertimestampfile = \ |
519 |
+ tempfile.mkstemp(dir=tmpdir) |
520 |
+ os.close(fd) |
521 |
+ if self.usersync_uid is not None: |
522 |
+ portage.util.apply_permissions(tmpservertimestampfile, |
523 |
+ uid=self.usersync_uid) |
524 |
+ command = rsynccommand[:] |
525 |
+ command.append(syncuri.rstrip("/") + \ |
526 |
+ "/metadata/timestamp.chk") |
527 |
+ command.append(tmpservertimestampfile) |
528 |
+ content = None |
529 |
+ pids = [] |
530 |
+ try: |
531 |
+ # Timeout here in case the server is unresponsive. The |
532 |
+ # --timeout rsync option doesn't apply to the initial |
533 |
+ # connection attempt. |
534 |
+ try: |
535 |
+ if self.rsync_initial_timeout: |
536 |
+ portage.exception.AlarmSignal.register( |
537 |
+ self.rsync_initial_timeout) |
538 |
+ |
539 |
+ pids.extend(portage.process.spawn( |
540 |
+ command, returnpid=True, |
541 |
+ **portage._native_kwargs(self.spawn_kwargs))) |
542 |
+ exitcode = os.waitpid(pids[0], 0)[1] |
543 |
+ if self.usersync_uid is not None: |
544 |
+ portage.util.apply_permissions(tmpservertimestampfile, |
545 |
+ uid=os.getuid()) |
546 |
+ content = portage.grabfile(tmpservertimestampfile) |
547 |
+ finally: |
548 |
+ if self.rsync_initial_timeout: |
549 |
+ portage.exception.AlarmSignal.unregister() |
550 |
+ try: |
551 |
+ os.unlink(tmpservertimestampfile) |
552 |
+ except OSError: |
553 |
+ pass |
554 |
+ except portage.exception.AlarmSignal: |
555 |
+ # timed out |
556 |
+ print('timed out') |
557 |
+ # With waitpid and WNOHANG, only check the |
558 |
+ # first element of the tuple since the second |
559 |
+ # element may vary (bug #337465). |
560 |
+ if pids and os.waitpid(pids[0], os.WNOHANG)[0] == 0: |
561 |
+ os.kill(pids[0], signal.SIGTERM) |
562 |
+ os.waitpid(pids[0], 0) |
563 |
+ # This is the same code rsync uses for timeout. |
564 |
+ exitcode = 30 |
565 |
+ else: |
566 |
+ if exitcode != os.EX_OK: |
567 |
+ if exitcode & 0xff: |
568 |
+ exitcode = (exitcode & 0xff) << 8 |
569 |
+ else: |
570 |
+ exitcode = exitcode >> 8 |
571 |
+ |
572 |
+ if content: |
573 |
+ try: |
574 |
+ servertimestamp = time.mktime(time.strptime( |
575 |
+ content[0], TIMESTAMP_FORMAT)) |
576 |
+ except (OverflowError, ValueError): |
577 |
+ pass |
578 |
+ del command, pids, content |
579 |
+ |
580 |
+ if exitcode == os.EX_OK: |
581 |
+ if (servertimestamp != 0) and (servertimestamp == timestamp): |
582 |
+ self.logger(self.xterm_titles, |
583 |
+ ">>> Cancelling sync -- Already current.") |
584 |
+ print() |
585 |
+ print(">>>") |
586 |
+ print(">>> Timestamps on the server and in the local repository are the same.") |
587 |
+ print(">>> Cancelling all further sync action. You are already up to date.") |
588 |
+ print(">>>") |
589 |
+ print(">>> In order to force sync, remove '%s'." % self.servertimestampfile) |
590 |
+ print(">>>") |
591 |
+ print() |
592 |
+ return is_synced, exitcode |
593 |
+ elif (servertimestamp != 0) and (servertimestamp < timestamp): |
594 |
+ self.logger(self.xterm_titles, |
595 |
+ ">>> Server out of date: %s" % syncuri) |
596 |
+ print() |
597 |
+ print(">>>") |
598 |
+ print(">>> SERVER OUT OF DATE: %s" % syncuri) |
599 |
+ print(">>>") |
600 |
+ print(">>> In order to force sync, remove '%s'." % self.servertimestampfile) |
601 |
+ print(">>>") |
602 |
+ print() |
603 |
+ exitcode = SERVER_OUT_OF_DATE |
604 |
+ elif (servertimestamp == 0) or (servertimestamp > timestamp): |
605 |
+ # actual sync |
606 |
+ command = rsynccommand + [syncuri+"/", self.repo.location] |
607 |
+ exitcode = None |
608 |
+ try: |
609 |
+ exitcode = portage.process.spawn(command, |
610 |
+ **portage._native_kwargs(self.spawn_kwargs)) |
611 |
+ finally: |
612 |
+ if exitcode is None: |
613 |
+ # interrupted |
614 |
+ exitcode = 128 + signal.SIGINT |
615 |
+ |
616 |
+ # 0 Success |
617 |
+ # 1 Syntax or usage error |
618 |
+ # 2 Protocol incompatibility |
619 |
+ # 5 Error starting client-server protocol |
620 |
+ # 35 Timeout waiting for daemon connection |
621 |
+ if exitcode not in (0, 1, 2, 5, 35): |
622 |
+ # If the exit code is not among those listed above, |
623 |
+ # then we may have a partial/inconsistent sync |
624 |
+ # state, so our previously read timestamp as well |
625 |
+ # as the corresponding file can no longer be |
626 |
+ # trusted. |
627 |
+ timestamp = 0 |
628 |
+ try: |
629 |
+ os.unlink(self.servertimestampfile) |
630 |
+ except OSError: |
631 |
+ pass |
632 |
+ |
633 |
+ if exitcode in [0,1,3,4,11,14,20,21]: |
634 |
+ is_synced = True |
635 |
+ elif exitcode in [1,3,4,11,14,20,21]: |
636 |
+ is_synced = True |
637 |
+ else: |
638 |
+ # Code 2 indicates protocol incompatibility, which is expected |
639 |
+ # for servers with protocol < 29 that don't support |
640 |
+ # --prune-empty-directories. Retry for a server that supports |
641 |
+ # at least rsync protocol version 29 (>=rsync-2.6.4). |
642 |
+ pass |
643 |
+ return is_synced, exitcode |