1 |
commit: 2383f21652384458d0fc7a05a0c70610f22719d3 |
2 |
Author: André Erdmann <dywi <AT> mailerd <DOT> de> |
3 |
AuthorDate: Tue Jun 26 15:37:24 2012 +0000 |
4 |
Commit: André Erdmann <dywi <AT> mailerd <DOT> de> |
5 |
CommitDate: Tue Jun 26 15:37:24 2012 +0000 |
6 |
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=2383f216 |
7 |
|
8 |
remote |
9 |
|
10 |
* can now be used to feed the overlay creation package queue |
11 |
|
12 |
modified: roverlay/remote/basicrepo.py |
13 |
modified: roverlay/remote/repo.py |
14 |
modified: roverlay/remote/repolist.py |
15 |
modified: roverlay/remote/repoloader.py |
16 |
modified: roverlay/remote/rsync.py |
17 |
|
18 |
--- |
19 |
roverlay/remote/basicrepo.py | 70 +++++++++++++++++++++++++++------- |
20 |
roverlay/remote/repo.py | 18 +++++--- |
21 |
roverlay/remote/repolist.py | 78 +++++++++++++++++++++++++++++--------- |
22 |
roverlay/remote/repoloader.py | 7 +++- |
23 |
roverlay/remote/rsync.py | 83 ++++++++++++++++++++++++++++------------ |
24 |
5 files changed, 190 insertions(+), 66 deletions(-) |
25 |
|
26 |
diff --git a/roverlay/remote/basicrepo.py b/roverlay/remote/basicrepo.py |
27 |
index 9ade3a2..1574850 100644 |
28 |
--- a/roverlay/remote/basicrepo.py |
29 |
+++ b/roverlay/remote/basicrepo.py |
30 |
@@ -8,6 +8,11 @@ DEFAULT_PROTOCOL = 'http' |
31 |
|
32 |
LOCALREPO_SRC_URI = 'http://localhost/R-Packages' |
33 |
|
34 |
+SYNC_SUCCESS = 1 |
35 |
+SYNC_FAIL = 2 |
36 |
+REPO_READY = 4 |
37 |
+ |
38 |
+ |
39 |
def normalize_uri ( uri, protocol, force_protocol=False ): |
40 |
|
41 |
if not protocol: |
42 |
@@ -53,8 +58,29 @@ class LocalRepo ( object ): |
43 |
else: |
44 |
self.src_uri = src_uri |
45 |
|
46 |
+ self.sync_status = 0 |
47 |
+ |
48 |
# --- end of __init__ (...) --- |
49 |
|
50 |
+ def ready ( self ): |
51 |
+ return bool ( self.sync_status & REPO_READY ) |
52 |
+ |
53 |
+ def fail ( self ): |
54 |
+ return bool ( self.sync_status & SYNC_FAIL ) |
55 |
+ |
56 |
+ def offline ( self ): |
57 |
+ return 0 == self.sync_status & SYNC_SUCCESS |
58 |
+ |
59 |
+ def _set_ready ( self, is_synced ): |
60 |
+ """comment TODO""" |
61 |
+ if is_synced: |
62 |
+ self.sync_status = SYNC_SUCCESS | REPO_READY |
63 |
+ else: |
64 |
+ self.sync_status = REPO_READY |
65 |
+ |
66 |
+ def _set_fail ( self ): |
67 |
+ self.sync_status = SYNC_FAIL |
68 |
+ |
69 |
def __str__ ( self ): |
70 |
return "repo '%s': DISTDIR '%s', SRC_URI '%s'" % ( |
71 |
self.name, self.distdir, self.src_uri |
72 |
@@ -79,7 +105,7 @@ class LocalRepo ( object ): |
73 |
if package_file is None: |
74 |
return self.src_uri |
75 |
else: |
76 |
- return '/'.join ( self.src_uri, package_file ) |
77 |
+ return '/'.join ( ( self.src_uri, package_file ) ) |
78 |
# --- end of get_src_uri (...) --- |
79 |
|
80 |
# get_src(...) -> get_src_uri(...) |
81 |
@@ -90,17 +116,26 @@ class LocalRepo ( object ): |
82 |
return os.path.isdir ( self.distdir ) |
83 |
# --- end of exists (...) --- |
84 |
|
85 |
- def nosync ( self ): |
86 |
- """Returns True if the repo is ready for overlay creation, else False. |
87 |
- Useful for basic local distfiles verification without downloading |
88 |
- anything. |
89 |
- """ |
90 |
- return self.exists() |
91 |
+ def sync ( self, sync_enabled=True ): |
92 |
+ """Syncs this repo.""" |
93 |
|
94 |
- # --- end of nosync (...) --- |
95 |
+ status = False |
96 |
+ if sync_enabled and hasattr ( self, '_dosync' ): |
97 |
+ status = self._dosync() |
98 |
|
99 |
- # sync() -> nosync(), LocalRepos don't have anything to sync |
100 |
- sync = nosync |
101 |
+ elif hasattr ( self, '_nosync'): |
102 |
+ status = self._nosync() |
103 |
+ |
104 |
+ else: |
105 |
+ status = self.exists() |
106 |
+ |
107 |
+ if status: |
108 |
+ self._set_ready ( is_synced=sync_enabled ) |
109 |
+ else: |
110 |
+ self._set_fail() |
111 |
+ |
112 |
+ return status |
113 |
+ # --- end of sync (...) --- |
114 |
|
115 |
def scan_distdir ( self, is_package=None ): |
116 |
"""Generator that scans the local distfiles dir of this repo and |
117 |
@@ -111,19 +146,26 @@ class LocalRepo ( object ): |
118 |
or None which means that all files are packages. |
119 |
Defaults to None. |
120 |
""" |
121 |
+ |
122 |
+ kw = { 'origin' : self } |
123 |
+ |
124 |
if is_package is None: |
125 |
# unfiltered variant |
126 |
|
127 |
for dirpath, dirnames, filenames in os.walk ( self.distdir ): |
128 |
+ kw ['distdir'] = dirpath if dirpath != self.distdir else None |
129 |
+ |
130 |
for pkg in filenames: |
131 |
- yield PackageInfo ( filename=pkg, origin=self ) |
132 |
+ yield PackageInfo ( filename=pkg, **kw ) |
133 |
|
134 |
elif hasattr ( is_package, '__call__' ): |
135 |
# filtered variant (adds an if is_package... before yield) |
136 |
for dirpath, dirnames, filenames in os.walk ( self.distdir ): |
137 |
+ kw ['distdir'] = dirpath if dirpath != self.distdir else None |
138 |
+ |
139 |
for pkg in filenames: |
140 |
if is_package ( os.path.join ( dirpath, pkg ) ): |
141 |
- yield PackageInfo ( filename=pkg, origin=self ) |
142 |
+ yield PackageInfo ( filename=pkg, **kw ) |
143 |
|
144 |
|
145 |
else: |
146 |
@@ -227,14 +269,14 @@ class RemoteRepo ( LocalRepo ): |
147 |
# get_remote(...) -> get_remote_uri(...) |
148 |
get_remote = get_remote_uri |
149 |
|
150 |
- def sync ( self ): |
151 |
+ def _dosync ( self ): |
152 |
"""Gets packages from remote(s) and returns True if the repo is ready |
153 |
for overlay creation, else False. |
154 |
|
155 |
Derived classes have to implement this method. |
156 |
""" |
157 |
raise Exception ( "RemoteRepo does not implement sync()." ) |
158 |
- # --- end of sync (...) --- |
159 |
+ # --- end of _dosync (...) --- |
160 |
|
161 |
def __str__ ( self ): |
162 |
return "repo '%s': DISTDIR '%s', SRC_URI '%s', REMOTE_URI '%s'" % ( |
163 |
|
164 |
diff --git a/roverlay/remote/repo.py b/roverlay/remote/repo.py |
165 |
index f54f448..febfaae 100644 |
166 |
--- a/roverlay/remote/repo.py |
167 |
+++ b/roverlay/remote/repo.py |
168 |
@@ -11,7 +11,7 @@ class RsyncRepo ( RemoteRepo ): |
169 |
def __init__ ( |
170 |
self, name, |
171 |
directory=None, src_uri=None, rsync_uri=None, base_uri=None, |
172 |
- extra_rsync_opts=None |
173 |
+ **rsync_kw |
174 |
): |
175 |
# super's init: name, remote protocol, directory_kw, **uri_kw |
176 |
# using '' as remote protocol which leaves uris unchanged when |
177 |
@@ -20,19 +20,23 @@ class RsyncRepo ( RemoteRepo ): |
178 |
name, '', directory=directory, |
179 |
src_uri=src_uri, remote_uri=rsync_uri, base_uri=base_uri |
180 |
) |
181 |
- self.extra_rsync_opts = extra_rsync_opts |
182 |
+ self.rsync_extra = rsync_kw |
183 |
+ |
184 |
+ self.sync_protocol = 'rsync' |
185 |
# --- end of __init__ (...) --- |
186 |
|
187 |
|
188 |
- def sync ( self ): |
189 |
+ def _dosync ( self ): |
190 |
retcode = None |
191 |
try: |
192 |
job = RsyncJob ( |
193 |
remote=self.remote_uri, distdir=self.distdir, |
194 |
run_now=True, |
195 |
- extra_opts=self.extra_rsync_opts |
196 |
+ **self.rsync_extra |
197 |
) |
198 |
- if job.returncode == 0: return True |
199 |
+ if job.returncode == 0: |
200 |
+ self._set_ready ( is_synced=True ) |
201 |
+ return True |
202 |
|
203 |
retcode = job.returncode |
204 |
except Exception as e: |
205 |
@@ -41,13 +45,13 @@ class RsyncRepo ( RemoteRepo ): |
206 |
logging.exception ( e ) |
207 |
retcode = '<undef>' |
208 |
|
209 |
- |
210 |
logging.error ( |
211 |
'Repo %s cannot be used for ebuild creation due to errors ' |
212 |
'while running rsync (return code was %s).' % ( self.name, retcode ) |
213 |
) |
214 |
+ self._set_fail() |
215 |
return False |
216 |
- # --- end of sync (...) --- |
217 |
+ # --- end of _dosync (...) --- |
218 |
|
219 |
def __str__ ( self ): |
220 |
return "rsync repo '%s': DISTDIR '%s', SRC_URI '%s', RSYNC_URI '%s'" \ |
221 |
|
222 |
diff --git a/roverlay/remote/repolist.py b/roverlay/remote/repolist.py |
223 |
index 4617057..ae740dd 100644 |
224 |
--- a/roverlay/remote/repolist.py |
225 |
+++ b/roverlay/remote/repolist.py |
226 |
@@ -1,13 +1,19 @@ |
227 |
+import logging |
228 |
|
229 |
from roverlay import config |
230 |
- |
231 |
from roverlay.remote.repoloader import read_repofile |
232 |
|
233 |
+LOGGER = logging.getLogger ( 'RepoList' ) |
234 |
+ |
235 |
class RepoList ( object ): |
236 |
|
237 |
def __init__ ( self ): |
238 |
self.repos = list() |
239 |
+ |
240 |
self.sync_enabled = True |
241 |
+ |
242 |
+ # if True: use all repos when looking for packages, even those that |
243 |
+ # could not be synced |
244 |
self.use_broken_repos = False |
245 |
|
246 |
def sort ( self ): |
247 |
@@ -25,28 +31,62 @@ class RepoList ( object ): |
248 |
self.load_file ( f ) |
249 |
# --- end of load (...) --- |
250 |
|
251 |
- def sync_all ( self, package_queue=None ): |
252 |
- q = None |
253 |
- if package_queue is None: |
254 |
- q = list() |
255 |
- add = q.append |
256 |
- else: |
257 |
- # TODO: _nowait? raises Exception when queue is full which is |
258 |
- # good in non-threaded execution |
259 |
- # -> timeout,.. |
260 |
- add = q.put |
261 |
+ def _queue_packages_from_repo ( self, repo, add_method ): |
262 |
+ if not repo.ready(): |
263 |
+ if self.use_broken_repos: |
264 |
+ # warn and continue |
265 |
+ pass |
266 |
+ else: |
267 |
+ # repo cannot be used |
268 |
+ LOGGER.warning ( "!!" ) |
269 |
+ return False |
270 |
+ |
271 |
+ for p in repo.scan_distdir(): |
272 |
+ LOGGER.debug ( "adding package %s from repo %s" % ( p, repo ) ) |
273 |
+ add_method ( p ) |
274 |
+ # --- end of _queue_packages_from_repo (...) --- |
275 |
+ |
276 |
+ def add_packages ( self, add_method ): |
277 |
+ for repo in self.repos: |
278 |
+ self._queue_packages_from_repo ( repo, add_method ) |
279 |
+ # --- end of add_packages (...) --- |
280 |
+ |
281 |
+ def _sync_all_repos_and_run ( |
282 |
+ self, |
283 |
+ when_repo_success=None, when_repo_fail=None, when_repo_done=None, |
284 |
+ when_all_done=None |
285 |
+ ): |
286 |
+ try_call = lambda f, *x, **z : None if f is None else f ( *x, **z ) |
287 |
+ |
288 |
+ LOGGER.debug ( "Syncing repos ..." ) |
289 |
+ for repo in self.repos: |
290 |
+ if repo.sync ( sync_enabled=self.sync_enabled ): |
291 |
+ # repo successfully synced |
292 |
+ try_call ( when_repo_success, repo ) |
293 |
+ else: |
294 |
+ # else log fail <> |
295 |
+ try_call ( when_repo_fail, repo ) |
296 |
|
297 |
+ try_call ( when_repo_done, repo ) |
298 |
|
299 |
- # !! TODO resume here. |
300 |
+ try_call ( when_all_done ) |
301 |
+ # --- end of _sync_all_repos_and_run (...) --- |
302 |
|
303 |
+ def sync ( self ): |
304 |
+ LOGGER.debug ( "Syncing repos ..." ) |
305 |
for repo in self.repos: |
306 |
- if repo.sync() if self.sync_enabled else repo.nosync(): |
307 |
- # scan repo and create package infos |
308 |
- for p in repo.scan_distdir(): add ( p ) |
309 |
- elif self.use_broken_repos: |
310 |
- # warn and scan repo |
311 |
- ## .. |
312 |
- for p in repo.scan_distdir(): add ( p ) |
313 |
+ repo.sync ( sync_enabled=self.sync_enabled ) |
314 |
+ # --- end of sync_all (...) --- |
315 |
+ |
316 |
+ def sync_and_add ( self, add_method ): |
317 |
+ """Syncs all repos and adds packages immediately to the package queue.""" |
318 |
+ # TODO: _nowait? raises Exception when queue is full which is |
319 |
+ # good in non-threaded execution |
320 |
+ # -> timeout,.. |
321 |
+ |
322 |
+ qput = lambda r: self._queue_packages_from_repo ( r, add_method ) |
323 |
+ |
324 |
+ self._sync_all_repos_and_run ( when_repo_done=qput ) |
325 |
|
326 |
# --- end of sync_all (...) --- |
327 |
|
328 |
|
329 |
diff --git a/roverlay/remote/repoloader.py b/roverlay/remote/repoloader.py |
330 |
index eae35c5..94dd5b7 100644 |
331 |
--- a/roverlay/remote/repoloader.py |
332 |
+++ b/roverlay/remote/repoloader.py |
333 |
@@ -44,13 +44,18 @@ def read_repofile ( repo_file, lenient=False ): |
334 |
src_uri = get ( 'src_uri' ) |
335 |
) |
336 |
elif repo_type == 'rsync': |
337 |
+ extra_opts = get ( 'extra_rsync_opts' ) |
338 |
+ if extra_opts: |
339 |
+ extra_opts = extra_opts.split ( ' ' ) |
340 |
+ |
341 |
repo = RsyncRepo ( |
342 |
name = get ( 'name', name ), |
343 |
directory = get ( 'directory' ), |
344 |
src_uri = get ( 'src_uri' ), |
345 |
rsync_uri = get ( 'rsync_uri' ), |
346 |
base_uri = get ( 'base_uri' ), |
347 |
- extra_rsync_opts = get ( 'extra_rsync_opts' ) |
348 |
+ extra_opts = extra_opts, |
349 |
+ recursive = get ( 'recursive', False ) == 'yes', |
350 |
) |
351 |
else: |
352 |
LOGGER.error ( "Unknown repo type %s for %s" % ( repo_type, name ) ) |
353 |
|
354 |
diff --git a/roverlay/remote/rsync.py b/roverlay/remote/rsync.py |
355 |
index e46d1db..f99c8a7 100644 |
356 |
--- a/roverlay/remote/rsync.py |
357 |
+++ b/roverlay/remote/rsync.py |
358 |
@@ -1,4 +1,5 @@ |
359 |
import os |
360 |
+import sys |
361 |
import subprocess |
362 |
|
363 |
from roverlay import config |
364 |
@@ -13,16 +14,24 @@ RSYNC_ENV = keepenv ( |
365 |
'RSYNC_PASSWORD', |
366 |
) |
367 |
|
368 |
+# TODO: |
369 |
+# either reraise an KeyboardInterrupt while running rsync (which stops script |
370 |
+# execution unless the interrupt is catched elsewhere) or just set a |
371 |
+# non-zero return code (-> 'repo cannot be used') |
372 |
+RERAISE_INTERRUPT = False |
373 |
+ |
374 |
|
375 |
# --recursive is not in the default opts, subdirs in CRAN/contrib are |
376 |
-# either R release (2.xx.x[-patches] or the package archive) |
377 |
+# either R releases (2.xx.x[-patches]) or the package archive |
378 |
DEFAULT_RSYNC_OPTS = ( |
379 |
'--links', # copy symlinks as symlinks, |
380 |
'--safe-links', # but ignore links outside of tree |
381 |
'--times', # |
382 |
'--compress', # FIXME: add lzo if necessary |
383 |
- '--delete', # |
384 |
+ '--dirs', # |
385 |
+ '--prune-empty-dirs', # |
386 |
'--force', # allow deletion of non-empty dirs |
387 |
+ '--delete', # |
388 |
'--human-readable', # |
389 |
'--stats', # |
390 |
'--chmod=ugo=r,u+w,Dugo+x', # 0755 for transferred dirs, 0644 for files |
391 |
@@ -30,11 +39,25 @@ DEFAULT_RSYNC_OPTS = ( |
392 |
|
393 |
class RsyncJob ( object ): |
394 |
def __init__ ( |
395 |
- self, remote=None, distdir=None, run_now=True, extra_opts=None |
396 |
+ self, remote=None, distdir=None, run_now=True, |
397 |
+ extra_opts=None, recursive=False |
398 |
): |
399 |
- self.remote = remote |
400 |
- self.distdir = distdir |
401 |
- self.extra_opts = None |
402 |
+ self.distdir = distdir |
403 |
+ |
404 |
+ # syncing directories, not files - always appending a slash at the end |
405 |
+ # of remote |
406 |
+ if remote [-1] != '/': |
407 |
+ self.remote = remote + '/' |
408 |
+ else: |
409 |
+ self.remote = remote |
410 |
+ |
411 |
+ if recursive: |
412 |
+ self.extra_opts = [ '--recursive' ] |
413 |
+ if extra_opts: |
414 |
+ self.extra_opts.extend ( extra_opts ) |
415 |
+ else: |
416 |
+ self.extra_opts = extra_opts |
417 |
+ |
418 |
|
419 |
if run_now: self.run() |
420 |
# --- end of __init__ (...) --- |
421 |
@@ -51,36 +74,46 @@ class RsyncJob ( object ): |
422 |
if max_bw is not None: |
423 |
argv.append ( '--bwlimit=%i' % max_bw ) |
424 |
|
425 |
- if self.extra_opts is not None: |
426 |
- if isinstance ( self.extra_opts, str ) or \ |
427 |
- not hasattr ( self.extra_opts, '__iter__' )\ |
428 |
- : |
429 |
- argv.append ( self.extra_opts ) |
430 |
- else: |
431 |
- argv.extend ( self.extra_opts ) |
432 |
+ if self.extra_opts: |
433 |
+ argv.extend ( self.extra_opts ) |
434 |
|
435 |
argv.extend ( ( self.remote, self.distdir ) ) |
436 |
|
437 |
- return argv |
438 |
+ |
439 |
+ # removing emty args from argv |
440 |
+ return tuple ( filter ( None, argv ) ) |
441 |
# --- end of _rsync_argv (...) --- |
442 |
|
443 |
def run ( self ): |
444 |
|
445 |
rsync_cmd = self._rsync_argv() |
446 |
+ print ( ' '.join ( rsync_cmd ) ) |
447 |
|
448 |
os.makedirs ( self.distdir, exist_ok=True ) |
449 |
|
450 |
# TODO pipe/log/.., running this in blocking mode until implemented |
451 |
+ try: |
452 |
+ proc = subprocess.Popen ( |
453 |
+ rsync_cmd, |
454 |
+ stdin=None, stdout=None, stderr=None, |
455 |
+ env=RSYNC_ENV |
456 |
+ ) |
457 |
+ |
458 |
+ if proc.communicate() != ( None, None ): |
459 |
+ raise AssertionError ( "expected None,None from communicate!" ) |
460 |
+ |
461 |
+ self.returncode = proc.returncode |
462 |
+ |
463 |
+ except KeyboardInterrupt: |
464 |
+ sys.stderr.write ( |
465 |
+ "\nKeyboard interrupt - waiting for rsync to exit...\n" |
466 |
+ ) |
467 |
+ if 'proc' in locals(): |
468 |
+ proc.communicate() |
469 |
+ self.returncode = proc.returncode |
470 |
+ else: |
471 |
+ self.returncode = 130 |
472 |
|
473 |
- proc = subprocess.Popen ( |
474 |
- rsync_cmd, |
475 |
- stdin=None, stdout=None, stderr=None, |
476 |
- env=RSYNC_ENV |
477 |
- ) |
478 |
- |
479 |
- if proc.communicate() != ( None, None ): |
480 |
- raise AssertionError ( "expected None,None from communicate!" ) |
481 |
- |
482 |
- self.returncode = proc.returncode |
483 |
- |
484 |
+ if RERAISE_INTERRUPT: |
485 |
+ raise |
486 |
# --- end of start (...) --- |