Gentoo Archives: gentoo-commits

From: "André Erdmann" <dywi@×××××××.de>
To: gentoo-commits@l.g.o
Subject: [gentoo-commits] proj/R_overlay:master commit in: roverlay/remote/
Date: Tue, 26 Jun 2012 15:43:29
Message-Id: 1340725044.2383f21652384458d0fc7a05a0c70610f22719d3.dywi@gentoo
1 commit: 2383f21652384458d0fc7a05a0c70610f22719d3
2 Author: André Erdmann <dywi <AT> mailerd <DOT> de>
3 AuthorDate: Tue Jun 26 15:37:24 2012 +0000
4 Commit: André Erdmann <dywi <AT> mailerd <DOT> de>
5 CommitDate: Tue Jun 26 15:37:24 2012 +0000
6 URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=2383f216
7
8 remote
9
10 * can now be used to feed the overlay creation package queue
11
12 modified: roverlay/remote/basicrepo.py
13 modified: roverlay/remote/repo.py
14 modified: roverlay/remote/repolist.py
15 modified: roverlay/remote/repoloader.py
16 modified: roverlay/remote/rsync.py
17
18 ---
19 roverlay/remote/basicrepo.py | 70 +++++++++++++++++++++++++++-------
20 roverlay/remote/repo.py | 18 +++++---
21 roverlay/remote/repolist.py | 78 +++++++++++++++++++++++++++++---------
22 roverlay/remote/repoloader.py | 7 +++-
23 roverlay/remote/rsync.py | 83 ++++++++++++++++++++++++++++------------
24 5 files changed, 190 insertions(+), 66 deletions(-)
25
26 diff --git a/roverlay/remote/basicrepo.py b/roverlay/remote/basicrepo.py
27 index 9ade3a2..1574850 100644
28 --- a/roverlay/remote/basicrepo.py
29 +++ b/roverlay/remote/basicrepo.py
30 @@ -8,6 +8,11 @@ DEFAULT_PROTOCOL = 'http'
31
32 LOCALREPO_SRC_URI = 'http://localhost/R-Packages'
33
34 +SYNC_SUCCESS = 1
35 +SYNC_FAIL = 2
36 +REPO_READY = 4
37 +
38 +
39 def normalize_uri ( uri, protocol, force_protocol=False ):
40
41 if not protocol:
42 @@ -53,8 +58,29 @@ class LocalRepo ( object ):
43 else:
44 self.src_uri = src_uri
45
46 + self.sync_status = 0
47 +
48 # --- end of __init__ (...) ---
49
50 + def ready ( self ):
51 + return bool ( self.sync_status & REPO_READY )
52 +
53 + def fail ( self ):
54 + return bool ( self.sync_status & SYNC_FAIL )
55 +
56 + def offline ( self ):
57 + return 0 == self.sync_status & SYNC_SUCCESS
58 +
59 + def _set_ready ( self, is_synced ):
60 + """comment TODO"""
61 + if is_synced:
62 + self.sync_status = SYNC_SUCCESS | REPO_READY
63 + else:
64 + self.sync_status = REPO_READY
65 +
66 + def _set_fail ( self ):
67 + self.sync_status = SYNC_FAIL
68 +
69 def __str__ ( self ):
70 return "repo '%s': DISTDIR '%s', SRC_URI '%s'" % (
71 self.name, self.distdir, self.src_uri
72 @@ -79,7 +105,7 @@ class LocalRepo ( object ):
73 if package_file is None:
74 return self.src_uri
75 else:
76 - return '/'.join ( self.src_uri, package_file )
77 + return '/'.join ( ( self.src_uri, package_file ) )
78 # --- end of get_src_uri (...) ---
79
80 # get_src(...) -> get_src_uri(...)
81 @@ -90,17 +116,26 @@ class LocalRepo ( object ):
82 return os.path.isdir ( self.distdir )
83 # --- end of exists (...) ---
84
85 - def nosync ( self ):
86 - """Returns True if the repo is ready for overlay creation, else False.
87 - Useful for basic local distfiles verification without downloading
88 - anything.
89 - """
90 - return self.exists()
91 + def sync ( self, sync_enabled=True ):
92 + """Syncs this repo."""
93
94 - # --- end of nosync (...) ---
95 + status = False
96 + if sync_enabled and hasattr ( self, '_dosync' ):
97 + status = self._dosync()
98
99 - # sync() -> nosync(), LocalRepos don't have anything to sync
100 - sync = nosync
101 + elif hasattr ( self, '_nosync'):
102 + status = self._nosync()
103 +
104 + else:
105 + status = self.exists()
106 +
107 + if status:
108 + self._set_ready ( is_synced=sync_enabled )
109 + else:
110 + self._set_fail()
111 +
112 + return status
113 + # --- end of sync (...) ---
114
115 def scan_distdir ( self, is_package=None ):
116 """Generator that scans the local distfiles dir of this repo and
117 @@ -111,19 +146,26 @@ class LocalRepo ( object ):
118 or None which means that all files are packages.
119 Defaults to None.
120 """
121 +
122 + kw = { 'origin' : self }
123 +
124 if is_package is None:
125 # unfiltered variant
126
127 for dirpath, dirnames, filenames in os.walk ( self.distdir ):
128 + kw ['distdir'] = dirpath if dirpath != self.distdir else None
129 +
130 for pkg in filenames:
131 - yield PackageInfo ( filename=pkg, origin=self )
132 + yield PackageInfo ( filename=pkg, **kw )
133
134 elif hasattr ( is_package, '__call__' ):
135 # filtered variant (adds an if is_package... before yield)
136 for dirpath, dirnames, filenames in os.walk ( self.distdir ):
137 + kw ['distdir'] = dirpath if dirpath != self.distdir else None
138 +
139 for pkg in filenames:
140 if is_package ( os.path.join ( dirpath, pkg ) ):
141 - yield PackageInfo ( filename=pkg, origin=self )
142 + yield PackageInfo ( filename=pkg, **kw )
143
144
145 else:
146 @@ -227,14 +269,14 @@ class RemoteRepo ( LocalRepo ):
147 # get_remote(...) -> get_remote_uri(...)
148 get_remote = get_remote_uri
149
150 - def sync ( self ):
151 + def _dosync ( self ):
152 """Gets packages from remote(s) and returns True if the repo is ready
153 for overlay creation, else False.
154
155 Derived classes have to implement this method.
156 """
157 raise Exception ( "RemoteRepo does not implement sync()." )
158 - # --- end of sync (...) ---
159 + # --- end of _dosync (...) ---
160
161 def __str__ ( self ):
162 return "repo '%s': DISTDIR '%s', SRC_URI '%s', REMOTE_URI '%s'" % (
163
164 diff --git a/roverlay/remote/repo.py b/roverlay/remote/repo.py
165 index f54f448..febfaae 100644
166 --- a/roverlay/remote/repo.py
167 +++ b/roverlay/remote/repo.py
168 @@ -11,7 +11,7 @@ class RsyncRepo ( RemoteRepo ):
169 def __init__ (
170 self, name,
171 directory=None, src_uri=None, rsync_uri=None, base_uri=None,
172 - extra_rsync_opts=None
173 + **rsync_kw
174 ):
175 # super's init: name, remote protocol, directory_kw, **uri_kw
176 # using '' as remote protocol which leaves uris unchanged when
177 @@ -20,19 +20,23 @@ class RsyncRepo ( RemoteRepo ):
178 name, '', directory=directory,
179 src_uri=src_uri, remote_uri=rsync_uri, base_uri=base_uri
180 )
181 - self.extra_rsync_opts = extra_rsync_opts
182 + self.rsync_extra = rsync_kw
183 +
184 + self.sync_protocol = 'rsync'
185 # --- end of __init__ (...) ---
186
187
188 - def sync ( self ):
189 + def _dosync ( self ):
190 retcode = None
191 try:
192 job = RsyncJob (
193 remote=self.remote_uri, distdir=self.distdir,
194 run_now=True,
195 - extra_opts=self.extra_rsync_opts
196 + **self.rsync_extra
197 )
198 - if job.returncode == 0: return True
199 + if job.returncode == 0:
200 + self._set_ready ( is_synced=True )
201 + return True
202
203 retcode = job.returncode
204 except Exception as e:
205 @@ -41,13 +45,13 @@ class RsyncRepo ( RemoteRepo ):
206 logging.exception ( e )
207 retcode = '<undef>'
208
209 -
210 logging.error (
211 'Repo %s cannot be used for ebuild creation due to errors '
212 'while running rsync (return code was %s).' % ( self.name, retcode )
213 )
214 + self._set_fail()
215 return False
216 - # --- end of sync (...) ---
217 + # --- end of _dosync (...) ---
218
219 def __str__ ( self ):
220 return "rsync repo '%s': DISTDIR '%s', SRC_URI '%s', RSYNC_URI '%s'" \
221
222 diff --git a/roverlay/remote/repolist.py b/roverlay/remote/repolist.py
223 index 4617057..ae740dd 100644
224 --- a/roverlay/remote/repolist.py
225 +++ b/roverlay/remote/repolist.py
226 @@ -1,13 +1,19 @@
227 +import logging
228
229 from roverlay import config
230 -
231 from roverlay.remote.repoloader import read_repofile
232
233 +LOGGER = logging.getLogger ( 'RepoList' )
234 +
235 class RepoList ( object ):
236
237 def __init__ ( self ):
238 self.repos = list()
239 +
240 self.sync_enabled = True
241 +
242 + # if True: use all repos when looking for packages, even those that
243 + # could not be synced
244 self.use_broken_repos = False
245
246 def sort ( self ):
247 @@ -25,28 +31,62 @@ class RepoList ( object ):
248 self.load_file ( f )
249 # --- end of load (...) ---
250
251 - def sync_all ( self, package_queue=None ):
252 - q = None
253 - if package_queue is None:
254 - q = list()
255 - add = q.append
256 - else:
257 - # TODO: _nowait? raises Exception when queue is full which is
258 - # good in non-threaded execution
259 - # -> timeout,..
260 - add = q.put
261 + def _queue_packages_from_repo ( self, repo, add_method ):
262 + if not repo.ready():
263 + if self.use_broken_repos:
264 + # warn and continue
265 + pass
266 + else:
267 + # repo cannot be used
268 + LOGGER.warning ( "!!" )
269 + return False
270 +
271 + for p in repo.scan_distdir():
272 + LOGGER.debug ( "adding package %s from repo %s" % ( p, repo ) )
273 + add_method ( p )
274 + # --- end of _queue_packages_from_repo (...) ---
275 +
276 + def add_packages ( self, add_method ):
277 + for repo in self.repos:
278 + self._queue_packages_from_repo ( repo, add_method )
279 + # --- end of add_packages (...) ---
280 +
281 + def _sync_all_repos_and_run (
282 + self,
283 + when_repo_success=None, when_repo_fail=None, when_repo_done=None,
284 + when_all_done=None
285 + ):
286 + try_call = lambda f, *x, **z : None if f is None else f ( *x, **z )
287 +
288 + LOGGER.debug ( "Syncing repos ..." )
289 + for repo in self.repos:
290 + if repo.sync ( sync_enabled=self.sync_enabled ):
291 + # repo successfully synced
292 + try_call ( when_repo_success, repo )
293 + else:
294 + # else log fail <>
295 + try_call ( when_repo_fail, repo )
296
297 + try_call ( when_repo_done, repo )
298
299 - # !! TODO resume here.
300 + try_call ( when_all_done )
301 + # --- end of _sync_all_repos_and_run (...) ---
302
303 + def sync ( self ):
304 + LOGGER.debug ( "Syncing repos ..." )
305 for repo in self.repos:
306 - if repo.sync() if self.sync_enabled else repo.nosync():
307 - # scan repo and create package infos
308 - for p in repo.scan_distdir(): add ( p )
309 - elif self.use_broken_repos:
310 - # warn and scan repo
311 - ## ..
312 - for p in repo.scan_distdir(): add ( p )
313 + repo.sync ( sync_enabled=self.sync_enabled )
314 + # --- end of sync_all (...) ---
315 +
316 + def sync_and_add ( self, add_method ):
317 + """Syncs all repos and adds packages immediately to the package queue."""
318 + # TODO: _nowait? raises Exception when queue is full which is
319 + # good in non-threaded execution
320 + # -> timeout,..
321 +
322 + qput = lambda r: self._queue_packages_from_repo ( r, add_method )
323 +
324 + self._sync_all_repos_and_run ( when_repo_done=qput )
325
326 # --- end of sync_all (...) ---
327
328
329 diff --git a/roverlay/remote/repoloader.py b/roverlay/remote/repoloader.py
330 index eae35c5..94dd5b7 100644
331 --- a/roverlay/remote/repoloader.py
332 +++ b/roverlay/remote/repoloader.py
333 @@ -44,13 +44,18 @@ def read_repofile ( repo_file, lenient=False ):
334 src_uri = get ( 'src_uri' )
335 )
336 elif repo_type == 'rsync':
337 + extra_opts = get ( 'extra_rsync_opts' )
338 + if extra_opts:
339 + extra_opts = extra_opts.split ( ' ' )
340 +
341 repo = RsyncRepo (
342 name = get ( 'name', name ),
343 directory = get ( 'directory' ),
344 src_uri = get ( 'src_uri' ),
345 rsync_uri = get ( 'rsync_uri' ),
346 base_uri = get ( 'base_uri' ),
347 - extra_rsync_opts = get ( 'extra_rsync_opts' )
348 + extra_opts = extra_opts,
349 + recursive = get ( 'recursive', False ) == 'yes',
350 )
351 else:
352 LOGGER.error ( "Unknown repo type %s for %s" % ( repo_type, name ) )
353
354 diff --git a/roverlay/remote/rsync.py b/roverlay/remote/rsync.py
355 index e46d1db..f99c8a7 100644
356 --- a/roverlay/remote/rsync.py
357 +++ b/roverlay/remote/rsync.py
358 @@ -1,4 +1,5 @@
359 import os
360 +import sys
361 import subprocess
362
363 from roverlay import config
364 @@ -13,16 +14,24 @@ RSYNC_ENV = keepenv (
365 'RSYNC_PASSWORD',
366 )
367
368 +# TODO:
369 +# either reraise an KeyboardInterrupt while running rsync (which stops script
370 +# execution unless the interrupt is catched elsewhere) or just set a
371 +# non-zero return code (-> 'repo cannot be used')
372 +RERAISE_INTERRUPT = False
373 +
374
375 # --recursive is not in the default opts, subdirs in CRAN/contrib are
376 -# either R release (2.xx.x[-patches] or the package archive)
377 +# either R releases (2.xx.x[-patches]) or the package archive
378 DEFAULT_RSYNC_OPTS = (
379 '--links', # copy symlinks as symlinks,
380 '--safe-links', # but ignore links outside of tree
381 '--times', #
382 '--compress', # FIXME: add lzo if necessary
383 - '--delete', #
384 + '--dirs', #
385 + '--prune-empty-dirs', #
386 '--force', # allow deletion of non-empty dirs
387 + '--delete', #
388 '--human-readable', #
389 '--stats', #
390 '--chmod=ugo=r,u+w,Dugo+x', # 0755 for transferred dirs, 0644 for files
391 @@ -30,11 +39,25 @@ DEFAULT_RSYNC_OPTS = (
392
393 class RsyncJob ( object ):
394 def __init__ (
395 - self, remote=None, distdir=None, run_now=True, extra_opts=None
396 + self, remote=None, distdir=None, run_now=True,
397 + extra_opts=None, recursive=False
398 ):
399 - self.remote = remote
400 - self.distdir = distdir
401 - self.extra_opts = None
402 + self.distdir = distdir
403 +
404 + # syncing directories, not files - always appending a slash at the end
405 + # of remote
406 + if remote [-1] != '/':
407 + self.remote = remote + '/'
408 + else:
409 + self.remote = remote
410 +
411 + if recursive:
412 + self.extra_opts = [ '--recursive' ]
413 + if extra_opts:
414 + self.extra_opts.extend ( extra_opts )
415 + else:
416 + self.extra_opts = extra_opts
417 +
418
419 if run_now: self.run()
420 # --- end of __init__ (...) ---
421 @@ -51,36 +74,46 @@ class RsyncJob ( object ):
422 if max_bw is not None:
423 argv.append ( '--bwlimit=%i' % max_bw )
424
425 - if self.extra_opts is not None:
426 - if isinstance ( self.extra_opts, str ) or \
427 - not hasattr ( self.extra_opts, '__iter__' )\
428 - :
429 - argv.append ( self.extra_opts )
430 - else:
431 - argv.extend ( self.extra_opts )
432 + if self.extra_opts:
433 + argv.extend ( self.extra_opts )
434
435 argv.extend ( ( self.remote, self.distdir ) )
436
437 - return argv
438 +
439 + # removing emty args from argv
440 + return tuple ( filter ( None, argv ) )
441 # --- end of _rsync_argv (...) ---
442
443 def run ( self ):
444
445 rsync_cmd = self._rsync_argv()
446 + print ( ' '.join ( rsync_cmd ) )
447
448 os.makedirs ( self.distdir, exist_ok=True )
449
450 # TODO pipe/log/.., running this in blocking mode until implemented
451 + try:
452 + proc = subprocess.Popen (
453 + rsync_cmd,
454 + stdin=None, stdout=None, stderr=None,
455 + env=RSYNC_ENV
456 + )
457 +
458 + if proc.communicate() != ( None, None ):
459 + raise AssertionError ( "expected None,None from communicate!" )
460 +
461 + self.returncode = proc.returncode
462 +
463 + except KeyboardInterrupt:
464 + sys.stderr.write (
465 + "\nKeyboard interrupt - waiting for rsync to exit...\n"
466 + )
467 + if 'proc' in locals():
468 + proc.communicate()
469 + self.returncode = proc.returncode
470 + else:
471 + self.returncode = 130
472
473 - proc = subprocess.Popen (
474 - rsync_cmd,
475 - stdin=None, stdout=None, stderr=None,
476 - env=RSYNC_ENV
477 - )
478 -
479 - if proc.communicate() != ( None, None ):
480 - raise AssertionError ( "expected None,None from communicate!" )
481 -
482 - self.returncode = proc.returncode
483 -
484 + if RERAISE_INTERRUPT:
485 + raise
486 # --- end of start (...) ---