1 |
commit: af13e5117e1f4fe4a4980e9ec0988378771fc4b4 |
2 |
Author: André Erdmann <dywi <AT> mailerd <DOT> de> |
3 |
AuthorDate: Thu Dec 13 14:55:28 2012 +0000 |
4 |
Commit: André Erdmann <dywi <AT> mailerd <DOT> de> |
5 |
CommitDate: Mon Jan 28 23:33:10 2013 +0000 |
6 |
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=af13e511 |
7 |
|
8 |
Add support for multiple manifest implementations |
9 |
|
10 |
Also added support for threaded Manifest file writing, though no available |
11 |
implementation supports that at the moment. |
12 |
|
13 |
--- |
14 |
config/R-overlay.conf.reference | 14 +++ |
15 |
doc/html/usage.html | 18 ++++ |
16 |
doc/rst/usage.rst | 20 ++++ |
17 |
roverlay/config/entrymap.py | 16 +++- |
18 |
roverlay/overlay/category.py | 66 +++++++++++-- |
19 |
roverlay/overlay/pkgdir/__init__.py | 68 ++++++++++++++ |
20 |
roverlay/overlay/{ => pkgdir}/manifest/__init__.py | 4 +- |
21 |
roverlay/overlay/{ => pkgdir}/manifest/env.py | 0 |
22 |
.../manifest/manifest_ebuild.py} | 2 +- |
23 |
roverlay/overlay/{ => pkgdir}/metadata/__init__.py | 2 +- |
24 |
.../overlay/{ => pkgdir}/metadata/abstractnodes.py | 0 |
25 |
roverlay/overlay/{ => pkgdir}/metadata/nodes.py | 2 +- |
26 |
.../{package.py => pkgdir/packagedir_base.py} | 98 ++++++++++---------- |
27 |
.../overlay/pkgdir/packagedir_ebuildmanifest.py | 56 +++++++++++ |
28 |
14 files changed, 299 insertions(+), 67 deletions(-) |
29 |
|
30 |
diff --git a/config/R-overlay.conf.reference b/config/R-overlay.conf.reference |
31 |
index a1ba48c..067e1a7 100644 |
32 |
--- a/config/R-overlay.conf.reference |
33 |
+++ b/config/R-overlay.conf.reference |
34 |
@@ -145,6 +145,20 @@ |
35 |
# *** NOT IN USE *** |
36 |
#EBUILD_HEADER = "ebuild_header.txt" |
37 |
|
38 |
+# OVERLAY_MANIFEST_IMPLEMENTATION |
39 |
+# Manifest implementation to be used |
40 |
+# Currently, there's only one implementation available, manifest |
41 |
+# writing using the ebuild(1) executable. |
42 |
+# * accepted values: |
43 |
+# -> 'default' -- use the default implementation |
44 |
+# -> 'none' -- do not use any implementation, |
45 |
+# this leads to runtime errors |
46 |
+# whenever write_manifest() is called |
47 |
+# -> 'external:ebuild' -- generate Manifest files using ebuild(1) |
48 |
+# |
49 |
+# * defaults to 'default' |
50 |
+# * alias: MANIFEST_IMPLEMENTATION |
51 |
+ |
52 |
# == other config files == |
53 |
|
54 |
# some config options are split from this file for various reasons: |
55 |
|
56 |
diff --git a/doc/html/usage.html b/doc/html/usage.html |
57 |
index 026d8b7..ff2a36f 100644 |
58 |
--- a/doc/html/usage.html |
59 |
+++ b/doc/html/usage.html |
60 |
@@ -1570,6 +1570,10 @@ This will pass '--bwlimit=<value>' to all rsync commands.</p> |
61 |
<dt>ECLASS</dt> |
62 |
<dd>Alias to <a class="reference internal" href="#overlay-eclass">OVERLAY_ECLASS</a>.</dd> |
63 |
</dl> |
64 |
+<dl class="docutils" id="manifest-implementation"> |
65 |
+<dt>MANIFEST_IMPLEMENTATION</dt> |
66 |
+<dd>Alias to <a class="reference internal" href="#overlay-manifest-implementation">OVERLAY_MANIFEST_IMPLEMENTATION</a>.</dd> |
67 |
+</dl> |
68 |
<dl class="docutils" id="overlay-category"> |
69 |
<dt>OVERLAY_CATEGORY</dt> |
70 |
<dd><p class="first">Sets the category of created ebuilds. There are no value type restrictions |
71 |
@@ -1602,6 +1606,20 @@ per R package. All others will be removed.</p> |
72 |
<p class="last">Defaults to <not set>, which disables this feature and keeps all ebuilds.</p> |
73 |
</dd> |
74 |
</dl> |
75 |
+<dl class="docutils" id="overlay-manifest-implementation"> |
76 |
+<dt>OVERLAY_MANIFEST_IMPLEMENTATION</dt> |
77 |
+<dd><p class="first">Sets the implementation to use for Manifest file writing. |
78 |
+Available choices are 'external:ebuild', 'default' and 'none'. |
79 |
+Defaults to 'default'.</p> |
80 |
+<div class="note last"> |
81 |
+<p class="first admonition-title">Note</p> |
82 |
+<p class="last">Choosing 'none' is destructive - <em>roverlay</em> will fail to function |
83 |
+whenever Manifest access is required. |
84 |
+Use the '--no-manifest' command line option to disable manifest |
85 |
+writing.</p> |
86 |
+</div> |
87 |
+</dd> |
88 |
+</dl> |
89 |
<dl class="docutils" id="overlay-name"> |
90 |
<dt>OVERLAY_NAME</dt> |
91 |
<dd><p class="first">Sets the name of the created overlay that will be written into |
92 |
|
93 |
diff --git a/doc/rst/usage.rst b/doc/rst/usage.rst |
94 |
index 31c81f0..46e5e00 100644 |
95 |
--- a/doc/rst/usage.rst |
96 |
+++ b/doc/rst/usage.rst |
97 |
@@ -1267,6 +1267,11 @@ RSYNC_BWLIMIT |
98 |
ECLASS |
99 |
Alias to OVERLAY_ECLASS_. |
100 |
|
101 |
+.. _MANIFEST_IMPLEMENTATION: |
102 |
+ |
103 |
+MANIFEST_IMPLEMENTATION |
104 |
+ Alias to OVERLAY_MANIFEST_IMPLEMENTATION_. |
105 |
+ |
106 |
.. _OVERLAY_CATEGORY: |
107 |
|
108 |
OVERLAY_CATEGORY |
109 |
@@ -1303,6 +1308,21 @@ OVERLAY_KEEP_NTH_LATEST |
110 |
|
111 |
Defaults to <not set>, which disables this feature and keeps all ebuilds. |
112 |
|
113 |
+.. _OVERLAY_MANIFEST_IMPLEMENTATION: |
114 |
+ |
115 |
+OVERLAY_MANIFEST_IMPLEMENTATION |
116 |
+ Sets the implementation to use for Manifest file writing. |
117 |
+ Available choices are 'external:ebuild', 'default' and 'none'. |
118 |
+ Defaults to 'default'. |
119 |
+ |
120 |
+ .. Note:: |
121 |
+ |
122 |
+ Choosing 'none' is destructive - *roverlay* will fail to function |
123 |
+ whenever Manifest access is required. |
124 |
+ Use the '--no-manifest' command line option to disable manifest |
125 |
+ writing. |
126 |
+ |
127 |
+ |
128 |
.. _OVERLAY_NAME: |
129 |
|
130 |
OVERLAY_NAME |
131 |
|
132 |
diff --git a/roverlay/config/entrymap.py b/roverlay/config/entrymap.py |
133 |
index ef83540..0d679c2 100644 |
134 |
--- a/roverlay/config/entrymap.py |
135 |
+++ b/roverlay/config/entrymap.py |
136 |
@@ -213,6 +213,17 @@ CONFIG_ENTRY_MAP = dict ( |
137 |
desc = "overlay name, e.g. 'R-Overlay'.", |
138 |
), |
139 |
|
140 |
+ overlay_manifest_implementation = dict ( |
141 |
+ desc = "manifest implementation to be used", |
142 |
+ choices = frozenset (( |
143 |
+ 'none', |
144 |
+ 'default' |
145 |
+ 'external:ebuild', |
146 |
+# 'external:portage', |
147 |
+# 'internal', |
148 |
+ )), |
149 |
+ ), |
150 |
+ |
151 |
# ebuild is used to create Manifest files |
152 |
ebuild_prog = dict ( |
153 |
path = [ 'TOOLS', 'ebuild_prog' ], |
154 |
@@ -229,8 +240,9 @@ CONFIG_ENTRY_MAP = dict ( |
155 |
|
156 |
|
157 |
# * alias |
158 |
- eclass = 'overlay_eclass', |
159 |
- keep_nth_latest = 'overlay_keep_nth_latest', |
160 |
+ eclass = 'overlay_eclass', |
161 |
+ keep_nth_latest = 'overlay_keep_nth_latest', |
162 |
+ manifest_implementation = 'overlay_manifest_implementation', |
163 |
|
164 |
# --- overlay |
165 |
|
166 |
|
167 |
diff --git a/roverlay/overlay/category.py b/roverlay/overlay/category.py |
168 |
index 5ddc9ad..9090bb7 100644 |
169 |
--- a/roverlay/overlay/category.py |
170 |
+++ b/roverlay/overlay/category.py |
171 |
@@ -22,7 +22,7 @@ try: |
172 |
except ImportError: |
173 |
import Queue as queue |
174 |
|
175 |
-from roverlay.overlay.package import PackageDir |
176 |
+from roverlay.overlay import pkgdir |
177 |
|
178 |
class Category ( object ): |
179 |
|
180 |
@@ -61,7 +61,7 @@ class Category ( object ): |
181 |
self._lock.acquire() |
182 |
try: |
183 |
if not pkg_name in self._subdirs: |
184 |
- newpkg = PackageDir ( |
185 |
+ newpkg = pkgdir.get_class() ( |
186 |
name = pkg_name, |
187 |
logger = self.logger, |
188 |
directory = self.physical_location + os.sep + pkg_name, |
189 |
@@ -83,15 +83,17 @@ class Category ( object ): |
190 |
|
191 |
returns: success |
192 |
""" |
193 |
- subdir = self._get_package_dir ( package_info ['name'] ) |
194 |
- return subdir.add ( package_info, **pkg_add_kw ) |
195 |
+ return self._get_package_dir ( package_info ['name'] ).add ( |
196 |
+ package_info, **pkg_add_kw |
197 |
+ ) |
198 |
# --- end of add (...) --- |
199 |
|
200 |
def empty ( self ): |
201 |
"""Returns True if this category contains 0 ebuilds.""" |
202 |
- return \ |
203 |
- len ( self._subdirs ) == 0 or \ |
204 |
- not False in ( d.empty() for d in self._subdirs.values() ) |
205 |
+ return ( |
206 |
+ len ( self._subdirs ) == 0 or |
207 |
+ all ( d.empty() for d in self._subdirs.values() ) |
208 |
+ ) |
209 |
# --- end of empty (...) --- |
210 |
|
211 |
def has ( self, subdir ): |
212 |
@@ -121,6 +123,34 @@ class Category ( object ): |
213 |
yield self.name + os.sep + name |
214 |
# --- end of list_packages (...) --- |
215 |
|
216 |
+ def supports_threadsafe_manifest_writing ( self, unsafe=True ): |
217 |
+ """Returns True if manifest writing is thread safe for this |
218 |
+ category, else False. Also returns True for empty categories. |
219 |
+ |
220 |
+ arguments: |
221 |
+ * unsafe -- if False : verify that all packages of this category support |
222 |
+ thread safe writing |
223 |
+ else : assume that all packages of this category are |
224 |
+ instances of the same class and check only the |
225 |
+ first one (=> faster). |
226 |
+ Defaults to True. |
227 |
+ |
228 |
+ """ |
229 |
+ if unsafe: |
230 |
+ try: |
231 |
+ return bool ( |
232 |
+ next ( iter ( self._subdirs.value ) ).MANIFEST_THREADSAFE |
233 |
+ ) |
234 |
+ except StopIteration: |
235 |
+ return True |
236 |
+ else: |
237 |
+ for pkgdir in self._subdirs.values(): |
238 |
+ #if not pkgdir.__class__.MANIFEST_THREADSAFE: |
239 |
+ if not pkgdir.MANIFEST_THREADSAFE: |
240 |
+ return False |
241 |
+ return True |
242 |
+ # --- end of supports_threadsafe_manifest_writing (...) --- |
243 |
+ |
244 |
def remove_empty ( self ): |
245 |
"""This removes all empty PackageDirs.""" |
246 |
with self._lock: |
247 |
@@ -133,7 +163,12 @@ class Category ( object ): |
248 |
"""Scans this category for existing ebuilds.""" |
249 |
for subdir in os.listdir ( self.physical_location ): |
250 |
if self.has_dir ( subdir ): |
251 |
- self._get_package_dir ( subdir ).scan ( **kw ) |
252 |
+ pkgdir = self._get_package_dir ( subdir ) |
253 |
+ try: |
254 |
+ pkgdir.scan ( **kw ) |
255 |
+ finally: |
256 |
+ if pkgdir.empty(): |
257 |
+ del self._subdirs [subdir] |
258 |
# --- end of scan (...) --- |
259 |
|
260 |
def show ( self, **show_kw ): |
261 |
@@ -182,6 +217,7 @@ class Category ( object ): |
262 |
overwrite_ebuilds = overwrite_ebuilds, |
263 |
keep_n_ebuilds = keep_n_ebuilds, |
264 |
cautious = cautious, |
265 |
+ write_manifest = write_manifest, |
266 |
) |
267 |
|
268 |
# start writing: |
269 |
@@ -198,9 +234,19 @@ class Category ( object ): |
270 |
# don't create more workers than write jobs available |
271 |
max_jobs = min ( max_jobs, len ( self._subdirs ) ) |
272 |
|
273 |
+ manifest_threadsafe = self.supports_threadsafe_manifest_writing ( |
274 |
+ unsafe=True |
275 |
+ ) |
276 |
+# manifest_threadsafe = True |
277 |
write_queue = queue.Queue() |
278 |
for package in self._subdirs.values(): |
279 |
write_queue.put_nowait ( package ) |
280 |
+# if not package.MANIFEST_THREADSAFE: |
281 |
+# manifest_threadsafe = False |
282 |
+ |
283 |
+ write_kwargs ['write_manifest'] = ( |
284 |
+ write_manifest and manifest_threadsafe |
285 |
+ ) |
286 |
|
287 |
workers = frozenset ( |
288 |
threading.Thread ( |
289 |
@@ -219,7 +265,7 @@ class Category ( object ): |
290 |
|
291 |
# write manifest files |
292 |
# fixme: debug print |
293 |
- if write_manifest: |
294 |
+ if write_manifest and ( not manifest_threadsafe ): |
295 |
#self.logger.info ( "Writing Manifest files for {}".format ( name ) ) |
296 |
print ( "Writing Manifest files ..." ) |
297 |
for package in self._subdirs.values(): |
298 |
@@ -227,7 +273,7 @@ class Category ( object ): |
299 |
|
300 |
else: |
301 |
for package in self._subdirs.values(): |
302 |
- package.write ( write_manifest=write_manifest, **write_kwargs ) |
303 |
+ package.write ( **write_kwargs ) |
304 |
|
305 |
self.remove_empty() |
306 |
# --- end of write (...) --- |
307 |
|
308 |
diff --git a/roverlay/overlay/pkgdir/__init__.py b/roverlay/overlay/pkgdir/__init__.py |
309 |
new file mode 100644 |
310 |
index 0000000..e4ebffa |
311 |
--- /dev/null |
312 |
+++ b/roverlay/overlay/pkgdir/__init__.py |
313 |
@@ -0,0 +1,68 @@ |
314 |
+# R overlay -- overlay package, package directory (<??>) |
315 |
+# -*- coding: utf-8 -*- |
316 |
+# Copyright (C) 2012 André Erdmann <dywi@×××××××.de> |
317 |
+# Distributed under the terms of the GNU General Public License; |
318 |
+# either version 2 of the License, or (at your option) any later version. |
319 |
+ |
320 |
+__all__ = [ 'create', 'get_class', ] |
321 |
+ |
322 |
+# the actual PackageDir implementation will be loaded after initialization |
323 |
+# of the config module (using importlib.import_module) |
324 |
+import importlib |
325 |
+ |
326 |
+import logging |
327 |
+ |
328 |
+import roverlay.config |
329 |
+ |
330 |
+ |
331 |
+# _package_dir_module is an imported module or None |
332 |
+_package_dir_module = None |
333 |
+_package_dir_class = None |
334 |
+ |
335 |
+_PACKAGE_DIR_IMPLEMENTATIONS = { |
336 |
+ 'none' : 'packagedir_base', |
337 |
+ 'default' : 'packagedir_ebuildmanifest', |
338 |
+ 'external:ebuild' : 'packagedir_ebuildmanifest', |
339 |
+# 'external:portage' : 'packagedir_portagemanifest', |
340 |
+# 'internal' : NotImplemented, |
341 |
+} |
342 |
+ |
343 |
+def _configure(): |
344 |
+ mf_impl = roverlay.config.get ( |
345 |
+ 'OVERLAY.manifest_implementation', |
346 |
+ _PACKAGE_DIR_IMPLEMENTATIONS ['default'] |
347 |
+ ) |
348 |
+ |
349 |
+ pkgdir_impl = _PACKAGE_DIR_IMPLEMENTATIONS.get ( mf_impl, None ) |
350 |
+ |
351 |
+ if pkgdir_impl is not None: |
352 |
+ global _package_dir_module |
353 |
+ _package_dir_module = importlib.import_module ( |
354 |
+ 'roverlay.overlay.pkgdir.' + pkgdir_impl |
355 |
+ ) |
356 |
+ |
357 |
+ global _package_dir_class |
358 |
+ if hasattr ( _package_dir_module, 'PackageDir' ): |
359 |
+ _package_dir_class = _package_dir_module.PackageDir |
360 |
+ else: |
361 |
+ _package_dir_class = _package_dir_module.PackageDirBase |
362 |
+ |
363 |
+ logging.getLogger ('pkgdir').debug ( |
364 |
+ 'Using {!r} as manifest implementation.'.format ( mf_impl ) |
365 |
+ ) |
366 |
+ else: |
367 |
+ # NameError, NotImplementedError, ...? |
368 |
+ raise Exception ( |
369 |
+ "PackageDir/Manifest implementation {} is unknown".format ( mf_impl ) |
370 |
+ ) |
371 |
+# --- end of configure (...) --- |
372 |
+ |
373 |
+def get_class(): |
374 |
+ if _package_dir_class is None: |
375 |
+ _configure() |
376 |
+ return _package_dir_class |
377 |
+# --- end of get_class (...) --- |
378 |
+ |
379 |
+def create ( *args, **kwargs ): |
380 |
+ return get_class() ( *args, **kwargs ) |
381 |
+# --- end of create (...) --- |
382 |
|
383 |
diff --git a/roverlay/overlay/manifest/__init__.py b/roverlay/overlay/pkgdir/manifest/__init__.py |
384 |
similarity index 89% |
385 |
rename from roverlay/overlay/manifest/__init__.py |
386 |
rename to roverlay/overlay/pkgdir/manifest/__init__.py |
387 |
index 3e8bcbf..f68c1b7 100644 |
388 |
--- a/roverlay/overlay/manifest/__init__.py |
389 |
+++ b/roverlay/overlay/pkgdir/manifest/__init__.py |
390 |
@@ -15,9 +15,9 @@ __all__ = [ 'create_manifest', ] |
391 |
import logging |
392 |
import threading |
393 |
|
394 |
-from roverlay.overlay.manifest import helpers |
395 |
+from roverlay.overlay.pkgdir.manifest import manifest_ebuild |
396 |
|
397 |
-_manifest_creation = helpers.ExternalManifestCreation ( lazy_init=True ) |
398 |
+_manifest_creation = manifest_ebuild.ExternalManifestCreation ( lazy_init=True ) |
399 |
# ExternalManifestCreation does not support threads (-> multiprocesses) |
400 |
# for one directory/overlay |
401 |
_manifest_lock = threading.Lock() |
402 |
|
403 |
diff --git a/roverlay/overlay/manifest/env.py b/roverlay/overlay/pkgdir/manifest/env.py |
404 |
similarity index 100% |
405 |
rename from roverlay/overlay/manifest/env.py |
406 |
rename to roverlay/overlay/pkgdir/manifest/env.py |
407 |
|
408 |
diff --git a/roverlay/overlay/manifest/helpers.py b/roverlay/overlay/pkgdir/manifest/manifest_ebuild.py |
409 |
similarity index 98% |
410 |
rename from roverlay/overlay/manifest/helpers.py |
411 |
rename to roverlay/overlay/pkgdir/manifest/manifest_ebuild.py |
412 |
index e30a5d4..b6e3fc0 100644 |
413 |
--- a/roverlay/overlay/manifest/helpers.py |
414 |
+++ b/roverlay/overlay/pkgdir/manifest/manifest_ebuild.py |
415 |
@@ -22,7 +22,7 @@ import subprocess |
416 |
|
417 |
from roverlay import config, strutil |
418 |
|
419 |
-from roverlay.overlay.manifest.env import ManifestEnv |
420 |
+from roverlay.overlay.pkgdir.manifest.env import ManifestEnv |
421 |
|
422 |
class ExternalManifestCreation ( object ): |
423 |
"""This class implements Manifest creation using the low level ebuild |
424 |
|
425 |
diff --git a/roverlay/overlay/metadata/__init__.py b/roverlay/overlay/pkgdir/metadata/__init__.py |
426 |
similarity index 98% |
427 |
rename from roverlay/overlay/metadata/__init__.py |
428 |
rename to roverlay/overlay/pkgdir/metadata/__init__.py |
429 |
index 49af593..fb6f374 100644 |
430 |
--- a/roverlay/overlay/metadata/__init__.py |
431 |
+++ b/roverlay/overlay/pkgdir/metadata/__init__.py |
432 |
@@ -15,7 +15,7 @@ __all__ = [ 'MetadataJob', ] |
433 |
|
434 |
import roverlay.config |
435 |
|
436 |
-from roverlay.overlay.metadata import nodes |
437 |
+from roverlay.overlay.pkgdir.metadata import nodes |
438 |
|
439 |
USE_FULL_DESCRIPTION = True |
440 |
|
441 |
|
442 |
diff --git a/roverlay/overlay/metadata/abstractnodes.py b/roverlay/overlay/pkgdir/metadata/abstractnodes.py |
443 |
similarity index 100% |
444 |
rename from roverlay/overlay/metadata/abstractnodes.py |
445 |
rename to roverlay/overlay/pkgdir/metadata/abstractnodes.py |
446 |
|
447 |
diff --git a/roverlay/overlay/metadata/nodes.py b/roverlay/overlay/pkgdir/metadata/nodes.py |
448 |
similarity index 98% |
449 |
rename from roverlay/overlay/metadata/nodes.py |
450 |
rename to roverlay/overlay/pkgdir/metadata/nodes.py |
451 |
index b3ea8d0..cdf29a8 100644 |
452 |
--- a/roverlay/overlay/metadata/nodes.py |
453 |
+++ b/roverlay/overlay/pkgdir/metadata/nodes.py |
454 |
@@ -12,7 +12,7 @@ and the MetadataRoot (<pkgmetadata>...</pkgmetadata>). |
455 |
""" |
456 |
|
457 |
# import abstract nodes |
458 |
-from roverlay.overlay.metadata.abstractnodes import \ |
459 |
+from roverlay.overlay.pkgdir.metadata.abstractnodes import \ |
460 |
MetadataNode, MetadataNodeNamedAccess, MetadataLeaf |
461 |
|
462 |
|
463 |
|
464 |
diff --git a/roverlay/overlay/package.py b/roverlay/overlay/pkgdir/packagedir_base.py |
465 |
similarity index 91% |
466 |
rename from roverlay/overlay/package.py |
467 |
rename to roverlay/overlay/pkgdir/packagedir_base.py |
468 |
index 297acf6..34b7ef5 100644 |
469 |
--- a/roverlay/overlay/package.py |
470 |
+++ b/roverlay/overlay/pkgdir/packagedir_base.py |
471 |
@@ -13,22 +13,28 @@ the existing package directory. |
472 |
Each PackageDir represents one package name (e.g. "seewave"). |
473 |
""" |
474 |
|
475 |
-__all__ = [ 'PackageDir', ] |
476 |
+__all__ = [ 'PackageDirBase', ] |
477 |
|
478 |
import os |
479 |
import sys |
480 |
import threading |
481 |
import shutil |
482 |
|
483 |
-from roverlay import util |
484 |
-from roverlay.overlay import manifest |
485 |
-from roverlay.packageinfo import PackageInfo |
486 |
-from roverlay.overlay.metadata import MetadataJob |
487 |
+from roverlay import util |
488 |
+from roverlay.packageinfo import PackageInfo |
489 |
+from roverlay.overlay.pkgdir.metadata import MetadataJob |
490 |
|
491 |
-SUPPRESS_EXCEPTIONS = True |
492 |
+class PackageDirBase ( object ): |
493 |
|
494 |
-class PackageDir ( object ): |
495 |
- EBUILD_SUFFIX = '.ebuild' |
496 |
+ EBUILD_SUFFIX = '.ebuild' |
497 |
+ SUPPRESS_EXCEPTIONS = True |
498 |
+ |
499 |
+ # MANIFEST_THREADSAFE (tri-state) |
500 |
+ # * None -- unknown (e.g. write_manifest() not implemented) |
501 |
+ # * False -- write_manifest() is not thread safe |
502 |
+ # * True -- ^ is thread safe |
503 |
+ # |
504 |
+ MANIFEST_THREADSAFE = None |
505 |
|
506 |
def __init__ ( self, |
507 |
name, logger, directory, get_header, runtime_incremental |
508 |
@@ -71,7 +77,6 @@ class PackageDir ( object ): |
509 |
|
510 |
# used to track changes for this package dir |
511 |
self.modified = False |
512 |
- self._manifest_package = None |
513 |
self._need_manifest = False |
514 |
self._need_metadata = False |
515 |
# --- end of __init__ (...) --- |
516 |
@@ -92,6 +97,13 @@ class PackageDir ( object ): |
517 |
return False |
518 |
# --- end of remove_ebuild_file (...) --- |
519 |
|
520 |
+ def _scan_add_package ( self, efile, pvr ): |
521 |
+ p = PackageInfo ( |
522 |
+ physical_only=True, pvr=pvr, ebuild_file=efile |
523 |
+ ) |
524 |
+ self._packages [ p ['ebuild_verstr'] ] = p |
525 |
+ # --- end of _scan_add_package (...) --- |
526 |
+ |
527 |
def add ( self, package_info, add_if_physical=False ): |
528 |
"""Adds a package to this PackageDir. |
529 |
|
530 |
@@ -123,8 +135,10 @@ class PackageDir ( object ): |
531 |
) |
532 |
) |
533 |
else: |
534 |
- # package has been added to this overlay before |
535 |
- self.logger.info ( |
536 |
+ # package has been added to this packagedir before, |
537 |
+ # this most likely happens if it is available via several |
538 |
+ # remotes |
539 |
+ self.logger.debug ( |
540 |
"'{PN}-{PVR}.ebuild' already exists, cannot add it!".format ( |
541 |
PN=self.name, PVR=shortver |
542 |
) |
543 |
@@ -355,8 +369,8 @@ class PackageDir ( object ): |
544 |
else: |
545 |
# $PN does not match directory name, warn about that |
546 |
self.logger.warning ( |
547 |
- "$PN does not match directory name, ignoring {!r}.".\ |
548 |
- format ( f ) |
549 |
+ "$PN {!r} does not match directory name, ignoring {!r}.".\ |
550 |
+ format ( pn, f ) |
551 |
) |
552 |
except: |
553 |
self.logger.warning ( |
554 |
@@ -368,10 +382,15 @@ class PackageDir ( object ): |
555 |
if os.path.isfile ( self.physical_location + os.sep + 'Manifest' ): |
556 |
for pvr, efile in scan_ebuilds(): |
557 |
if pvr not in self._packages: |
558 |
- p = PackageInfo ( |
559 |
- physical_only=True, pvr=pvr, ebuild_file=efile |
560 |
- ) |
561 |
- self._packages [ p ['ebuild_verstr'] ] = p |
562 |
+ try: |
563 |
+ self._scan_add_package ( efile, pvr ) |
564 |
+ except ValueError as ve: |
565 |
+ self.logger.error ( |
566 |
+ "Failed to add ebuild {!r} due to {!r}.".format ( |
567 |
+ efile, ve |
568 |
+ ) |
569 |
+ ) |
570 |
+ raise |
571 |
# --- end of scan (...) --- |
572 |
|
573 |
def show ( self, stream=sys.stderr ): |
574 |
@@ -414,7 +433,7 @@ class PackageDir ( object ): |
575 |
* write_manifest -- if set and False: don't write the Manifest file |
576 |
* write_metadata -- if set and False: don't write the metadata file |
577 |
* overwrite_ebuilds -- whether to overwrite ebuilds, |
578 |
- None means autodetect, enable overwriting |
579 |
+ None means autodetect: enable overwriting |
580 |
if not modified since last write |
581 |
Defaults to False |
582 |
* cleanup -- clean up after writing |
583 |
@@ -449,10 +468,14 @@ class PackageDir ( object ): |
584 |
# write ebuilds |
585 |
if self.modified and write_ebuilds: |
586 |
success = self.write_ebuilds ( |
587 |
- # None ~ not modified |
588 |
- overwrite = overwrite_ebuilds \ |
589 |
- if overwrite_ebuilds is not None \ |
590 |
- else not self.modified |
591 |
+ # ( overwrite == None ) <= not modified, which is not |
592 |
+ # possible in this if-branch |
593 |
+ |
594 |
+ overwrite = bool ( overwrite_ebuilds ) |
595 |
+ |
596 |
+# overwrite = overwrite_ebuilds \ |
597 |
+# if overwrite_ebuilds is not None \ |
598 |
+# else not self.modified |
599 |
) |
600 |
|
601 |
# cautious: remove ebuilds after writing them |
602 |
@@ -485,8 +508,7 @@ class PackageDir ( object ): |
603 |
arguments: |
604 |
* shared_fh -- if set and not None: don't use own file handles |
605 |
(i.e. write files), write everything into shared_fh |
606 |
- * overwrite -- write ebuilds that have been written before, |
607 |
- defaults to True |
608 |
+ * overwrite -- write ebuilds that have been written before |
609 |
""" |
610 |
ebuild_header = self.get_header() |
611 |
|
612 |
@@ -574,33 +596,9 @@ class PackageDir ( object ): |
613 |
raises: |
614 |
* Exception if no ebuild exists |
615 |
""" |
616 |
- |
617 |
- # it should be sufficient to call create_manifest for one ebuild, |
618 |
- # choosing the latest one that exists in self.physical_location and |
619 |
- # has enough data (DISTDIR, EBUILD_FILE) for this task. |
620 |
- # Additionally, all DISTDIRs (multiple repos, sub directories) have |
621 |
- # to be collected and passed to Manifest creation. |
622 |
- # => collect suitable PackageInfo objects from self._packages |
623 |
- # |
624 |
- pkgs_for_manifest = tuple ( |
625 |
- p for p in self._packages.values() \ |
626 |
- if p.has ( 'distdir', 'ebuild_file' ) |
627 |
+ raise NotImplementedError ( |
628 |
+ "write_manifest() needs to be implemented by derived classes." |
629 |
) |
630 |
- |
631 |
- if pkgs_for_manifest: |
632 |
- if manifest.create_manifest ( pkgs_for_manifest, nofail=False ): |
633 |
- self._need_manifest = False |
634 |
- return True |
635 |
- elif ignore_empty: |
636 |
- return True |
637 |
- else: |
638 |
- raise Exception ( |
639 |
- 'In {mydir}: No ebuild written so far! ' |
640 |
- 'I really don\'t know what do to!'.format ( |
641 |
- mydir=self.physical_location |
642 |
- ) ) |
643 |
- |
644 |
- return False |
645 |
# --- end of write_manifest (...) --- |
646 |
|
647 |
def write_metadata ( self, shared_fh=None ): |
648 |
|
649 |
diff --git a/roverlay/overlay/pkgdir/packagedir_ebuildmanifest.py b/roverlay/overlay/pkgdir/packagedir_ebuildmanifest.py |
650 |
new file mode 100644 |
651 |
index 0000000..e1961dd |
652 |
--- /dev/null |
653 |
+++ b/roverlay/overlay/pkgdir/packagedir_ebuildmanifest.py |
654 |
@@ -0,0 +1,56 @@ |
655 |
+# R overlay -- overlay package, package directory |
656 |
+# -*- coding: utf-8 -*- |
657 |
+# Copyright (C) 2012 André Erdmann <dywi@×××××××.de> |
658 |
+# Distributed under the terms of the GNU General Public License; |
659 |
+# either version 2 of the License, or (at your option) any later version. |
660 |
+ |
661 |
+__all__ = [ 'PackageDir', ] |
662 |
+ |
663 |
+from roverlay.overlay.pkgdir import manifest |
664 |
+from roverlay.overlay.pkgdir import packagedir_base |
665 |
+ |
666 |
+class PackageDir ( packagedir_base.PackageDirBase ): |
667 |
+ """ |
668 |
+ PackageDir class that uses the ebuild executable for Manifest writing. |
669 |
+ """ |
670 |
+ |
671 |
+ MANIFEST_THREADSAFE = False |
672 |
+ |
673 |
+ def write_manifest ( self, ignore_empty=False ): |
674 |
+ """Generates and writes the Manifest file for this package. |
675 |
+ |
676 |
+ expects: called after writing metadata/ebuilds |
677 |
+ |
678 |
+ returns: success (True/False) |
679 |
+ |
680 |
+ raises: |
681 |
+ * Exception if no ebuild exists |
682 |
+ """ |
683 |
+ |
684 |
+ # it should be sufficient to call create_manifest for one ebuild, |
685 |
+ # choosing the latest one that exists in self.physical_location and |
686 |
+ # has enough data (DISTDIR, EBUILD_FILE) for this task. |
687 |
+ # Additionally, all DISTDIRs (multiple repos, sub directories) have |
688 |
+ # to be collected and passed to Manifest creation. |
689 |
+ # => collect suitable PackageInfo objects from self._packages |
690 |
+ # |
691 |
+ pkgs_for_manifest = tuple ( |
692 |
+ p for p in self._packages.values() \ |
693 |
+ if p.has ( 'distdir', 'ebuild_file' ) |
694 |
+ ) |
695 |
+ |
696 |
+ if pkgs_for_manifest: |
697 |
+ if manifest.create_manifest ( pkgs_for_manifest, nofail=False ): |
698 |
+ self._need_manifest = False |
699 |
+ return True |
700 |
+ elif ignore_empty: |
701 |
+ return True |
702 |
+ else: |
703 |
+ raise Exception ( |
704 |
+ 'In {mydir}: No ebuild written so far! ' |
705 |
+ 'I really don\'t know what do to!'.format ( |
706 |
+ mydir=self.physical_location |
707 |
+ ) ) |
708 |
+ |
709 |
+ return False |
710 |
+ # --- end of write_manifest (...) --- |