1 |
commit: daed3bde53ce1fec42ad806813bb2798321b76d4 |
2 |
Author: André Erdmann <dywi <AT> mailerd <DOT> de> |
3 |
AuthorDate: Tue Jul 24 16:21:07 2012 +0000 |
4 |
Commit: André Erdmann <dywi <AT> mailerd <DOT> de> |
5 |
CommitDate: Tue Jul 24 16:21:07 2012 +0000 |
6 |
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=daed3bde |
7 |
|
8 |
overlay: split incremental/runtime_incremental |
9 |
|
10 |
Also added a _writeable bool to Overlay which controls |
11 |
whether writing is allowed or not. |
12 |
|
13 |
geändert: roverlay/overlay/category.py |
14 |
geändert: roverlay/overlay/creator.py |
15 |
geändert: roverlay/overlay/package.py |
16 |
geändert: roverlay/overlay/root.py |
17 |
|
18 |
--- |
19 |
roverlay/overlay/category.py | 36 +++++++++----- |
20 |
roverlay/overlay/creator.py | 20 +++---- |
21 |
roverlay/overlay/package.py | 23 ++++++-- |
22 |
roverlay/overlay/root.py | 113 +++++++++++++++++++++++++++++------------ |
23 |
4 files changed, 129 insertions(+), 63 deletions(-) |
24 |
|
25 |
diff --git a/roverlay/overlay/category.py b/roverlay/overlay/category.py |
26 |
index fab6373..a0b1607 100644 |
27 |
--- a/roverlay/overlay/category.py |
28 |
+++ b/roverlay/overlay/category.py |
29 |
@@ -16,25 +16,35 @@ class Category ( object ): |
30 |
|
31 |
WRITE_JOBCOUNT = 3 |
32 |
|
33 |
- def __init__ ( self, name, logger, directory, get_header, incremental ): |
34 |
+ def __init__ ( self, |
35 |
+ name, logger, directory, get_header, runtime_incremental |
36 |
+ ): |
37 |
"""Initializes a overlay/portage category (such as 'app-text', 'sci-R'). |
38 |
|
39 |
arguments: |
40 |
- * name -- name of the category |
41 |
- * logger -- parent logger |
42 |
- * directory -- filesystem location |
43 |
- * get_header -- function that returns an ebuild header |
44 |
+ * name -- name of the category |
45 |
+ * logger -- parent logger |
46 |
+ * directory -- filesystem location |
47 |
+ * get_header -- function that returns an ebuild header |
48 |
+ * runtime_incremental -- enable/disable runtime incremental writing |
49 |
+ for this category (and all created PackageDirs) |
50 |
""" |
51 |
- self.logger = logger.getChild ( name ) |
52 |
- self.name = name |
53 |
- self._lock = threading.RLock() |
54 |
- self._subdirs = dict() |
55 |
- self.physical_location = directory |
56 |
- self.get_header = get_header |
57 |
- self.incremental = incremental |
58 |
+ self.logger = logger.getChild ( name ) |
59 |
+ self.name = name |
60 |
+ self._lock = threading.RLock() |
61 |
+ self._subdirs = dict() |
62 |
+ self.physical_location = directory |
63 |
+ self.get_header = get_header |
64 |
+ self.runtime_incremental = runtime_incremental |
65 |
# --- end of __init__ (...) --- |
66 |
|
67 |
def _get_package_dir ( self, pkg_name ): |
68 |
+ """Returns a PackageDir object for pkg_name. |
69 |
+ (so that <new object>.name == pkg_name and pkg_name in self._subdirs) |
70 |
+ |
71 |
+ arguments: |
72 |
+ * pkg_name -- |
73 |
+ """ |
74 |
if not pkg_name in self._subdirs: |
75 |
self._lock.acquire() |
76 |
try: |
77 |
@@ -44,7 +54,7 @@ class Category ( object ): |
78 |
logger = self.logger, |
79 |
directory = self.physical_location + os.sep + pkg_name, |
80 |
get_header = self.get_header, |
81 |
- incremental = self.incremental |
82 |
+ runtime_incremental = self.runtime_incremental |
83 |
) |
84 |
self._subdirs [pkg_name] = newpkg |
85 |
finally: |
86 |
|
87 |
diff --git a/roverlay/overlay/creator.py b/roverlay/overlay/creator.py |
88 |
index d12e0a5..60e9517 100644 |
89 |
--- a/roverlay/overlay/creator.py |
90 |
+++ b/roverlay/overlay/creator.py |
91 |
@@ -82,18 +82,16 @@ class OverlayCreator ( object ): |
92 |
# this queue is used to propagate exceptions from threads |
93 |
self._err_queue = errorqueue.ErrorQueue() |
94 |
|
95 |
- self.can_write_overlay = allow_write |
96 |
- self.write_incremental = allow_write and USE_INCREMENTAL_WRITE |
97 |
- |
98 |
# init overlay using config values |
99 |
self.overlay = Overlay ( |
100 |
- name=config.get_or_fail ( 'OVERLAY.name' ), |
101 |
- logger=self.logger, |
102 |
- directory=config.get_or_fail ( 'OVERLAY.dir' ), |
103 |
- default_category= config.get_or_fail ( 'OVERLAY.category' ), |
104 |
- eclass_files=config.get ( 'OVERLAY.eclass_files', None ), |
105 |
- ebuild_header=config.get ( 'EBUILD.default_header', None ), |
106 |
- incremental=self.write_incremental |
107 |
+ name = config.get_or_fail ( 'OVERLAY.name' ), |
108 |
+ logger = self.logger, |
109 |
+ directory = config.get_or_fail ( 'OVERLAY.dir' ), |
110 |
+ default_category = config.get_or_fail ( 'OVERLAY.category' ), |
111 |
+ eclass_files = config.get ( 'OVERLAY.eclass_files', None ), |
112 |
+ ebuild_header = config.get ( 'EBUILD.default_header', None ), |
113 |
+ incremental = USE_INCREMENTAL_WRITE, |
114 |
+ write_allowed = allow_write |
115 |
) |
116 |
|
117 |
self.depresolver = easyresolver.setup ( self._err_queue ) |
118 |
@@ -219,7 +217,7 @@ class OverlayCreator ( object ): |
119 |
|
120 |
arguments: |
121 |
""" |
122 |
- if self.can_write_overlay: |
123 |
+ if self.overlay.writeable(): |
124 |
start = time.time() |
125 |
self.overlay.write() |
126 |
self._timestamp ( "overlay written", start ) |
127 |
|
128 |
diff --git a/roverlay/overlay/package.py b/roverlay/overlay/package.py |
129 |
index d2662bd..bedbce3 100644 |
130 |
--- a/roverlay/overlay/package.py |
131 |
+++ b/roverlay/overlay/package.py |
132 |
@@ -13,15 +13,25 @@ SUPPRESS_EXCEPTIONS = True |
133 |
class PackageDir ( object ): |
134 |
EBUILD_SUFFIX = '.ebuild' |
135 |
|
136 |
- def __init__ ( self, name, logger, directory, get_header, incremental ): |
137 |
+ def __init__ ( self, |
138 |
+ name, logger, directory, get_header, runtime_incremental |
139 |
+ ): |
140 |
"""Initializes a PackageDir which contains ebuilds, metadata and |
141 |
a Manifest file. |
142 |
|
143 |
arguments: |
144 |
- * name -- name of the directory (${PN} in ebuilds) |
145 |
- * logger -- parent logger |
146 |
- * directory -- filesystem location of this PackageDir |
147 |
- * get_header -- function that returns an ebuild header |
148 |
+ * name -- name of the directory (${PN} in ebuilds) |
149 |
+ * logger -- parent logger |
150 |
+ * directory -- filesystem location of this PackageDir |
151 |
+ * get_header -- function that returns an ebuild header |
152 |
+ * runtime_incremental -- enable/disable runtime incremental ebuild |
153 |
+ writing. This trades speed (disabled) for |
154 |
+ memory consumption (enabled) 'cause it will |
155 |
+ write _all_ successfully created ebuilds |
156 |
+ directly after they've been created. |
157 |
+ Writing all ebuilds at once is generally faster |
158 |
+ (+threading), but all PackageInfos must be |
159 |
+ kept in memory for that. |
160 |
""" |
161 |
self.logger = logger.getChild ( name ) |
162 |
self.name = name |
163 |
@@ -30,7 +40,7 @@ class PackageDir ( object ): |
164 |
self._packages = dict() |
165 |
self.physical_location = directory |
166 |
self.get_header = get_header |
167 |
- self.runtime_incremental = incremental |
168 |
+ self.runtime_incremental = runtime_incremental |
169 |
|
170 |
self._metadata = MetadataJob ( |
171 |
filepath = self.physical_location + os.sep + 'metadata.xml', |
172 |
@@ -250,6 +260,7 @@ class PackageDir ( object ): |
173 |
self._need_manifest = True |
174 |
self._need_metadata = True |
175 |
self.modified = True |
176 |
+ # FIXME: delete PackageInfo if ebuild creation did not succeed |
177 |
if self.runtime_incremental: |
178 |
with self._lock: |
179 |
return self.write_ebuilds ( overwrite=False ) |
180 |
|
181 |
diff --git a/roverlay/overlay/root.py b/roverlay/overlay/root.py |
182 |
index 219fd70..ae3267f 100644 |
183 |
--- a/roverlay/overlay/root.py |
184 |
+++ b/roverlay/overlay/root.py |
185 |
@@ -22,23 +22,49 @@ class Overlay ( object ): |
186 |
|
187 |
def __init__ ( |
188 |
self, |
189 |
- name, logger, directory, |
190 |
- default_category, eclass_files, |
191 |
+ name, |
192 |
+ logger, |
193 |
+ directory, |
194 |
+ default_category, |
195 |
+ eclass_files, |
196 |
ebuild_header, |
197 |
- incremental |
198 |
+ write_allowed, |
199 |
+ incremental, |
200 |
+ runtime_incremental=True |
201 |
): |
202 |
- self.name = name |
203 |
- self.logger = logger.getChild ( 'overlay' ) |
204 |
- self.physical_location = directory |
205 |
- self.default_category = default_category |
206 |
- self.eclass_files = eclass_files |
207 |
+ """Initializes an overlay. |
208 |
|
209 |
- self.ignore_existing_ebuilds = False |
210 |
- |
211 |
- self._profiles_dir = self.physical_location + os.sep + 'profiles' |
212 |
- self._catlock = threading.Lock() |
213 |
- self._categories = dict() |
214 |
- self._header = EbuildHeader ( ebuild_header ) |
215 |
+ arguments: |
216 |
+ * name -- name of this overlay |
217 |
+ * logger -- parent logger to use |
218 |
+ * directory -- filesystem location of this overlay |
219 |
+ * default_category -- category of packages being added without a |
220 |
+ specific category |
221 |
+ * eclass_files -- eclass files to import and |
222 |
+ inherit in all ebuilds |
223 |
+ * ebuild_header -- the header text included in all created ebuilds |
224 |
+ * write_allowed -- whether writing is allowed |
225 |
+ * incremental -- enable/disable incremental writing: |
226 |
+ use already existing ebuilds (don't recreate |
227 |
+ them) |
228 |
+ * runtime_incremental -- see package.py:PackageDir.__init__ (...), |
229 |
+ Defaults to ?FIXME? |
230 |
+ """ |
231 |
+ self.name = name |
232 |
+ self.logger = logger.getChild ( 'overlay' ) |
233 |
+ self.physical_location = directory |
234 |
+ self.default_category = default_category |
235 |
+ |
236 |
+ self._eclass_files = eclass_files |
237 |
+ #self._incremental = incremental |
238 |
+ # disable runtime_incremental if writing not allowed |
239 |
+ self._runtime_incremental = write_allowed and runtime_incremental |
240 |
+ self._writeable = write_allowed |
241 |
+ |
242 |
+ self._profiles_dir = self.physical_location + os.sep + 'profiles' |
243 |
+ self._catlock = threading.Lock() |
244 |
+ self._categories = dict() |
245 |
+ self._header = EbuildHeader ( ebuild_header ) |
246 |
|
247 |
# fixme or ignore: calculating eclass names twice, |
248 |
# once here and another time when calling _init_overlay |
249 |
@@ -46,13 +72,12 @@ class Overlay ( object ): |
250 |
self._get_eclass_import_info ( only_eclass_names=True ) |
251 |
) ) |
252 |
|
253 |
- self.incremental = incremental |
254 |
- if self.incremental: |
255 |
+ #if self._incremental: |
256 |
+ if incremental: |
257 |
# this is multiple-run incremental writing (in contrast to runtime |
258 |
# incremental writing, which writes ebuilds as soon as they're |
259 |
- # ready) FIXME: split incremental <-> runtime_incremental |
260 |
+ # ready) |
261 |
self.scan() |
262 |
- |
263 |
# --- end of __init__ (...) --- |
264 |
|
265 |
def _get_category ( self, category ): |
266 |
@@ -70,7 +95,7 @@ class Overlay ( object ): |
267 |
self.logger, |
268 |
self.physical_location + os.sep + category, |
269 |
get_header=self._header.get, |
270 |
- incremental=self.incremental |
271 |
+ runtime_incremental=self._runtime_incremental |
272 |
) |
273 |
self._categories [category] = newcat |
274 |
finally: |
275 |
@@ -89,9 +114,9 @@ class Overlay ( object ): |
276 |
|
277 |
raises: AssertionError if a file does not end with '.eclass'. |
278 |
""" |
279 |
- if self.eclass_files: |
280 |
+ if self._eclass_files: |
281 |
|
282 |
- for eclass in self.eclass_files: |
283 |
+ for eclass in self._eclass_files: |
284 |
dest = os.path.splitext ( os.path.basename ( eclass ) ) |
285 |
|
286 |
if dest[1] == '.eclass' or ( not dest[1] and not '.' in dest[0] ): |
287 |
@@ -116,7 +141,7 @@ class Overlay ( object ): |
288 |
* Exception if copying fails |
289 |
""" |
290 |
|
291 |
- if self.eclass_files: |
292 |
+ if self._eclass_files: |
293 |
# import eclass files |
294 |
eclass_dir = self.physical_location + os.sep + 'eclass' |
295 |
try: |
296 |
@@ -240,6 +265,10 @@ class Overlay ( object ): |
297 |
yield kwargs |
298 |
# --- end of list_rule_kwargs (...) --- |
299 |
|
300 |
+ def readonly ( self ): |
301 |
+ return not self._writeable |
302 |
+ # --- end of readonly (...) --- |
303 |
+ |
304 |
def scan ( self, **kw ): |
305 |
def scan_categories(): |
306 |
for x in os.listdir ( self.physical_location ): |
307 |
@@ -271,6 +300,10 @@ class Overlay ( object ): |
308 |
cat.show ( **show_kw ) |
309 |
# --- end of show (...) --- |
310 |
|
311 |
+ def writeable ( self ): |
312 |
+ return self._writeable |
313 |
+ # --- end of writeable (...) --- |
314 |
+ |
315 |
def write ( self ): |
316 |
"""Writes the overlay to its physical location (filesystem), including |
317 |
metadata and Manifest files as well as cleanup actions. |
318 |
@@ -285,14 +318,21 @@ class Overlay ( object ): |
319 |
! TODO/FIXME/DOC: This is not thread-safe, it's expected to be called |
320 |
when ebuild creation is done. |
321 |
""" |
322 |
- self._init_overlay ( reimport_eclass=True ) |
323 |
- |
324 |
- for cat in self._categories.values(): |
325 |
- cat.write ( |
326 |
- overwrite_ebuilds=False, |
327 |
- keep_n_ebuilds=config.get ( 'OVERLAY.keep_nth_latest', None ), |
328 |
- cautious=True |
329 |
- ) |
330 |
+ if self._writeable: |
331 |
+ self._init_overlay ( reimport_eclass=True ) |
332 |
+ |
333 |
+ for cat in self._categories.values(): |
334 |
+ cat.write ( |
335 |
+ overwrite_ebuilds=False, |
336 |
+ keep_n_ebuilds=config.get ( 'OVERLAY.keep_nth_latest', None ), |
337 |
+ cautious=True |
338 |
+ ) |
339 |
+ else: |
340 |
+ # FIXME debug print |
341 |
+ print ( |
342 |
+ "Dropped write request for readonly overlay {}!".format ( |
343 |
+ self.name |
344 |
+ ) ) |
345 |
# --- end of write (...) --- |
346 |
|
347 |
def write_manifest ( self, **manifest_kw ): |
348 |
@@ -305,7 +345,14 @@ class Overlay ( object ): |
349 |
|
350 |
returns: None (implicit) |
351 |
""" |
352 |
- # FIXME: it would be good to ensure that profiles/categories exist |
353 |
- for cat in self._categories.values(): |
354 |
- cat.write_manifest ( **manifest_kw ) |
355 |
+ if self._writeable: |
356 |
+ # FIXME: it would be good to ensure that profiles/categories exist |
357 |
+ for cat in self._categories.values(): |
358 |
+ cat.write_manifest ( **manifest_kw ) |
359 |
+ else: |
360 |
+ # FIXME debug print |
361 |
+ print ( |
362 |
+ "Dropped write_manifest request for readonly overlay {}!".format ( |
363 |
+ self.name |
364 |
+ ) ) |
365 |
# --- end of write_manifest (...) --- |