[gentoo-commits] proj/R_overlay:master commit in: roverlay/overlay/metadata/, roverlay/overlay/

[André Erdmann]

commit:     041f770675b288ef3f6a04670b9c4e399e4e26f0
Author:     André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Thu Aug  2 15:10:56 2012 +0000
Commit:     André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Thu Aug  2 15:10:56 2012 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=041f7706

overlay: comments, string formatting and strutil

also readded RERAISE to category.py's threaded overlay writing

---
 roverlay/overlay/category.py               |   10 +++++---
 roverlay/overlay/creator.py                |   13 +----------
 roverlay/overlay/metadata/abstractnodes.py |   30 +++++++++++++--------------
 roverlay/overlay/metadata/nodes.py         |    3 +-
 roverlay/overlay/package.py                |   28 +++++++++++--------------
 roverlay/overlay/root.py                   |   23 +++++++++++++--------
 6 files changed, 49 insertions(+), 58 deletions(-)

diff --git a/roverlay/overlay/category.py b/roverlay/overlay/category.py
index 3551b81..7bf93c8 100644
--- a/roverlay/overlay/category.py
+++ b/roverlay/overlay/category.py
@@ -158,10 +158,9 @@ class Category ( object ):
 					pkg.write ( write_manifest=False, **write_kw )
 				except queue.Empty:
 					break
-				#except ( Exception, KeyboardInterrupt ) as e:
 				except Exception as e:
-					# FIXME: reintroduce RERAISE
-					self.logger.exception ( e )
+					#self.logger.exception ( e )
+					self.RERAISE = e
 		# --- end of run_write_queue (...) ---
 
 		if len ( self._subdirs ) == 0: return
@@ -177,7 +176,7 @@ class Category ( object ):
 
 		max_jobs = self.__class__.WRITE_JOBCOUNT
 
-		# FIXME/TODO: what's an reasonable number of min package dirs to
+		# What's an reasonable number of min package dirs to
 		# start threaded writing?
 		# Ignoring it for now (and expecting enough pkg dirs)
 		if max_jobs > 1:
@@ -201,6 +200,9 @@ class Category ( object ):
 			for w in workers: w.start()
 			for w in workers: w.join()
 
+			if hasattr ( self, 'RERAISE' ) and self.RERAISE is not None:
+				raise self.RERAISE
+
 			self.remove_empty()
 
 			# write manifest files

diff --git a/roverlay/overlay/creator.py b/roverlay/overlay/creator.py
index 5f346fe..6b42867 100644
--- a/roverlay/overlay/creator.py
+++ b/roverlay/overlay/creator.py
@@ -196,7 +196,7 @@ class OverlayCreator ( object ):
 		delta = _stop - start
 
 		self.logger.debug (
-			"timestamp: %s (after %f seconds)" % ( description, delta )
+			"timestamp: {} (after {} seconds)".format ( description, delta )
 		)
 		return _stop
 	# --- end of _timestamp (...) ---
@@ -214,10 +214,7 @@ class OverlayCreator ( object ):
 	# --- end of add_package (...) ---
 
 	def write_overlay ( self ):
-		"""Writes the overlay.
-
-		arguments:
-		"""
+		"""Writes the overlay."""
 		if self.overlay.writeable():
 			start = time.time()
 			self.overlay.write()
@@ -306,7 +303,6 @@ class OverlayCreator ( object ):
 
 				if do_close:
 					self._err_queue.push ( context=-1, error=None )
-					# fixme: remove enabled?
 					for w in self._workers: w.enabled = False
 				else:
 					for w in self._workers: w.stop_when_empty()
@@ -373,11 +369,6 @@ class OverlayCreator ( object ):
 		arguments:
 		* package_info --
 		"""
-		# ... TODO
-		#  * increase the number of successful/failed packages,
-		#  * request an incremental write to save memory etc.
-
-		# if <>:
 		if package_info ['ebuild'] is not None:
 			self.create_success.inc()
 			if package_info.overlay_package_ref.new_ebuild():

diff --git a/roverlay/overlay/metadata/abstractnodes.py b/roverlay/overlay/metadata/abstractnodes.py
index 284ea20..060d369 100644
--- a/roverlay/overlay/metadata/abstractnodes.py
+++ b/roverlay/overlay/metadata/abstractnodes.py
@@ -2,6 +2,7 @@
 # Copyright 2006-2012 Gentoo Foundation
 # Distributed under the terms of the GNU General Public License v2
 
+from roverlay import strutil
 import re
 import textwrap
 
@@ -16,7 +17,6 @@ class _MetadataBasicNode ( object ):
 	"""
 
 	# these chars lead to metadata.bad: invalid token
-	# TODO/FIXME: look into repoman's / portage's source to get _all_ chars
 	INVALID_CHARS = "&<>"
 
 	def __init__ ( self, name, flags ):
@@ -92,7 +92,6 @@ class _MetadataBasicNode ( object ):
 		Does nothing if self._verify is not implemented, else dies on error.
 		"""
 		if hasattr ( self, '_verify' ) and not self._verify():
-			# todo, verify could return ( Status, ErrorMessages ) etc.
 			raise Exception ( "verification failed for a metadata node." )
 	# --- end of _do_verify (...) ---
 
@@ -197,10 +196,14 @@ class MetadataLeaf ( _MetadataBasicNode ):
 		if not self._text_wrapper is None:
 			self._text_wrapper.subsequent_indent = self.text_indent
 
-	def _value_str ( self ):
+	def _default_value_str ( self ):
 		"""Returns the value string. Derived classes may override this."""
 		#if self.value_format == ?: format value ~
-		return str ( self.value )
+		return re.sub (
+			"[" + self.__class__.INVALID_CHARS + "]",
+			'',
+			strutil.ascii_filter ( str ( self.value ) )
+		)
 	# --- end of _value_str (...) ---
 
 	def _pretty_value_str ( self ):
@@ -208,7 +211,6 @@ class MetadataLeaf ( _MetadataBasicNode ):
 		Not used here, but subclasses can use it by simply writing
 		'_value_str = MetadataLeaf._pretty_value_str' in the class body.
 		"""
-		# FIXME/TODO: could move this func to util
 		if not self.value: return ""
 
 		if self._text_wrapper is None:
@@ -218,13 +220,8 @@ class MetadataLeaf ( _MetadataBasicNode ):
 				width=self.linewidth if hasattr ( self, 'linewidth' ) else 50
 			)
 
-		val_lines = self._text_wrapper.wrap (
-			re.sub (
-				"[{}]".format ( self.__class__.INVALID_CHARS ),
-				'',
-				self.value
-			)
-		)
+		val_lines = self._text_wrapper.wrap ( self._default_value_str() )
+
 		if len ( val_lines ) < 1:
 			# why?
 			return ""
@@ -242,10 +239,11 @@ class MetadataLeaf ( _MetadataBasicNode ):
 		self._do_verify()
 		if self.print_node_name:
 			return "{indent}<{name}{flags}>{value}</{name}>".format (
-				indent=self.indent,
-				name=self.name,
-				flags=self._flagstr(),
-				value=self._value_str(),
+				indent = self.indent,
+				name   = self.name,
+				flags  = self._flagstr(),
+				value  = self._value_str() \
+					if hasattr ( self, '_value_str' ) else self._default_value_str()
 			)
 		else:
 			# not very useful, but allows to insert strings as nodes

diff --git a/roverlay/overlay/metadata/nodes.py b/roverlay/overlay/metadata/nodes.py
index 0002267..5f97181 100644
--- a/roverlay/overlay/metadata/nodes.py
+++ b/roverlay/overlay/metadata/nodes.py
@@ -2,7 +2,6 @@
 # Copyright 2006-2012 Gentoo Foundation
 # Distributed under the terms of the GNU General Public License v2
 
-from roverlay import util
 # import abstract nodes
 from roverlay.overlay.metadata.abstractnodes import \
 	MetadataNode, MetadataNodeNamedAccess, MetadataLeaf
@@ -71,7 +70,7 @@ class MetadataRoot ( MetadataNodeNamedAccess ):
 		if not self.empty():
 			fh.write ( MetadataRoot.HEADER )
 			fh.write ( '\n' )
-			fh.write ( util.ascii_filter ( self.to_str() ) )
+			fh.write ( self.to_str() )
 			fh.write ( '\n' )
 			return True
 		else:

diff --git a/roverlay/overlay/package.py b/roverlay/overlay/package.py
index 434b1b1..722be8b 100644
--- a/roverlay/overlay/package.py
+++ b/roverlay/overlay/package.py
@@ -122,8 +122,6 @@ class PackageDir ( object ):
 		if added:
 			# add a link to this PackageDir into the package info,
 			# !! package_info <-> self (double-linked)
-			# FIXME: remove physical_only flag from PackageInfo if
-			#         overlay_package_ref can be used for that
 			package_info.overlay_package_ref = self
 			return True
 		else:
@@ -267,8 +265,17 @@ class PackageDir ( object ):
 			"Kept {kept}/{total} ebuilds.".format ( kept=kept, total=ecount )
 		)
 
-		# FIXME: Manifest is now invalid and dir could be "empty" (no ebuilds)
-		# FIXME: force metadata regeneration
+		if self._need_metadata:
+			self.generate_metadata ( skip_if_existent=False )
+
+		# Manifest is now invalid,
+		#  need_manifest is set to True in purge_package()
+		#
+		# metadata will be re-written when calling write()
+		#
+		# dir could be "empty" (no ebuilds),
+		#  which is solved when calling fs_cleanup(),
+		#  implicitly called by write()
 	# --- end of keep_nth_latest (...) ---
 
 	def list_versions ( self ):
@@ -296,6 +303,7 @@ class PackageDir ( object ):
 			p = self._packages [pvr]
 			del self._packages [pvr]
 			self._remove_ebuild_file ( p )
+			self._need_metadata = True
 			return p
 		except Exception as e:
 			self.logger.exception ( e )
@@ -450,7 +458,6 @@ class PackageDir ( object ):
 				self.virtual_cleanup()
 				self.fs_cleanup()
 
-			# FIXME / TODO call fs_cleanup
 		# -- lock
 		return success
 	# --- end of write (...) ---
@@ -483,10 +490,6 @@ class PackageDir ( object ):
 				fh.write ( str ( ebuild ) )
 				fh.write ( '\n' )
 
-				# adjust owner/perm? TODO
-				#if shared_fh is None:
-				#	chmod 0644 or 0444
-				#	chown 250.250
 				_success = True
 			except IOError as e:
 				self.logger.exception ( e )
@@ -574,13 +577,6 @@ class PackageDir ( object ):
 		elif ignore_empty:
 			return True
 		else:
-			# FIXME: debug statements
-			# FIXME: remove excpetion, maybe delete Manifest in this case,..
-			for pvr, p in self._packages.items():
-				print ( "{} {} ebuild={} efile={} has={}".format (
-					pvr, p, p.has ('ebuild'), p ['ebuild_file'], self.has_ebuilds()
-				) )
-
 			raise Exception (
 				'In {mydir}: No ebuild written so far! '
 				'I really don\'t know what do to!'.format (

diff --git a/roverlay/overlay/root.py b/roverlay/overlay/root.py
index 3efdf66..5cbcef3 100644
--- a/roverlay/overlay/root.py
+++ b/roverlay/overlay/root.py
@@ -31,7 +31,7 @@ class Overlay ( object ):
 		write_allowed,
 		incremental,
 		skip_manifest,
-		runtime_incremental=True
+		runtime_incremental=False
 	):
 		"""Initializes an overlay.
 
@@ -52,7 +52,7 @@ class Overlay ( object ):
 		                         !!! The created overlay cannot be used with
 		                         emerge/portage
 		* runtime_incremental -- see package.py:PackageDir.__init__ (...),
-		                          Defaults to ?FIXME?
+		                          Defaults to False (saves memory but costs time)
 
 		"""
 		self.name                 = name
@@ -73,7 +73,7 @@ class Overlay ( object ):
 
 		self.skip_manifest        = skip_manifest
 
-		# fixme or ignore: calculating eclass names twice,
+		# calculating eclass names twice,
 		# once here and another time when calling _init_overlay
 		self._header.set_eclasses ( frozenset (
 			self._get_eclass_import_info ( only_eclass_names=True )
@@ -279,7 +279,6 @@ class Overlay ( object ):
 	def scan ( self, **kw ):
 		def scan_categories():
 			for x in os.listdir ( self.physical_location ):
-				# FIXME could use a better check here
 				if '-' in x and self.has_dir ( x ):
 					yield self._get_category ( x )
 		# --- end of scan_categories (...) ---
@@ -322,8 +321,8 @@ class Overlay ( object ):
 
 		raises: IOError
 
-		! TODO/FIXME/DOC: This is not thread-safe, it's expected to be called
-		when ebuild creation is done.
+		Note: This is not thread-safe, it's expected to be called when
+		ebuild creation is done.
 		"""
 		if self._writeable:
 			self._init_overlay ( reimport_eclass=True )
@@ -354,9 +353,15 @@ class Overlay ( object ):
 		returns: None (implicit)
 		"""
 		if self._writeable and not self.skip_manifest:
-			# FIXME: it would be good to ensure that profiles/categories exist
-			for cat in self._categories.values():
-				cat.write_manifest ( **manifest_kw )
+			# profiles/categories is required for successful Manifest
+			# creation
+			if os.path.isfile ( self._profiles_dir + os.sep + 'categories' ):
+				for cat in self._categories.values():
+					cat.write_manifest ( **manifest_kw )
+			else:
+				raise Exception (
+					'profiles/categories is missing - cannot write Manifest files!'
+				)
 		elif not self.skip_manifest:
 			# FIXME debug print
 			print (