[gentoo-commits] proj/R_overlay:master commit in: roverlay/overlay/metadata/, roverlay/overlay/

[André Erdmann]

commit:     548e6ceede8e8ba8d75492656656275f13b20613
Author:     André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Thu Jul 19 16:53:08 2012 +0000
Commit:     André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Thu Jul 19 16:53:28 2012 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=548e6cee

incremental overlay writing

prepare for OverlayCreator

	geändert:   roverlay/overlay/category.py
	geändert:   roverlay/overlay/creator.py
	geändert:   roverlay/overlay/metadata/__init__.py
	geändert:   roverlay/overlay/metadata/nodes.py
	geändert:   roverlay/overlay/package.py
	geändert:   roverlay/overlay/root.py

---
 roverlay/overlay/category.py          |   24 +-
 roverlay/overlay/creator.py           |   11 +-
 roverlay/overlay/metadata/__init__.py |   84 ++++--
 roverlay/overlay/metadata/nodes.py    |   40 +--
 roverlay/overlay/package.py           |  535 +++++++++++++++------------------
 roverlay/overlay/root.py              |   23 +-
 6 files changed, 354 insertions(+), 363 deletions(-)

diff --git a/roverlay/overlay/category.py b/roverlay/overlay/category.py
index afa2456..9cf8ad9 100644
--- a/roverlay/overlay/category.py
+++ b/roverlay/overlay/category.py
@@ -68,6 +68,13 @@ class Category ( object ):
 			not False in ( d.empty() for d in self._subdirs.values() )
 	# --- end of empty (...) ---
 
+	def finalize_write_incremental ( self ):
+		for subdir in self._subdirs.values():
+			if subdir.modified:
+				subdir.write_incremental()
+			subdir.finalize_write_incremental()
+	# --- end of finalize_write_incremental (...) ---
+
 	def _get_package_dir ( self, pkg_name ):
 		if not pkg_name in self._subdirs:
 			self._lock.acquire()
@@ -84,7 +91,7 @@ class Category ( object ):
 		return self._subdirs [pkg_name]
 	# --- end of _get_package_dir (...) ---
 
-	def add ( self, package_info, write_after_add=False, header=None ):
+	def add ( self, package_info, **pkg_add_kw ):
 		"""Adds a package to this category.
 
 		arguments:
@@ -93,13 +100,7 @@ class Category ( object ):
 		returns: success
 		"""
 		subdir = self._get_package_dir ( package_info ['name'] )
-		if subdir.add ( package_info ):
-			if write_after_add:
-				roverlay.util.dodir ( subdir.physical_location )
-				subdir.write_incremental ( default_header=header )
-			return True
-		else:
-			return False
+		return subdir.add ( package_info, **pkg_add_kw )
 	# --- end of add (...) ---
 
 	def generate_metadata ( self, **metadata_kw ):
@@ -164,9 +165,10 @@ class Category ( object ):
 				# so collect the list of package dirs before iterating over it
 				subdirs = tuple ( self._subdirs.values() )
 
-			for package in subdirs:
-				roverlay.util.dodir ( package.physical_location )
-				package.write_incremental ( **write_kw )
+			for subdir in subdirs:
+				if subdir.modified:
+					roverlay.util.dodir ( subdir.physical_location )
+					subdir.write_incremental ( **write_kw )
 		except Exception as e:
 			self.logger.exception ( e )
 	# --- end of write_incremental (...) ---

diff --git a/roverlay/overlay/creator.py b/roverlay/overlay/creator.py
index 3a6e0ed..dd7c11e 100644
--- a/roverlay/overlay/creator.py
+++ b/roverlay/overlay/creator.py
@@ -214,15 +214,17 @@ class OverlayCreator ( object ):
 
 	def add_package_file ( self, package_file ):
 		"""Adds a single R package."""
+		raise Exception ( "to be removed" )
 		self._pkg_queue.put ( PackageInfo ( filepath=package_file ) )
 		self.package_added.inc()
-	# --- end of add_package (...) ---
+	# --- end of add_package_file (...) ---
 
 	def add_package_files ( self, *package_files ):
 		"""Adds multiple R packages."""
+		raise Exception ( "to be removed" )
 		for p in package_files: self.add_package_file ( p )
 		self.package_added.inc()
-	# --- end of add_packages (...) ---
+	# --- end of add_package_files (...) ---
 
 	def write_overlay ( self, incremental=False ):
 		"""Writes the overlay.
@@ -232,7 +234,10 @@ class OverlayCreator ( object ):
 		"""
 		if self.can_write_overlay:
 			start = time.time()
-			self.overlay.write()
+			if incremental:
+				self.overlay.write_incremental()
+			else:
+				self.overlay.write()
 			self._timestamp ( "overlay written", start )
 		else:
 			self.logger.warning ( "Not allowed to write overlay!" )

diff --git a/roverlay/overlay/metadata/__init__.py b/roverlay/overlay/metadata/__init__.py
index e4938b5..58fc414 100644
--- a/roverlay/overlay/metadata/__init__.py
+++ b/roverlay/overlay/metadata/__init__.py
@@ -11,37 +11,53 @@ USE_FULL_DESCRIPTION = True
 class MetadataJob ( object ):
 	"""R package description data -> metadata.xml interface."""
 
-	def __init__ ( self, logger ):
+	def __init__ ( self, filepath, logger ):
 		"""Initializes a MetadataJob.
 
 		arguments:
-		* logger       -- parent logger to use
+		* filepath -- path where the metadata file will be written to
+		* logger   -- parent logger to use
 		"""
-		self.logger    = logger.getChild ( 'metadata' )
-		self._metadata = nodes.MetadataRoot()
-		# reserved for future usage ("dominant ebuilds": when ebuildjobs
-		# share one metadata instance etc.)
-		self.package_info = None
-		self.filename     = 'metadata.xml'
+		self.logger        = logger.getChild ( 'metadata' )
+		self._package_info = None
+		self.filepath      = filepath
+		# no longer storing self._metadata, which will only be created twice
+		# when running show() (expected 1x write per PackageInfo instance)
 	# --- end of __init__ (...) ---
 
-	def update ( self, package_info ):
-		"""Updates the metadata using the given description data.
+	def empty ( self ):
+		return self._package_info is None
+	# --- end of empty (...) ---
 
-		It's expected that this method is called when Ebuild creation is done.
+	def update ( self, package_info ):
+		"""Updates the metadata.
+		Actually, this won't create any metadata, it will only set the
+		PackageInfo object to be used for metadata creation.
 
 		arguments:
-		* desc_data -- description data read from R package
-		* package_info -- reserved for future usage
-
-		returns: None (implicit)
+		* package_info --
 		"""
-		data = package_info ['desc_data']
+		if package_info.has ( 'desc_data' ) and \
+			package_info.compare_version ( self._package_info ) > 0:
+				self._package_info = package_info
+	# --- end of update (...) ---
 
-		mref = self._metadata
+	def update_using_iterable ( self, package_info_iter ):
+		for package_info in package_info_iter:
+			self.update ( package_info )
+	# --- end of update_using_iterable (...) ---
 
-		max_textline_width = roverlay.config.get ( 'METADATA.linewidth', 65 )
+	def _create ( self ):
+		"""Creates metadata (MetadataRoot) using the stored PackageInfo.
+
+		It's expected that this method is called when Ebuild creation is done.
+
+		returns: created metadata
+		"""
+		mref = nodes.MetadataRoot()
+		data = self._package_info ['desc_data']
 
+		max_textline_width = roverlay.config.get ( 'METADATA.linewidth', 65 )
 
 		description = None
 
@@ -67,22 +83,38 @@ class MetadataJob ( object ):
 		#if package_info ['has_suggests']:
 		#	mref.add_useflag ( 'R_suggests', 'install optional dependencies' )
 
+		return mref
 	# --- end of update (...) ---
 
-	def write ( self, _file ):
+	def _write ( self, fh ):
 		"""Writes the metadata into a file.
 
 		arguments:
 		* _file -- file to write, either a file handle or string in which case
 		           a file object will be created
 
-	  returns: True if writing succeeds, else False
+		returns: True if writing succeeds, else False
 
-	  raises: Exception if no metadata to write
-	  """
-		if self._metadata.empty():
-			raise Exception ( "not enough metadata to write!" )
-			#return False
+		raises: Exception if no metadata to write
+		"""
+		if self._create().write_file ( fh ):
+			return True
 		else:
-			return self._metadata.write_file ( _file )
+			raise Exception ( "not enough metadata to write!" )
+	# --- end of _write (...) ---
+
+	show = _write
+
+	def write ( self ):
+		_success = False
+		try:
+			fh = open ( self.filepath, 'w' )
+			self._write ( fh )
+			_success = True
+		except any as e:
+			self.logger.exception ( e )
+		finally:
+			if 'fh' in locals() and fh: fh.close()
+
+		return _success
 	# --- end of write (...) ---

diff --git a/roverlay/overlay/metadata/nodes.py b/roverlay/overlay/metadata/nodes.py
index 6f27deb..02d5482 100644
--- a/roverlay/overlay/metadata/nodes.py
+++ b/roverlay/overlay/metadata/nodes.py
@@ -57,46 +57,24 @@ class MetadataRoot ( MetadataNodeNamedAccess ):
 		return use_node
 	# --- end of add_useflag (...) ---
 
-	def write_file ( self, _file ):
+	def write_file ( self, fh ):
 		"""Writes the metadata to a file.
 
 		arguments:
-		* _file -- either a File object or a string
+		* fh -- a File object
 
 		returns: success True/False
 
 		raises: *passes IOError
 		"""
-		to_write = util.ascii_filter ( self.to_str() )
-
-		own_fh  = False
-		fh      = None
-		success = False
-
-		newline = '\n'
-
-		try:
-			if isinstance ( _file, str ):
-				own_fh = True
-				fh     = open ( _file, 'w' )
-			else:
-				fh     = _file
-
-
+		if not self.empty():
 			fh.write ( MetadataRoot.HEADER )
-			fh.write ( newline )
-			fh.write ( to_write )
-			fh.write ( newline )
-
-			success = True
-
-		except IOError:
-			# log this TODO
-			pass
-		finally:
-			if own_fh and fh: fh.close()
-
-		return success
+			fh.write ( '\n' )
+			fh.write ( util.ascii_filter ( self.to_str() ) )
+			fh.write ( '\n' )
+			return True
+		else:
+			return False
 	# --- end of write_file (...) ---
 
 

diff --git a/roverlay/overlay/package.py b/roverlay/overlay/package.py
index c0b4294..71b7ea9 100644
--- a/roverlay/overlay/package.py
+++ b/roverlay/overlay/package.py
@@ -1,20 +1,18 @@
-# R Overlay -- overlay module, package dir (subdir of category)
-# Copyright 2006-2012 Gentoo Foundation
-# Distributed under the terms of the GNU General Public License v2
+# replacement for package.py
 
-import threading
 import os
 import sys
+import threading
 
 from roverlay                  import manifest
 from roverlay.packageinfo      import PackageInfo
 from roverlay.overlay.metadata import MetadataJob
 
 SUPPRESS_EXCEPTIONS = True
-EBUILD_SUFFIX = '.ebuild'
 
-class PackageDir ( object ):
 
+class PackageDir ( object ):
+	EBUILD_SUFFIX = '.ebuild'
 
 	def __init__ ( self, name, logger, directory ):
 		"""Initializes a PackageDir which contains ebuilds, metadata and
@@ -30,47 +28,131 @@ class PackageDir ( object ):
 		self._lock             = threading.RLock()
 		# { <version> : <PackageInfo> }
 		self._packages         = dict()
-		self._metadata         = None
 		self.physical_location = directory
 
-		self._package_for_manifest = None
+		self._metadata         = MetadataJob (
+			filepath = self.physical_location + os.sep + 'metadata.xml',
+			logger   = self.logger
+		)
+
+		# <dir>/<PN>-<PVR>.ebuild
+		self.ebuild_filepath_format = \
+			self.physical_location + os.sep + \
+			self.name + "-{PVR}" + self.__class__.EBUILD_SUFFIX
 
 		# used to track changes for this package dir
-		self.modified          = False
+		self.modified = False
+		self._manifest_package = None
+		self._need_manifest = False
+		self._need_metadata = False
 	# --- end of __init__ (...) ---
 
-	def list_versions ( self ):
-		return self._packages.keys()
-	# --- end of list_versions (...) ---
+	def new_ebuild ( self ):
+		"""Called when a new ebuild has been created for this PackageDir."""
+		self._need_manifest = True
+		self.modified       = True
+	# --- end of new_ebuild (...) ---
 
-	def has_manifest ( self ):
-		return os.path.isfile (
-			self.physical_location + os.sep + 'Manifest'
-		)
-	# --- end of has_manifest (...) ---
+	def add ( self, package_info, add_if_physical=False ):
+		"""Adds a package to this PackageDir.
 
-	def has_metadata ( self ):
-		return os.path.isfile (
-			self.physical_location + os.sep + 'metadata.xml'
-		)
-	# --- end of has_metadata (...) ---
+		arguments:
+		* package_info    --
+		* add_if_physical -- add package even if it exists as ebuild file
+		                      (-> overwrite old ebuilds)
+
+		returns: success as bool
+
+		raises: Exception when ebuild already exists.
+		"""
+		shortver = package_info ['ebuild_verstr']
+		_success = False
+		try:
+			self._lock.acquire()
+			if shortver in self._packages:
+				# package exists, check if it existed before script invocation
+				if self._packages [shortver] ['physical_only']:
+					if not skip_if_physical:
+						# ignore ebuilds that exist as file
+						self._packages [shortver] = package_info
+						_success = True
+					else:
+						self.logger.debug (
+							"'{PN}-{PVR}.ebuild' exists as file, skipping.".format (
+								PN=self.name, PVR=shortver
+							)
+						)
+				else:
+					# package has been added to this overlay before
+					self.logger.info (
+						"'{PN}-{PVR}.ebuild' already exists, cannot add it!".format (
+						PN=self.name, PVR=shortver
+						)
+					)
+			else:
+				self._packages [shortver] = package_info
+				_success = True
+
+		finally:
+			self._lock.release()
+
+		return _success
+	# --- end of add (...) ---
+
+	def empty ( self ):
+		"""Returns True if no ebuilds stored, else False.
+		Note that "not empty" does not mean "have ebuilds to write".
+		"""
+		return len ( self._packages ) == 0
+	# --- end of empty (...) ---
+
+	def finalize_write_incremental ( self ):
+		with self._lock:
+			if self._need_metadata:
+				self.write_metadata()
+			if self._need_manifest:
+				self.write_manifest()
+	# --- end of finalize_write_incremental (...) ---
+
+	def generate_metadata ( self, skip_if_existent, **ignored_kw ):
+		"""Creates metadata for this package.
+
+		arguments:
+		* skip_if_existent -- do not create if metadata already exist
+		"""
+		with self._lock:
+			if self._metadata.empty() or not skip_if_existent:
+				self._metadata.update_using_iterable ( self._packages.values() )
+	# --- end of generate_metadata (...) ---
 
-	def get_ebuilds ( self ):
-		for x in os.listdir ( self.physical_location ):
-			if x.endswith ( EBUILD_SUFFIX ):
-				yield self.physical_location + os.sep + x
-	# --- end of get_ebuilds (...) ---
+	def list_versions ( self ):
+		return self._packages.keys()
+	# --- end of list_versions (...) ---
 
-	def _scan_ebuilds ( self ):
-		"""Searches for ebuilds in self.physical_location."""
-		elen = len ( EBUILD_SUFFIX )
+	def scan ( self, **kw ):
+		"""Scans the filesystem location of this package for existing
+		ebuilds and adds them.
+		"""
+		def scan_ebuilds():
+			"""Searches for ebuilds in self.physical_location."""
+			elen = len ( self.__class__.EBUILD_SUFFIX )
+			def ebuild_split_pvr ( _file ):
+				if _file.endswith ( self.__class__.EBUILD_SUFFIX ):
+					return _file [ : - elen ].split ( '-', 1 )
+				else:
+					return ( None, None )
+			# --- end of is_ebuild (...) ---
 
-		for f in os.listdir ( self.physical_location ):
-			if f.endswith ( EBUILD_SUFFIX ):
+			# assuming that self.physical_location exists
+			#  (should be verified by category.py:Category)
+			for f in os.listdir ( self.physical_location ):
 				try:
 					# filename without suffix ~= ${PF} := ${PN}-${PVR}
-					pn, pvr = f [ : - elen ].split ( '-', 1 )
-					if pn == self.name:
+					pn, pvr = ebuild_split_pvr ( f )
+					if pn is None:
+						# not an ebuild
+						pass
+					elif pn == self.name:
 						yield pvr
 					else:
 						# $PN does not match directory name, warn about that
@@ -78,298 +160,161 @@ class PackageDir ( object ):
 							"$PN does not match directory name, ignoring {!r}.".\
 							format ( f )
 						)
-
 				except:
 					self.logger.warning (
 						"ebuild {!r} has an invalid file name!".format ( f )
 					)
+		# --- end of scan_ebuilds (...) ---
 
-	# --- end of _scan_ebuilds (...) ---
-
-	def scan ( self, **kw ):
-		"""Scans the filesystem location of this package for existing
-		ebuilds and adds them.
-		"""
-		for pvr in self._scan_ebuilds():
+		for pvr in scan_ebuilds():
 			if pvr not in self._packages:
 				p = PackageInfo ( physical_only=True, pvr=pvr )
 				self._packages [ p ['ebuild_verstr'] ] = p
 	# --- end of scan (...) ---
 
-	def empty ( self ):
-		"""Returns True if no ebuilds stored, else False."""
-		return len ( self._packages ) == 0
-	# --- end of empty (...) ---
-
-	def _get_ebuild_filepath ( self, pvr ):
-		"""Returns the path to the ebuild file.
-
-		arguments:
-		* pvr -- version number with the revision (${PVR} in ebuilds)
-		"""
-		return "{root}{sep}{PN}-{PVR}{EBUILD_SUFFIX}".format (
-			root=self.physical_location, sep=os.sep,
-			PN=self.name, PVR=pvr, EBUILD_SUFFIX=EBUILD_SUFFIX
-		)
-	# --- end of _get_ebuild_filepath (...) ---
-
-	def write_incremental ( self, default_header ):
-		self.write_ebuilds ( header=default_header, overwrite=False )
-	# --- end of write_incremental (...) ---
-
-	def write_metadata ( self, shared_fh=None ):
-		"""Writes metadata for this package."""
-		try:
-
-			self._regen_metadata()
-
-			if shared_fh is None:
-				fh = open (
-					self.physical_location + os.sep + self._metadata.filename, 'w'
-				)
-			else:
-				fh = shared_fh
-
-			self._metadata.write ( fh )
-
-		except IOError as e:
-
-			self.logger.error (
-				"Failed to write metadata file {}.".format ( mfile )
-			)
-			self.logger.exception ( e )
-
-		finally:
-			if shared_fh is None and 'fh' in locals() and fh:
-				fh.close()
-	# --- end of write_metadata (...) ---
-
-	def write_ebuild ( self, efile, ebuild, header, shared_fh=None ):
-		"""Writes an ebuild.
+	def show ( self, stream=sys.stderr, default_header=None ):
+		"""Prints this dir (the ebuilds and the metadata) into a stream.
 
 		arguments:
-		* efile     -- file to write
-		* ebuild    -- ebuild object to write (has to have a __str__ method)
-		* header    -- ebuild header to write (^)
-		* shared_fh -- optional, see write_ebuilds()
-		"""
-		_success = False
-		try:
-			fh = open ( efile, 'w' ) if shared_fh is None else shared_fh
-			if header is not None:
-				fh.write ( str ( header ) )
-				fh.write ( '\n\n' )
-			fh.write ( str ( ebuild ) )
-			fh.write ( '\n' )
-
-			# adjust owner/perm? TODO
-			#if shared_fh is None:
-			#	chmod 0644 or 0444
-			#	chown 250.250
-			_success = True
-		except IOError as e:
-			self.logger.exception ( e )
-		finally:
-			if shared_fh is None and 'fh' in locals() and fh:
-				fh.close()
-
-		return _success
-	# --- end of write_ebuild (...) ---
+		* stream -- stream to use, defaults to sys.stderr
 
-	def write_ebuilds ( self, header, shared_fh=None, overwrite=True ):
-		"""Writes all ebuilds.
+		returns: None (implicit)
 
-		arguments:
-		* header    -- ebuild header
-		* shared_fh -- if set and not None: don't use own file handles (i.e.
-		               write files), write everything into shared_fh
+		raises:
+		* IOError
 		"""
-		for ver, p_info in self._packages.items():
-			if not p_info ['physical_only'] and p_info ['ebuild']:
-				efile = self._get_ebuild_filepath ( ver )
-
-				if not overwrite and efile == p_info ['ebuild_file']:
-					print ( efile + " exists, skipping write()." )
-
-
-				elif self.write_ebuild (
-					efile, p_info ['ebuild'], header, shared_fh
-				):
-					if shared_fh is None:
-						# this marks the package as 'written to fs'
-						p_info.update_now (
-							ebuild_file=efile,
-							remove_auto='ebuild_written'
-						)
-
-						self._package_for_manifest = p_info
-
-						self.logger.info ( "Wrote ebuild {}.".format ( efile ) )
-				else:
-					self.logger.error (
-						"Couldn't write ebuild {}.".format ( efile )
-					)
-	# --- end of write_ebuilds (...) ---
+		return self.write ( default_header=default_header, shared_fh=stream )
+	# --- end of show (...) ---
 
 	def write ( self,
-		default_header=None, write_manifest=True, shared_fh=None
+		default_header=None, shared_fh=None,
+		write_ebuilds=True, write_manifest=True, write_metadata=True,
+		overwrite_ebuilds=True
 	):
 		"""Writes this directory to its (existent!) filesystem location.
 
 		arguments:
 		* default_header    -- ebuild header to write
-		* write_manifest -- if set and False: don't write the Manifest file
+		* shared_fh         -- if set and not None: write everyting into <fh>
+		* write_ebuilds     -- if set and False: don't write ebuilds
+		* write_manifest    -- if set and False: don't write the Manifest file
+		* write_metadata    -- if set and False: don't write the metadata file
+		* overwrite_ebuilds -- if set and False: don't overwrite ebuilds
 
 		returns: None (implicit)
 
 		raises:
-		* IOError
+		* IOError (?)
 		"""
-		self._lock.acquire()
-		try:
+		with self._lock:
 			# mkdir not required here, overlay.Category does this
 
 			# write ebuilds
-			self.write_ebuilds ( header=default_header, shared_fh=shared_fh )
+			if write_ebuilds:
+				self.write_ebuilds (
+					default_header=default_header, shared_fh=shared_fh,
+					overwrite=overwrite_ebuilds
+				)
 
 			# write metadata
-			self.write_metadata ( shared_fh=shared_fh )
+			if write_metadata:
+				self.write_metadata ( shared_fh=shared_fh )
 
-			if write_manifest and shared_fh is not None:
+			# write manifest (only if shared_fh is None)
+			if write_manifest and shared_fh is None:
 				self.write_manifest()
-
-		finally:
-			self._lock.release()
 	# --- end of write (...) ---
 
-	def show ( self, stream=sys.stderr, default_header=None ):
-		"""Prints this dir (the ebuilds and the metadata) into a stream.
-
-		arguments:
-		* stream -- stream to use, defaults to sys.stderr
-
-		returns: None (implicit)
-
-		raises:
-		* IOError
-		"""
-		self.write (
-			default_header=default_header, shared_fh=stream, write_manifest=False
-		)
-	# --- end of show (...) ---
-
-	def _latest_package ( self, pkg_filter=None, use_lock=False ):
-		"""Returns the package info with the highest version number.
-
-		arguments:
-		* pkg_filter -- either None or a callable,
-		                 None: do not filter packages
-		                 else: ignore package if it does not pass the filter
-		* use_lock   -- if True: hold lock while searching
-		"""
-		first  = True
-		retver = None
-		retpkg = None
-
-		if use_lock: self._lock.acquire()
-		try:
-			for p in self._packages.values():
-				if pkg_filter is None or pkg_filter ( p ):
-					newver = p ['version']
-					if first or newver > retver:
-						retver = newver
-						retpkg = p
-						first  = False
-		finally:
-			if use_lock: self._lock.release()
-		return retpkg
-	# --- end of _latest_package (...) ---
-
-	def add ( self, package_info ):
-		"""Adds a package to this PackageDir.
+	def write_ebuilds ( self, default_header, overwrite, shared_fh=None ):
+		"""Writes all ebuilds.
 
 		arguments:
-		* package_info --
-
-		returns: success as bool
-
-		raises: Exception when ebuild already exists.
+		* default_header -- ebuild header
+		* shared_fh      -- if set and not None: don't use own file handles
+		                     (i.e. write files), write everything into shared_fh
+		* overwrite      -- write ebuilds that have been written before,
+		                     defaults to True
 		"""
-		shortver = package_info ['ebuild_verstr']
-
-		def already_exists ():
-			if shortver in self._packages:
-				self.logger.info (
-					"'{PN}-{PVR}.ebuild' already exists, cannot add it!".format (
-						PN=self.name, PVR=shortver
-					)
-				)
-				return True
-			else:
-				return False
-		# --- end of already_exists (...) ---
-
-		_success = False
-
-		if not already_exists():
+		def write_ebuild ( efile, ebuild ):
+			"""Writes an ebuild.
+
+			arguments:
+			* efile  -- file to write
+			* ebuild -- ebuild object to write (has to have a __str__ method)
+			* (default_header from write_ebuilds())
+			* (shared_fh from write_ebuilds())
+			"""
+			_success = False
 			try:
-				self._lock.acquire()
-				if not already_exists():
-					self._packages [shortver] = package_info
-					self.modified = True
-					_success = True
+				fh = open ( efile, 'w' ) if shared_fh is None else shared_fh
+				if default_header is not None:
+					fh.write ( str ( default_header ) )
+					fh.write ( '\n\n' )
+				fh.write ( str ( ebuild ) )
+				fh.write ( '\n' )
+
+				# adjust owner/perm? TODO
+				#if shared_fh is None:
+				#	chmod 0644 or 0444
+				#	chown 250.250
+				_success = True
+			except IOError as e:
+				self.logger.exception ( e )
 			finally:
-				self._lock.release()
+				if shared_fh is None and 'fh' in locals() and fh:
+					fh.close()
 
-		return _success
-	# --- end of add (...) ---
+			return _success
+		# --- end of write_ebuild (...) ---
 
-	def _regen_metadata ( self ):
-		"""Regenerates the metadata."""
-		self.generate_metadata (
-			skip_if_existent=True,
-			use_all_packages=False,
-			use_old_metadata=False
-		)
-	# --- end of _regen_metadata (...) ---
+		def ebuilds_to_write():
+			"""Yields all ebuilds that are ready to be written."""
 
-	def generate_metadata (
-		self,
-		skip_if_existent=False, use_all_packages=False, use_old_metadata=False
-	):
-		"""Creates metadata for this package.
+			for ver, p_info in self._packages.items():
+				if p_info.has ( 'ebuild' ) and not p_info ['physical_only']:
+					efile = self.ebuild_filepath_format.format ( PVR=ver )
 
-		arguments:
-		* skip_if_existent -- do not create if metadata already exist
-		* use_all_packages -- TODO in metadata
-		* use_old_metadata -- TODO in metadata
-		"""
-		if use_old_metadata or use_all_packages:
-			raise Exception ( "using >1 package for metadata.xml is TODO!" )
+					if efile != p_info ['ebuild_file'] or overwrite:
+						yield ( efile, p_info )
+					# else efile exists
+		# --- end of ebuilds_to_write (...) ---
 
-		if skip_if_existent and not self._metadata is None: return
+		all_ebuilds_written = True
 
-		self._lock.acquire()
-		try:
+		for efile, p_info in ebuilds_to_write():
+			if write_ebuild ( efile, p_info ['ebuild'] ):
+				self._need_manifest = True
+
+				# update metadata for each successfully written ebuild
+				#  (self._metadata knows how to handle this request)
+				self._metadata.update ( p_info )
 
-			if self._metadata is None or not use_old_metadata:
-				del self._metadata
-				self._metadata = MetadataJob ( self.logger )
+				if shared_fh is None:
+					# this marks the package as 'written to fs'
+					p_info.update_now (
+						ebuild_file=efile,
+						remove_auto='ebuild_written'
+					)
 
-			if use_all_packages:
-				for p_info in self._packages:
-					self._metadata.update ( p_info )
+					self.logger.info ( "Wrote ebuild {}.".format ( efile ) )
 			else:
-				self._metadata.update ( self._latest_package() )
+				all_ebuilds_written = False
+				self.logger.error (
+					"Couldn't write ebuild {}.".format ( efile )
+				)
 
-		finally:
-			self._lock.release()
-	# --- end of generate_metadata (...) ---
+		self.modified = not all_ebuilds_written
+	# --- end of write_ebuilds (...) ---
+
+	def write_incremental ( self, default_header ):
+		with self._lock:
+			self.write_ebuilds ( default_header=default_header, overwrite=False )
+	# --- end of write_incremental (...) ---
 
 	def write_manifest ( self ):
 		"""Generates and writes the Manifest file for this package.
 
-		expects: called in self.write(), after writing metadata/ebuilds
+		expects: called after writing metadata/ebuilds
 
 		returns: None (implicit)
 
@@ -378,27 +323,47 @@ class PackageDir ( object ):
 		"""
 
 		# it should be sufficient to call create_manifest for one ebuild,
-		#  choosing the latest one here that exists in self.physical_location.
+		#  choosing the latest one that exists in self.physical_location and
+		#  has enough data (DISTDIR, EBUILD_FILE) for this task.
 		#
 		# metadata.xml's full path cannot be used for manifest creation here
 		#  'cause DISTDIR would be unknown
 		#
-#		pkg_info_for_manifest = self._latest_package (
-#			pkg_filter=lambda pkg : pkg ['ebuild_file'] is not None,
-#			use_lock=True
-#		)
 
-		if self._package_for_manifest is None:
-			# ? FIXME
+		if self._manifest_package is not None:
+			manifest.create_manifest ( self._manifest_package, nofail=False )
+			self._need_manifest = False
+		else:
 			raise Exception (
 				"No ebuild written so far! I really don't know what do to!"
 			)
-		else:
-			# TODO: manifest creation interface is single threaded,
-			#        may want to 'fix' this later
-			manifest.create_manifest (
-				self._package_for_manifest, nofail=False,
-				#ebuild_file=...
+	# --- end of write_manifest (...) ---
+
+	def write_metadata ( self, shared_fh=None ):
+		"""Writes metadata for this package."""
+		if self._manifest_package is None and not shared_fh:
+			self.logger.error (
+				'write_metadata() requested, but no ebuild written so far! '
+				'This will most likely result in a corrupt Manifest.'
 			)
 
-	# --- end of write_manifest (...) ---
+		try:
+			self.generate_metadata ( skip_if_existent=True )
+
+			if self._metadata.write() \
+				if shared_fh is None else self._metadata.show ( shared_fh ) \
+			:
+				self._need_metadata = False
+				self._need_manifest = True
+				return True
+			else:
+				self.logger.error (
+					"Failed to write metadata file {}.".format (
+						self._metadata.filepath
+					)
+				)
+				return False
+		except:
+			# already logged
+			return False
+	# --- end of write_metadata (...) ---

diff --git a/roverlay/overlay/root.py b/roverlay/overlay/root.py
index 26be02a..e0446df 100644
--- a/roverlay/overlay/root.py
+++ b/roverlay/overlay/root.py
@@ -74,6 +74,8 @@ class Overlay ( object ):
 		self.default_category  = default_category
 		self.eclass_files      = eclass_files
 
+		self.ignore_existing_ebuilds = False
+
 		self._profiles_dir     = self.physical_location + os.sep + 'profiles'
 		self._catlock          = threading.Lock()
 		self._categories       = dict()
@@ -143,7 +145,7 @@ class Overlay ( object ):
 		return self._categories [category]
 	# --- end of _get_category (...) ---
 
-	def add ( self, package_info, write_after_add=False, category=None ):
+	def add ( self, package_info, category=None ):
 		"""Adds a package to this overlay.
 
 		arguments:
@@ -158,12 +160,9 @@ class Overlay ( object ):
 		)
 
 		if write_after_add:
-			util.dodir ( cat.physical_location, mkdir_p=True )
-			return cat.add (
-				package_info, write_after_add=True, header = self._get_header()
-			)
-		else:
-			return cat.add ( package_info, write_after_add=False )
+			raise Exception ( "add~write_after_add: to be removed." )
+
+		return cat.add ( package_info, write_after_add=False )
 	# --- end of add (...) ---
 
 	def show ( self, **show_kw ):
@@ -228,6 +227,16 @@ class Overlay ( object ):
 			self._incremental_write_lock.release()
 	# --- end of write_incremental (...) ---
 
+	def finalize_write_incremental ( self ):
+		"""Writes metadata + Manifest for all packages."""
+		self._init_overlay ( reimport_eclass=True, make_profiles_dir=True )
+
+		for cat in self._categories.values():
+			cat.finalize_write_incremental()
+
+		self._write_categories ( only_active=True )
+	# --- end of finalize_incremental (...) ---
+
 	def generate_metadata ( self, **metadata_kw ):
 		"""Tells the overlay's categories to create metadata.
 		You don't have to call this before write()/show() unless you want to use