[gentoo-commits] proj/R_overlay:master commit in: roverlay/overlay/manifest/, roverlay/overlay/

[André Erdmann]

commit:     ee870b2a6f8a3df4a794dc82fda1bd63a4f6d147
Author:     André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Tue Jul 24 15:39:07 2012 +0000
Commit:     André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Tue Jul 24 15:39:07 2012 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=ee870b2a

incremental overlay writing: merge write() funcs

* one write function for overlay/{root,category,package}.py
  that replaces write_incremental, write, finalize_write_incremental

* also fixed some issues (proper PackageDir cleanup, make threaded writing
  controllable)

	geändert:   roverlay/overlay/category.py
	geändert:   roverlay/overlay/creator.py
	geändert:   roverlay/overlay/manifest/__init__.py
	geändert:   roverlay/overlay/package.py
	geändert:   roverlay/overlay/root.py

---
 roverlay/overlay/category.py          |  141 +++++++----------
 roverlay/overlay/creator.py           |   29 +---
 roverlay/overlay/manifest/__init__.py |    4 +-
 roverlay/overlay/package.py           |  278 +++++++++++++++++++++------------
 roverlay/overlay/root.py              |   96 +++---------
 5 files changed, 266 insertions(+), 282 deletions(-)

diff --git a/roverlay/overlay/category.py b/roverlay/overlay/category.py
index 33149c4..fab6373 100644
--- a/roverlay/overlay/category.py
+++ b/roverlay/overlay/category.py
@@ -12,8 +12,6 @@ except ImportError:
 
 from roverlay.overlay.package import PackageDir
 
-import roverlay.util
-
 class Category ( object ):
 
 	WRITE_JOBCOUNT = 3
@@ -49,8 +47,6 @@ class Category ( object ):
 						incremental = self.incremental
 					)
 					self._subdirs [pkg_name] = newpkg
-					if self.incremental:
-						roverlay.util.dodir ( newpkg.physical_location )
 			finally:
 				self._lock.release()
 
@@ -76,26 +72,6 @@ class Category ( object ):
 			not False in ( d.empty() for d in self._subdirs.values() )
 	# --- end of empty (...) ---
 
-	def finalize_write_incremental ( self ):
-		for subdir in self._subdirs.values():
-			if subdir.modified:
-				subdir.write_incremental()
-			subdir.finalize_write_incremental()
-	# --- end of finalize_write_incremental (...) ---
-
-	def generate_metadata ( self, **metadata_kw ):
-		"""Generates metadata for all packages in this category.
-		Metadata are automatically generated when calling write().
-
-		arguments:
-		* **metadata_kw -- see PackageDir.generate_metadata(...)
-
-		returns: None (implicit)
-		"""
-		for package in self._subdirs.values():
-			package.generate_metadata ( **metadata_kw )
-	# --- end of generate_metadata (...) ---
-
 	def has ( self, subdir ):
 		return subdir in self._subdirs
 	# --- end of has (...) ---
@@ -104,13 +80,6 @@ class Category ( object ):
 		return os.path.isdir ( self.physical_location + os.sep + _dir )
 	# --- end of has_category (...) ---
 
-	def keep_nth_latest ( self, *args, **kwargs ):
-		"""See package.py:PackageDir:keep_nth_latest."""
-		for subdir in self._subdirs.values():
-			subdir.keep_nth_latest ( *args, **kwargs )
-			subdir.fs_cleanup()
-	# --- end of keep_nth_latest (...) ---
-
 	def list_packages ( self, for_deprules=False ):
 		"""Lists all packages in this category.
 		Yields <category>/<package name> or a dict (see for_deprules below).
@@ -130,6 +99,14 @@ class Category ( object ):
 					yield self.name + os.sep + name
 	# --- end of list_packages (...) ---
 
+	def remove_empty ( self ):
+		"""This removes all empty PackageDirs."""
+		with self._lock:
+			for key in tuple ( self._subdirs.keys() ):
+				if self._subdirs [key].check_empty():
+					del self._subdirs [key]
+	# --- end of remove_empty (...) ---
+
 	def scan ( self, **kw ):
 		"""Scans this category for existing ebuilds."""
 		for subdir in os.listdir ( self.physical_location ):
@@ -146,7 +123,7 @@ class Category ( object ):
 			package.show ( **show_kw )
 	# --- end of show (...) ---
 
-	def write ( self, **write_kw ):
+	def write ( self, overwrite_ebuilds, keep_n_ebuilds, cautious ):
 		"""Writes this category to its filesystem location.
 
 		returns: None (implicit)
@@ -156,75 +133,75 @@ class Category ( object ):
 
 			arguments:
 			* q        -- queue
-			* write_kw --
+			* write_kw -- keywords for write(...)
 			"""
 			try:
 				while not q.empty():
-					pkg = q.get_nowait()
-					pkg.write ( write_manifest=False, **write_kw )
-
+					try:
+						pkg = q.get_nowait()
+						# remove manifest writing from threaded writing since it's
+						# single-threaded
+						pkg.write ( write_manifest=False, **write_kw )
+					#except ( Exception, KeyboardInterrupt ) as e:
+					except Exception as e:
+						# FIXME: reintroduce RERAISE
+						self.logger.exception ( e )
 			except queue.Empty:
 				pass
-			except ( Exception, KeyboardInterrupt ) as e:
-				self.RERAISE_EXCEPTION = e
 		# --- end of run_write_queue (...) ---
 
+		if len ( self._subdirs ) == 0: return
+
+		# determine write keyword args
+		write_kwargs = dict (
+			overwrite_ebuilds = overwrite_ebuilds,
+			keep_n_ebuilds    = keep_n_ebuilds,
+			cautious          = cautious,
+		)
+
+		# start writing:
+
 		max_jobs = self.__class__.WRITE_JOBCOUNT
 
-		# todo len.. > 3: what's an reasonable number of min package dirs to
-		#                  start threaded writing?
-		if max_jobs > 1 and len ( self._subdirs ) > 3:
+		# FIXME/TODO: what's an reasonable number of min package dirs to
+		# start threaded writing?
+		# Ignoring it for now (and expecting enough pkg dirs)
+		if max_jobs > 1:
 
-			# writing 1..self.__class__.WRITE_JOBCOUNT package dirs at once
+			# writing <=max_jobs package dirs at once
 
-			modified_packages = tuple (
-				p for p in self._subdirs.values() if p.modified
-			)
-			if len ( modified_packages ) > 0:
-				write_queue = queue.Queue()
-				for package in modified_packages:
-					roverlay.util.dodir ( package.physical_location )
-					write_queue.put_nowait ( package )
+			# don't create more workers than write jobs available
+			max_jobs = min ( max_jobs, len ( self._subdirs ) )
 
-				workers = (
-					threading.Thread (
-						target=run_write_queue,
-						args=( write_queue, write_kw )
-					) for n in range ( max_jobs )
-				)
+			write_queue = queue.Queue()
+			for package in self._subdirs.values():
+				write_queue.put_nowait ( package )
 
-				for w in workers: w.start()
-				for w in workers: w.join()
+			workers = frozenset (
+				threading.Thread (
+					target=run_write_queue,
+					args=( write_queue, write_kwargs )
+				) for n in range ( max_jobs )
+			)
+
+			for w in workers: w.start()
+			for w in workers: w.join()
 
-				if hasattr ( self, 'RERAISE_EXCEPTION' ):
-					raise self.RERAISE_EXCEPTION
+			self.remove_empty()
 
-				# write manifest files
-				for package in modified_packages:
-					package.write_manifest()
+			# write manifest files
+			# fixme: debug print
+			#self.logger.info ( "Writing Manifest files for {}".format ( name ) )
+			print ( "Writing Manifest files ..." )
+			for package in self._subdirs.values():
+				package.write_manifest ( ignore_empty=True )
 
 		else:
 			for package in self._subdirs.values():
-				if package.modified:
-					roverlay.util.dodir ( package.physical_location )
-					package.write ( write_manifest=True, **write_kw )
-	# --- end of write (...) ---
+				package.write ( **write_kwargs )
 
-	def write_incremental ( self, **write_kw ):
-		"""Writes this category incrementally."""
-		try:
-			with self._lock:
-				# new package dirs could be added during overlay writing,
-				# so collect the list of package dirs before iterating over it
-				subdirs = tuple ( self._subdirs.values() )
-
-			for subdir in subdirs:
-				if subdir.modified:
-					roverlay.util.dodir ( subdir.physical_location )
-					subdir.write_incremental ( **write_kw )
-		except Exception as e:
-			self.logger.exception ( e )
-	# --- end of write_incremental (...) ---
+			self.remove_empty()
+	# --- end of write (...) ---
 
 	def write_manifest ( self, **manifest_kw ):
 		"""Generates Manifest files for all packages in this category.

diff --git a/roverlay/overlay/creator.py b/roverlay/overlay/creator.py
index 9de063c..d12e0a5 100644
--- a/roverlay/overlay/creator.py
+++ b/roverlay/overlay/creator.py
@@ -214,32 +214,15 @@ class OverlayCreator ( object ):
 			self.package_added.inc()
 	# --- end of add_package (...) ---
 
-	def add_package_file ( self, package_file ):
-		"""Adds a single R package."""
-		raise Exception ( "to be removed" )
-		self._pkg_queue.put ( PackageInfo ( filepath=package_file ) )
-		self.package_added.inc()
-	# --- end of add_package_file (...) ---
-
-	def add_package_files ( self, *package_files ):
-		"""Adds multiple R packages."""
-		raise Exception ( "to be removed" )
-		for p in package_files: self.add_package_file ( p )
-		self.package_added.inc()
-	# --- end of add_package_files (...) ---
-
 	def write_overlay ( self ):
 		"""Writes the overlay.
 
 		arguments:
 		"""
 		if self.can_write_overlay:
-			if self.write_incremental:
-				self.overlay.finalize_write_incremental()
-			else:
-				start = time.time()
-				self.overlay.write()
-				self._timestamp ( "overlay written", start )
+			start = time.time()
+			self.overlay.write()
+			self._timestamp ( "overlay written", start )
 		else:
 			self.logger.warning ( "Not allowed to write overlay!" )
 	# --- end of write_overlay (...) ---
@@ -304,7 +287,6 @@ class OverlayCreator ( object ):
 
 		self._close_workers()
 		close_resolver()
-		self.overlay.keep_nth_latest ( n=1 )
 		self.closed = True
 	# --- end of close (...) ---
 
@@ -429,8 +411,9 @@ class OverlayCreator ( object ):
 		if self.NUMTHREADS > 0:
 			start = time.time()
 			self.logger.warning (
-				"Running in concurrent mode with %i threads." % self.NUMTHREADS
-			)
+				"Running in concurrent mode with {num} threads.".format (
+					num=self.NUMTHREADS
+			) )
 			self._workers = frozenset (
 				self._get_worker ( start_now=True ) \
 					for n in range ( self.NUMTHREADS )

diff --git a/roverlay/overlay/manifest/__init__.py b/roverlay/overlay/manifest/__init__.py
index 9a49c7f..d9f9c0a 100644
--- a/roverlay/overlay/manifest/__init__.py
+++ b/roverlay/overlay/manifest/__init__.py
@@ -11,8 +11,6 @@ _manifest_creation = helpers.ExternalManifestCreation()
 # for one directory/overlay
 _manifest_lock = threading.Lock()
 
-
-
 def create_manifest ( package_info_list, nofail=False ):
 	"""Creates a Manifest for package_info, using the <<best>> implementation
 	available.
@@ -35,4 +33,6 @@ def create_manifest ( package_info_list, nofail=False ):
 			raise
 	finally:
 		_manifest_lock.release()
+
+	return ret
 # --- end of create_manifest (...) ---

diff --git a/roverlay/overlay/package.py b/roverlay/overlay/package.py
index a2e4d9b..d2662bd 100644
--- a/roverlay/overlay/package.py
+++ b/roverlay/overlay/package.py
@@ -120,6 +120,21 @@ class PackageDir ( object ):
 			return False
 	# --- end of add (...) ---
 
+	def check_empty ( self ):
+		"""Similar to empty(),
+		but also removes the directory of this PackageDir.
+		"""
+		if len ( self._packages ) == 0:
+			if os.path.isdir ( self.physical_location ):
+				try:
+					os.rmdir ( self.physical_location )
+				except Exception as e:
+					self.logger.exception ( e )
+			return True
+		else:
+			return False
+	# --- end of check_empty (...) ---
+
 	def empty ( self ):
 		"""Returns True if no ebuilds stored, else False.
 		Note that "not empty" doesn't mean "has ebuilds to write" or "has
@@ -129,25 +144,6 @@ class PackageDir ( object ):
 		return len ( self._packages ) == 0
 	# --- end of empty (...) ---
 
-	def finalize_write_incremental ( self ):
-		"""Method that finalizes incremental writing, i.e. write outstanding
-		ebuilds and write metadata.xml, Manifest.
-		"""
-		with self._lock:
-			if self.has_ebuilds():
-				if self.modified:
-					self.write_ebuilds ( overwrite=False )
-				if self._need_metadata:
-					self.write_metadata()
-				if self._need_manifest:
-					self.write_manifest()
-			else:
-				self.logger.critical (
-					"<<todo>>: please clean up this dir: {}.".format (
-						self.physical_location
-				) )
-	# --- end of finalize_write_incremental (...) ---
-
 	def fs_cleanup ( self ):
 		"""Cleans up the filesystem location of this package dir.
 		To be called after keep_nth_latest, calls finalize_write_incremental().
@@ -158,11 +154,9 @@ class PackageDir ( object ):
 		# --- end of rmtree_error (...) ---
 
 		with self._lock:
-			if self.has_ebuilds():
-				# !!! FIXME this doesn't work if no ebuilds written, but
-				# old ones removed -> DISTDIR unknown during Manifest creation
-				self.finalize_write_incremental()
-			elif os.path.isdir ( self.physical_location ):
+			if os.path.isdir ( self.physical_location ) \
+				and not self.has_ebuilds() \
+			:
 				# destroy self.physical_location
 				shutil.rmtree ( self.physical_location, onerror=rmtree_error )
 	# --- end of fs_cleanup (...) ---
@@ -193,53 +187,58 @@ class PackageDir ( object ):
 		arguments:
 		* n        -- # of packages/ebuilds to keep
 		* cautious -- if True: be extra careful, verify that ebuilds exist
+		                       as file; note that this will ignore all
+		                       ebuilds that haven't been written to the file-
+		                       system yet (which implies an extra overhead,
+		                       you'll have to write all ebuilds first)
 		"""
-
-		# create the list of packages to iterate over,
-		# * package has to have an ebuild_file
-		# * sort them by version in reverse order (latest package gets index 0)
+		def is_ebuild_cautious ( p_tuple ):
+			# package has to have an ebuild_file that exists
+			efile = p_tuple [1] ['ebuild_file' ]
+			if efile is not None:
+				return os.path.isfile ( efile )
+			else:
+				return False
+		# --- end of is_ebuild_cautious (...) ---
+
+		def is_ebuild ( p_tuple ):
+			# package has to have an ebuild_file or an ebuild entry
+			return (
+				p_tuple [1] ['ebuild_file'] or p_tuple [1] ['ebuild']
+			) is not None
+		# --- end of is_ebuild (...) ---
+
+		# create the list of packages to iterate over (cautious/non-cautious),
+		# sort them by version in reverse order
 		packages = reversed ( sorted (
 			filter (
-				lambda p : p [1] ['ebuild_file'] is not None,
-				self._packages.items()
+				function=is_ebuild if not cautious else is_ebuild_cautious,
+				iterable=self._packages.items()
 			),
 			key=lambda p : p [1] ['version']
 		) )
 
+		if n < 1:
+			raise Exception ( "Must keep more than zero ebuilds." )
+
 		kept   = 0
 		ecount = 0
-		if not cautious:
-			# could use a slice here, too
-			for pvr, pkg in packages:
-				ecount += 1
-				if kept < n:
-					printself.logger.debug ( "Keeping {pvr}.".format ( pvr=pvr ) )
-					kept += 1
-				else:
-					self.logger.debug ( "Removing {pvr}.".format ( pvr=pvr ) )
-					self.purge_package ( pvr )
-		else:
-			for pvr, pkg in packages:
-				ecount += 1
-				if os.path.isfile ( pkg ['ebuild_file'] ):
-					if kept < n:
-						self.logger.debug ( "Keeping {pvr}.".format ( pvr=pvr ) )
-						kept += 1
-					else:
-						self.logger.debug ( "Removing {pvr}.".format ( pvr=pvr ) )
-						self.purge_package ( pvr )
-				else:
-					self.logger.error (
-						"{efile} is assumed to exist as file but doesn't!".format (
-							efile=pkg ['ebuild_file']
-					) )
 
-		# FIXME/IGNORE: this doesn't count inexistent files as kept
+		for pvr, pkg in packages:
+			ecount += 1
+			if kept < n:
+				self.logger.debug ( "Keeping {pvr}.".format ( pvr=pvr ) )
+				kept += 1
+			else:
+				self.logger.debug ( "Removing {pvr}.".format ( pvr=pvr ) )
+				self.purge_package ( pvr )
+
 		self.logger.debug (
 			"Kept {kept}/{total} ebuilds.".format ( kept=kept, total=ecount )
 		)
 
 		# FIXME: Manifest is now invalid and dir could be "empty" (no ebuilds)
+		# FIXME: force metadata regeneration
 	# --- end of keep_nth_latest (...) ---
 
 	def list_versions ( self ):
@@ -252,7 +251,8 @@ class PackageDir ( object ):
 		self._need_metadata = True
 		self.modified       = True
 		if self.runtime_incremental:
-			return self.write_incremental()
+			with self._lock:
+				return self.write_ebuilds ( overwrite=False )
 		else:
 			return True
 	# --- end of new_ebuild (...) ---
@@ -325,48 +325,104 @@ class PackageDir ( object ):
 		arguments:
 		* stream -- stream to use, defaults to sys.stderr
 
-		returns: None (implicit)
+		returns: True
 
 		raises:
-		* IOError
+		* passes all exceptions (IOError, ..)
 		"""
-		return self.write ( shared_fh=stream )
+		self.write_ebuilds ( overwrite=True, shared_fh=stream )
+		self.write_metadata ( shared_fh=stream )
+		return True
 	# --- end of show (...) ---
 
+	def virtual_cleanup ( self ):
+		"""Removes all PackageInfos from this structure that don't have an
+		'ebuild_file' entry.
+		"""
+		with self._lock:
+			# keyset may change during this method
+			for pvr in tuple ( self._packages.keys() ):
+				if self._packages [pvr] ['ebuild_file'] is None:
+					del self._packages [pvr]
+		# -- lock
+	# --- end of virtual_cleanup (...) ---
+
 	def write ( self,
-		shared_fh=None, overwrite_ebuilds=True,
-		write_ebuilds=True, write_manifest=True, write_metadata=True
+		overwrite_ebuilds=False,
+		write_ebuilds=True, write_manifest=True, write_metadata=True,
+		cleanup=True, keep_n_ebuilds=None, cautious=True
 	):
 		"""Writes this directory to its (existent!) filesystem location.
 
 		arguments:
-		* shared_fh         -- if set and not None: write everyting into <fh>
 		* write_ebuilds     -- if set and False: don't write ebuilds
 		* write_manifest    -- if set and False: don't write the Manifest file
 		* write_metadata    -- if set and False: don't write the metadata file
-		* overwrite_ebuilds -- if set and False: don't overwrite ebuilds
-
-		returns: None (implicit)
+		* overwrite_ebuilds -- whether to overwrite ebuilds,
+		                        None means autodetect, enable overwriting
+		                        if not modified since last write
+		                        Defaults to False
+		* cleanup           -- clean up after writing
+		                        Defaults to True
+		* keep_n_ebuilds    -- # of ebuilds to keep (remove all others),
+		                        Defaults to None (disable) and implies cleanup
+		* cautious          -- be cautious when keeping the nth latest ebuilds,
+		                       this has some overhead
+		                       Defaults to True
+
+		returns: success (True/False)
 
 		raises:
-		* IOError (?)
+		* passes IOError
 		"""
-		with self._lock:
-			# mkdir not required here, overlay.Category does this
+		# NOTE, replaces:
+		# * old write: overwrite_ebuilds=True
+		# * finalize_write_incremental : no extra args
+		# * write_incremental : write_manifest=False, write_metadata=False,
+		#                        cleanup=False (or use write_ebuilds)
+		# BREAKS: show(), which has its own method/function now
 
-			# write ebuilds
-			if write_ebuilds:
-				self.write_ebuilds (
-					overwrite=overwrite_ebuilds, shared_fh=shared_fh
-				)
+		cleanup = cleanup or ( keep_n_ebuilds is not None )
 
-			# write metadata
-			if write_metadata:
-				self.write_metadata ( shared_fh=shared_fh )
+		success = True
+		with self._lock:
+			if self.has_ebuilds():
+				# not cautious: remove ebuilds before writing them
+				if not cautious and keep_n_ebuilds is not None:
+					self.keep_nth_latest ( n=keep_n_ebuilds, cautious=False )
+
+				# write ebuilds
+				if self.modified and write_ebuilds:
+					success = self.write_ebuilds (
+						# None ~ not modified
+						overwrite = overwrite_ebuilds \
+							if overwrite_ebuilds is not None \
+							else not self.modified
+					)
 
-			# write manifest (only if shared_fh is None)
-			if write_manifest and shared_fh is None:
-				self.write_manifest()
+				# cautious: remove ebuilds after writing them
+				if cautious and keep_n_ebuilds is not None:
+					self.keep_nth_latest ( n=keep_n_ebuilds, cautious=True )
+
+				# write metadata
+				if self._need_metadata and write_metadata:
+					# don't mess around with short-circuit bool evaluation
+					if not self.write_metadata():
+						success = False
+
+				# write manifest (only if shared_fh is None)
+				if self._need_manifest and write_manifest:
+					if not self.write_manifest():
+						success = False
+			# -- has_ebuilds?
+
+			if cleanup:
+				self.virtual_cleanup()
+				self.fs_cleanup()
+
+			# FIXME / TODO call fs_cleanup
+		# -- lock
+		return success
 	# --- end of write (...) ---
 
 	def write_ebuilds ( self, overwrite, shared_fh=None ):
@@ -390,7 +446,6 @@ class PackageDir ( object ):
 			"""
 			_success = False
 			try:
-				util.dodir ( self.physical_location )
 				fh = open ( efile, 'w' ) if shared_fh is None else shared_fh
 				if ebuild_header is not None:
 					fh.write ( str ( ebuild_header ) )
@@ -426,7 +481,14 @@ class PackageDir ( object ):
 
 		all_ebuilds_written = True
 
+		# don't call dodir if shared_fh is set
+		hasdir = bool ( shared_fh is not None )
+
 		for efile, p_info in ebuilds_to_write():
+			if not hasdir:
+				util.dodir ( self.physical_location, mkdir_p=True )
+				hasdir = True
+
 			if write_ebuild ( efile, p_info ['ebuild'] ):
 				self._need_manifest = True
 
@@ -452,17 +514,12 @@ class PackageDir ( object ):
 		return all_ebuilds_written
 	# --- end of write_ebuilds (...) ---
 
-	def write_incremental ( self ):
-		with self._lock:
-			return self.write_ebuilds ( overwrite=False )
-	# --- end of write_incremental (...) ---
-
-	def write_manifest ( self ):
+	def write_manifest ( self, ignore_empty=False ):
 		"""Generates and writes the Manifest file for this package.
 
 		expects: called after writing metadata/ebuilds
 
-		returns: None (implicit)
+		returns: success (True/False)
 
 		raises:
 		* Exception if no ebuild exists
@@ -471,47 +528,64 @@ class PackageDir ( object ):
 		# it should be sufficient to call create_manifest for one ebuild,
 		#  choosing the latest one that exists in self.physical_location and
 		#  has enough data (DISTDIR, EBUILD_FILE) for this task.
+		#  Additionally, all DISTDIRs (multiple repos, sub directories) have
+		#  to be collected and passed to Manifest creation.
+		#  => collect suitable PackageInfo objects from self._packages
 		#
-		# metadata.xml's full path cannot be used for manifest creation here
-		#  'cause DISTDIR would be unknown
-		#
-
-		# collect suitable PackageInfo instances
 		pkgs_for_manifest = tuple (
 			p for p in self._packages.values() \
 				if p.has ( 'distdir', 'ebuild_file' )
 		)
 
 		if pkgs_for_manifest:
-			manifest.create_manifest ( pkgs_for_manifest, nofail=False )
-			self._need_manifest = False
+			if manifest.create_manifest ( pkgs_for_manifest, nofail=False ):
+				self._need_manifest = False
+				return True
+		elif ignore_empty:
+			return True
 		else:
+			# FIXME: debug statements
+			# FIXME: remove excpetion, maybe delete Manifest in this case,..
+			for pvr, p in self._packages.items():
+				print ( "{} {} ebuild={} efile={} has={}".format (
+					pvr, p, p.has ('ebuild'), p ['ebuild_file'], self.has_ebuilds()
+				) )
+
 			raise Exception (
-				"No ebuild written so far! I really don't know what do to!"
-			)
+				'In {mydir}: No ebuild written so far! '
+				'I really don\'t know what do to!'.format (
+					mydir=self.physical_location
+			) )
+
+		return False
 	# --- end of write_manifest (...) ---
 
 	def write_metadata ( self, shared_fh=None ):
-		"""Writes metadata for this package."""
+		"""Writes metadata for this package.
+
+		returns: success (True/False)
+		"""
+		success = False
 		try:
 			self.generate_metadata ( skip_if_existent=True )
 
 			if shared_fh is None:
+				util.dodir ( self.physical_location, mkdir_p=True )
 				if self._metadata.write():
 					self._need_metadata = False
 					self._need_manifest = True
-					return True
+					success = True
 				else:
 					self.logger.error (
 						"Failed to write metadata file {}.".format (
 							self._metadata.filepath
 						)
 					)
-					return False
 			else:
 				self._metadata.show ( shared_fh )
-				return True
+				success = True
 		except Exception as e:
 			self.logger.exception ( e )
-			return False
+
+		return success
 	# --- end of write_metadata (...) ---

diff --git a/roverlay/overlay/root.py b/roverlay/overlay/root.py
index 363bdb5..219fd70 100644
--- a/roverlay/overlay/root.py
+++ b/roverlay/overlay/root.py
@@ -48,10 +48,10 @@ class Overlay ( object ):
 
 		self.incremental = incremental
 		if self.incremental:
-
-			self._incremental_write_lock = threading.Lock()
+			# this is multiple-run incremental writing (in contrast to runtime
+			# incremental writing, which writes ebuilds as soon as they're
+			# ready) FIXME: split incremental <-> runtime_incremental
 			self.scan()
-			self._init_overlay ( reimport_eclass=True )
 
 	# --- end of __init__ (...) ---
 
@@ -73,8 +73,6 @@ class Overlay ( object ):
 						incremental=self.incremental
 					)
 					self._categories [category] = newcat
-					if self.incremental:
-						util.dodir ( newcat.physical_location )
 			finally:
 				self._catlock.release()
 
@@ -226,50 +224,10 @@ class Overlay ( object ):
 		return cat.add ( package_info )
 	# --- end of add (...) ---
 
-	def finalize_write_incremental ( self ):
-		"""Writes metadata + Manifest for all packages."""
-		for cat in self._categories.values():
-			cat.finalize_write_incremental()
-	# --- end of finalize_incremental (...) ---
-
-	def generate_manifest ( self, **manifest_kw ):
-		"""Generates Manifest files for all ebuilds in this overlay that exist
-		physically/in filesystem.
-		Manifest files are automatically created when calling write().
-
-		arguments:
-		* **manifest_kw -- see PackageDir.generate_manifest(...)
-
-		returns: None (implicit)
-		"""
-		for cat in self._categories.values():
-			cat.generate_manifest ( **manifest_kw )
-	# --- end of generate_manifest (...) ---
-
-	def generate_metadata ( self, **metadata_kw ):
-		"""Tells the overlay's categories to create metadata.
-		You don't have to call this before write()/show() unless you want to use
-		special metadata options.
-
-		arguments:
-		* **metadata_kw -- keywords for package.PackageDir.generate_metadata(...)
-
-		returns: None (implicit)
-		"""
-		for cat in self._categories.values():
-			cat.generate_metadata ( **metadata_kw )
-	# --- end of generate_metadata (...) ---
-
 	def has_dir ( self, _dir ):
 		return os.path.isdir ( self.physical_location + os.sep + _dir )
 	# --- end of has_category (...) ---
 
-	def keep_nth_latest ( self, *args, **kwargs ):
-		"""See package.py:PackageDir:keep_nth_latest."""
-		for cat in self._categories.values():
-			cat.keep_nth_latest ( *args, **kwargs )
-	# --- end of keep_nth_latest (...) ---
-
 	def list_packages ( self, for_deprules=True ):
 		for cat in self._categories.values():
 			for package in cat.list_packages ( for_deprules=True ):
@@ -313,9 +271,9 @@ class Overlay ( object ):
 			cat.show ( **show_kw )
 	# --- end of show (...) ---
 
-	def write ( self, **write_kw ):
+	def write ( self ):
 		"""Writes the overlay to its physical location (filesystem), including
-		metadata and Manifest files.
+		metadata and Manifest files as well as cleanup actions.
 
 		arguments:
 		* **write_kw -- keywords for package.PackageDir.write(...)
@@ -327,35 +285,27 @@ class Overlay ( object ):
 		! TODO/FIXME/DOC: This is not thread-safe, it's expected to be called
 		when ebuild creation is done.
 		"""
-		raise Exception ( "to be removed/replaced" )
-		# writing profiles/ here, rewriting categories/ later
 		self._init_overlay ( reimport_eclass=True )
 
 		for cat in self._categories.values():
-			if not cat.empty():
-				util.dodir ( cat.physical_location )
-				cat.write ( **write_kw )
+			cat.write (
+				overwrite_ebuilds=False,
+				keep_n_ebuilds=config.get ( 'OVERLAY.keep_nth_latest', None ),
+				cautious=True
+			)
 	# --- end of write (...) ---
 
-	def write_incremental ( self, **write_kw ):
-		"""Writes all ebuilds that have been modified since the last write call.
-		Note that there are currently two modes of incremental writing:
-		(a) per-PackageDir incremental writing triggered by the new_ebuild()
-		event method and (b) "batched" incremental writing (this method) which
-		writes all modified PackageDirs.
-		"""
-		# FIXME merge with write(), making incremental writing the only option
-		# FIXME finalize_write_incremental?
-		if not self._incremental_write_lock.acquire():
-			# another incremental write is running, drop this request
-			return
+	def write_manifest ( self, **manifest_kw ):
+		"""Generates Manifest files for all ebuilds in this overlay that exist
+		physically/in filesystem.
+		Manifest files are automatically created when calling write().
 
-		try:
-			util.dodir ( self.physical_location )
-			cats = tuple ( self._categories.values() )
-			for cat in cats:
-				util.dodir ( cat.physical_location )
-				cat.write_incremental ( **write_kw )
-		finally:
-			self._incremental_write_lock.release()
-	# --- end of write_incremental (...) ---
+		arguments:
+		* **manifest_kw -- see PackageDir.generate_manifest(...)
+
+		returns: None (implicit)
+		"""
+		# FIXME: it would be good to ensure that profiles/categories exist
+		for cat in self._categories.values():
+			cat.write_manifest ( **manifest_kw )
+	# --- end of write_manifest (...) ---