[gentoo-commits] proj/R_overlay:master commit in: /, roverlay/, roverlay/overlay/

[André Erdmann]

commit:     34861f2cd5d5db7ca3ba99a8bc62ab5e75bbfd34
Author:     André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Fri Jul 20 16:14:04 2012 +0000
Commit:     André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Fri Jul 20 16:14:04 2012 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=34861f2c

incremental overlay writing (#3)

* Ebuilds can now be written incrementally (but without deletion of old ebuilds)
* Can now create dependency rules for scanned ebuilds / ebuilds in the overlay
* Ebuilds will now only be created if necessary (= does not already exist as file
  or Manifest is missing). This will later be configurable.

	geändert:   roverlay.py
	geändert:   roverlay/overlay/category.py
	geändert:   roverlay/overlay/creator.py
	neue Datei:   roverlay/overlay/header.py
	geändert:   roverlay/overlay/package.py
	geändert:   roverlay/overlay/root.py
	geändert:   roverlay/packageinfo.py

---
 roverlay.py                  |    6 +-
 roverlay/overlay/category.py |  202 ++++++++++++++++++++++--------------------
 roverlay/overlay/creator.py  |   78 ++++++++--------
 roverlay/overlay/header.py   |   37 ++++++++
 roverlay/overlay/package.py  |  147 +++++++++++++++++--------------
 roverlay/overlay/root.py     |   96 +++++++-------------
 roverlay/packageinfo.py      |    7 +-
 7 files changed, 304 insertions(+), 269 deletions(-)

diff --git a/roverlay.py b/roverlay.py
index ae67d6a..3504fab 100755
--- a/roverlay.py
+++ b/roverlay.py
@@ -221,9 +221,7 @@ def run_overlay_create():
 	#run_sync()
 	try:
 		global overlay
-		overlay = OverlayCreator()
-		# explicitly allow overlay writing (FIXME: remove that in OverlayCreator)
-		overlay.can_write_overlay = OPTION ( 'write_overlay' )
+		overlay = OverlayCreator ( allow_write=OPTION ( 'write_overlay' ) )
 
 		repo_list.add_packages ( overlay.add_package )
 
@@ -259,6 +257,6 @@ if 'create' in actions: run_overlay_create()
 
 if len ( actions ) > len ( actions_done ):
 	die (
-		"Some actions (out of %r) could not be performed!" % actions,
+		"Some actions (out of {!r}) could not be performed!".format ( actions ),
 		DIE.CMD_LEFTOVER
 	)

diff --git a/roverlay/overlay/category.py b/roverlay/overlay/category.py
index 9cf8ad9..1d0bf97 100644
--- a/roverlay/overlay/category.py
+++ b/roverlay/overlay/category.py
@@ -18,73 +18,39 @@ class Category ( object ):
 
 	WRITE_JOBCOUNT = 3
 
-	def __init__ ( self, name, logger, directory ):
+	def __init__ ( self, name, logger, directory, get_header, incremental ):
 		"""Initializes a overlay/portage category (such as 'app-text', 'sci-R').
 
 		arguments:
-		* name      -- name of the category
-		* logger    -- parent logger
-		* directory -- filesystem location
+		* name       -- name of the category
+		* logger     -- parent logger
+		* directory  -- filesystem location
+		* get_header -- function that returns an ebuild header
 		"""
 		self.logger            = logger.getChild ( name )
 		self.name              = name
 		self._lock             = threading.RLock()
 		self._subdirs          = dict()
 		self.physical_location = directory
+		self.get_header        = get_header
+		self.incremental       = incremental
 	# --- end of __init__ (...) ---
 
-	def has ( self, subdir ):
-		return subdir in self._subdirs
-	# --- end of has (...) ---
-
-	def list_packages ( self, print_category=True ):
-		if print_category:
-			for package in self._subdirs.keys():
-				yield self.name + os.sep + package
-		else:
-			for package in self._subdirs.keys():
-				yield package
-	# --- end of list_packages (...) ---
-
-	def has_dir ( self, _dir ):
-		return os.path.isdir ( self.physical_location + os.sep + _dir )
-	# --- end of has_category (...) ---
-
-	def _scan_packages ( self ):
-		for x in os.listdir ( self.physical_location ):
-			if self.has_dir ( x ):
-				yield self._get_package_dir ( x )
-	# --- end of _scan_packages (...) ---
-
-	def scan ( self, **kw ):
-		for pkg in self._scan_packages():
-			pkg.scan ( **kw )
-	# --- end of scan (...) ---
-
-	def empty ( self ):
-		"""Returns True if this category contains 0 ebuilds."""
-		return \
-			len ( self._subdirs ) == 0 or \
-			not False in ( d.empty() for d in self._subdirs.values() )
-	# --- end of empty (...) ---
-
-	def finalize_write_incremental ( self ):
-		for subdir in self._subdirs.values():
-			if subdir.modified:
-				subdir.write_incremental()
-			subdir.finalize_write_incremental()
-	# --- end of finalize_write_incremental (...) ---
-
 	def _get_package_dir ( self, pkg_name ):
 		if not pkg_name in self._subdirs:
 			self._lock.acquire()
 			try:
 				if not pkg_name in self._subdirs:
-					self._subdirs [pkg_name] = PackageDir (
-						pkg_name,
-						self.logger,
-						self.physical_location + os.sep + pkg_name
+					newpkg = PackageDir (
+						name        = pkg_name,
+						logger      = self.logger,
+						directory   = self.physical_location + os.sep + pkg_name,
+						get_header  = self.get_header,
+						incremental = self.incremental
 					)
+					self._subdirs [pkg_name] = newpkg
+					if self.incremental:
+						roverlay.util.dodir ( newpkg.physical_location )
 			finally:
 				self._lock.release()
 
@@ -103,6 +69,20 @@ class Category ( object ):
 		return subdir.add ( package_info, **pkg_add_kw )
 	# --- end of add (...) ---
 
+	def empty ( self ):
+		"""Returns True if this category contains 0 ebuilds."""
+		return \
+			len ( self._subdirs ) == 0 or \
+			not False in ( d.empty() for d in self._subdirs.values() )
+	# --- end of empty (...) ---
+
+	def finalize_write_incremental ( self ):
+		for subdir in self._subdirs.values():
+			if subdir.modified:
+				subdir.write_incremental()
+			subdir.finalize_write_incremental()
+	# --- end of finalize_write_incremental (...) ---
+
 	def generate_metadata ( self, **metadata_kw ):
 		"""Generates metadata for all packages in this category.
 		Metadata are automatically generated when calling write().
@@ -116,18 +96,39 @@ class Category ( object ):
 			package.generate_metadata ( **metadata_kw )
 	# --- end of generate_metadata (...) ---
 
-	def write_manifest ( self, **manifest_kw ):
-		"""Generates Manifest files for all packages in this category.
-		Manifest files are automatically created when calling write().
+	def has ( self, subdir ):
+		return subdir in self._subdirs
+	# --- end of has (...) ---
 
-		arguments:
-		* **manifest_kw -- see PackageDir.write_manifest(...)
+	def has_dir ( self, _dir ):
+		return os.path.isdir ( self.physical_location + os.sep + _dir )
+	# --- end of has_category (...) ---
 
-		returns: None (implicit)
+	def list_packages ( self, for_deprules=False ):
+		"""Lists all packages in this category.
+		Yields <category>/<package name> or a dict (see for_deprules below).
+
+		arguments:
+		* for_deprules -- if set and True:
+		                   yield keyword args for dependency rules
 		"""
-		for package in self._subdirs.values():
-			package.write_manifest ( **manifest_kw )
-	# --- end of write_manifest (...) ---
+
+		for name, subdir in self._subdirs.items():
+			if not subdir.empty():
+				if for_deprules:
+					yield dict (
+						dep_str=name, resolving_package=self.name + os.sep + name
+					)
+				else:
+					yield self.name + os.sep + name
+	# --- end of list_packages (...) ---
+
+	def scan ( self, **kw ):
+		"""Scans this category for existing ebuilds."""
+		for subdir in os.listdir ( self.physical_location ):
+			if self.has_dir ( subdir ):
+				self._get_package_dir ( subdir ).scan ( **kw )
+	# --- end of scan (...) ---
 
 	def show ( self, **show_kw ):
 		"""Prints this category (its ebuild and metadata files).
@@ -138,46 +139,28 @@ class Category ( object ):
 			package.show ( **show_kw )
 	# --- end of show (...) ---
 
-	def _run_write_queue ( self, q, write_kw ):
-		"""Calls <package>.write for every <package> received from the queue.
-
-		arguments:
-		* q        -- queue
-		* write_kw --
-		"""
-		try:
-			while not q.empty():
-				pkg = q.get_nowait()
-				pkg.write ( write_manifest=False, **write_kw )
-
-		except queue.Empty:
-			pass
-		except ( Exception, KeyboardInterrupt ) as e:
-			self.RERAISE_EXCEPTION = e
-
-	# --- end of _run_write_queue (...) ---
-
-	def write_incremental ( self, **write_kw ):
-		"""Writes this category incrementally."""
-		try:
-			with self._lock:
-				# new package dirs could be added during overlay writing,
-				# so collect the list of package dirs before iterating over it
-				subdirs = tuple ( self._subdirs.values() )
-
-			for subdir in subdirs:
-				if subdir.modified:
-					roverlay.util.dodir ( subdir.physical_location )
-					subdir.write_incremental ( **write_kw )
-		except Exception as e:
-			self.logger.exception ( e )
-	# --- end of write_incremental (...) ---
-
 	def write ( self, **write_kw ):
 		"""Writes this category to its filesystem location.
 
 		returns: None (implicit)
 		"""
+		def run_write_queue ( q, write_kw ):
+			"""Calls <package>.write for every <package> received from the queue.
+
+			arguments:
+			* q        -- queue
+			* write_kw --
+			"""
+			try:
+				while not q.empty():
+					pkg = q.get_nowait()
+					pkg.write ( write_manifest=False, **write_kw )
+
+			except queue.Empty:
+				pass
+			except ( Exception, KeyboardInterrupt ) as e:
+				self.RERAISE_EXCEPTION = e
+		# --- end of run_write_queue (...) ---
 
 		max_jobs = self.__class__.WRITE_JOBCOUNT
 
@@ -198,7 +181,7 @@ class Category ( object ):
 
 				workers = (
 					threading.Thread (
-						target=self._run_write_queue,
+						target=run_write_queue,
 						args=( write_queue, write_kw )
 					) for n in range ( max_jobs )
 				)
@@ -219,3 +202,32 @@ class Category ( object ):
 					roverlay.util.dodir ( package.physical_location )
 					package.write ( write_manifest=True, **write_kw )
 	# --- end of write (...) ---
+
+	def write_incremental ( self, **write_kw ):
+		"""Writes this category incrementally."""
+		try:
+			with self._lock:
+				# new package dirs could be added during overlay writing,
+				# so collect the list of package dirs before iterating over it
+				subdirs = tuple ( self._subdirs.values() )
+
+			for subdir in subdirs:
+				if subdir.modified:
+					roverlay.util.dodir ( subdir.physical_location )
+					subdir.write_incremental ( **write_kw )
+		except Exception as e:
+			self.logger.exception ( e )
+	# --- end of write_incremental (...) ---
+
+	def write_manifest ( self, **manifest_kw ):
+		"""Generates Manifest files for all packages in this category.
+		Manifest files are automatically created when calling write().
+
+		arguments:
+		* **manifest_kw -- see PackageDir.write_manifest(...)
+
+		returns: None (implicit)
+		"""
+		for package in self._subdirs.values():
+			package.write_manifest ( **manifest_kw )
+	# --- end of write_manifest (...) ---

diff --git a/roverlay/overlay/creator.py b/roverlay/overlay/creator.py
index dd7c11e..114f12d 100644
--- a/roverlay/overlay/creator.py
+++ b/roverlay/overlay/creator.py
@@ -23,9 +23,7 @@ from roverlay.packageinfo        import PackageInfo
 
 from roverlay.recipe             import easyresolver
 
-# this is used as the default value for can_write_overlay in OverlayCreator
-# instances
-OVERLAY_WRITE_ALLOWED = False
+USE_INCREMENTAL_WRITE = True
 
 class PseudoAtomicCounter ( object ):
 
@@ -74,7 +72,7 @@ class OverlayCreator ( object ):
 
 	LOGGER = logging.getLogger ( 'OverlayCreator' )
 
-	def __init__ ( self, logger=None ):
+	def __init__ ( self, logger=None, allow_write=True ):
 
 		if logger is None:
 			self.logger = self.__class__.LOGGER
@@ -84,6 +82,9 @@ class OverlayCreator ( object ):
 		# this queue is used to propagate exceptions from threads
 		self._err_queue = errorqueue.ErrorQueue()
 
+		self.can_write_overlay = allow_write
+		self.write_incremental = allow_write and USE_INCREMENTAL_WRITE
+
 		# init overlay using config values
 		self.overlay = Overlay (
 			name=config.get_or_fail ( 'OVERLAY.name' ),
@@ -91,10 +92,12 @@ class OverlayCreator ( object ):
 			directory=config.get_or_fail ( 'OVERLAY.dir' ),
 			default_category= config.get_or_fail ( 'OVERLAY.category' ),
 			eclass_files=config.get ( 'OVERLAY.eclass_files', None ),
-			ebuild_header=config.get ( 'EBUILD.default_header', None )
+			ebuild_header=config.get ( 'EBUILD.default_header', None ),
+			incremental=self.write_incremental
 		)
 
 		self.depresolver = easyresolver.setup ( self._err_queue )
+		self.depresolver.make_selfdep_pool ( self.overlay.list_rule_kwargs )
 
 		self.NUMTHREADS  = config.get ( 'EBUILD.jobcount', 0 )
 
@@ -107,9 +110,6 @@ class OverlayCreator ( object ):
 		self._workers   = None
 		self._runlock   = threading.RLock()
 
-		self.can_write_overlay = OVERLAY_WRITE_ALLOWED
-		self.write_incremental = True
-
 		self.closed = False
 
 		# queued packages counter,
@@ -208,8 +208,10 @@ class OverlayCreator ( object ):
 		arguments:
 		* package_info --
 		"""
-		self._pkg_queue.put ( package_info )
-		self.package_added.inc()
+		if self.overlay.add ( package_info ):
+			self._pkg_queue.put ( package_info )
+			# FIXME package_added is now the # of packages queued for creation
+			self.package_added.inc()
 	# --- end of add_package (...) ---
 
 	def add_package_file ( self, package_file ):
@@ -226,19 +228,18 @@ class OverlayCreator ( object ):
 		self.package_added.inc()
 	# --- end of add_package_files (...) ---
 
-	def write_overlay ( self, incremental=False ):
+	def write_overlay ( self ):
 		"""Writes the overlay.
 
 		arguments:
-		* incremental -- (TODO)
 		"""
 		if self.can_write_overlay:
-			start = time.time()
-			if incremental:
-				self.overlay.write_incremental()
+			if self.write_incremental:
+				self.overlay.finalize_write_incremental()
 			else:
+				start = time.time()
 				self.overlay.write()
-			self._timestamp ( "overlay written", start )
+				self._timestamp ( "overlay written", start )
 		else:
 			self.logger.warning ( "Not allowed to write overlay!" )
 	# --- end of write_overlay (...) ---
@@ -267,6 +268,7 @@ class OverlayCreator ( object ):
 		self._runlock.acquire()
 		try:
 			self.join()
+			self.depresolver.reload_pools()
 			self._make_workers()
 		except:
 			self._err_queue.push ( context=-1, error=None )
@@ -282,28 +284,28 @@ class OverlayCreator ( object ):
 
 	def close ( self ):
 		"""Closes this OverlayCreator."""
-		self._close_workers()
-		self._close_resolver()
-		self.closed = True
-	# --- end of close (...) ---
+		def close_resolver():
+			"""Tells the dependency resolver to close.
+			This is useful 'cause certain depres listener modules will write files
+			when told to exit.
+			"""
+			if self.depresolver is None: return
 
-	def _close_resolver ( self ):
-		"""Tells the dependency resolver to close.
-		This is useful 'cause certain depres listener modules will write files
-		when told to exit.
-		"""
-		if self.depresolver is None: return
+			self._runlock.acquire()
 
-		self._runlock.acquire()
+			try:
+				if self.depresolver is not None:
+					self.depresolver.close()
+					del self.depresolver
+					self.depresolver = None
+			finally:
+				self._runlock.release()
+		# --- end of close_resolver (...) ---
 
-		try:
-			if self.depresolver is not None:
-				self.depresolver.close()
-				del self.depresolver
-				self.depresolver = None
-		finally:
-			self._runlock.release()
-	# --- end of _close_resolver (...) ---
+		self._close_workers()
+		close_resolver()
+		self.closed = True
+	# --- end of close (...) ---
 
 	def _waitfor_workers ( self, do_close ):
 		"""Waits until the workers are done.
@@ -396,12 +398,8 @@ class OverlayCreator ( object ):
 		# if <>:
 		if package_info ['ebuild'] is not None:
 			self.create_success.inc()
-			if self.overlay.add (
-				package_info,
-				write_after_add=self.write_incremental and self.can_write_overlay
-			):
+			if package_info.overlay_package_ref.new_ebuild():
 				self.overlay_added.inc()
-
 		else:
 			self.create_fail.inc()
 

diff --git a/roverlay/overlay/header.py b/roverlay/overlay/header.py
new file mode 100644
index 0000000..b6d15d7
--- /dev/null
+++ b/roverlay/overlay/header.py
@@ -0,0 +1,37 @@
+class EbuildHeader ( object ):
+	def __init__ ( self, default_header ):
+		self.default_header = default_header
+		self.eclasses       = ()
+
+		self._cached_header = None
+	# --- end of __init__ (...) ---
+
+	def set_eclasses ( self, eclass_names ):
+		self.eclasses = eclass_names
+	# --- end of set_eclasses (...) ---
+
+	def get ( self, use_cached=True ):
+		if self._cached_header is None or not use_cached:
+			self._cached_header = self._make()
+		return self._cached_header
+	# --- end of get (...) ---
+
+	def _make ( self ):
+		if self.eclasses:
+			inherit = 'inherit ' + ' '.join ( sorted ( self.eclasses ) )
+		else:
+			inherit = None
+
+		# header and inherit is expected and therefore the first condition here
+		if inherit and self.default_header:
+			return self.default_header + '\n' + inherit
+
+		elif inherit:
+			return inherit
+
+		elif self.default_header:
+			return self.default_header
+
+		else:
+			return None
+	# --- end of _make (...) ---

diff --git a/roverlay/overlay/package.py b/roverlay/overlay/package.py
index 71b7ea9..5b73788 100644
--- a/roverlay/overlay/package.py
+++ b/roverlay/overlay/package.py
@@ -1,10 +1,8 @@
-# replacement for package.py
-
 import os
 import sys
 import threading
 
-from roverlay                  import manifest
+from roverlay.overlay          import manifest
 from roverlay.packageinfo      import PackageInfo
 from roverlay.overlay.metadata import MetadataJob
 
@@ -14,23 +12,26 @@ SUPPRESS_EXCEPTIONS = True
 class PackageDir ( object ):
 	EBUILD_SUFFIX = '.ebuild'
 
-	def __init__ ( self, name, logger, directory ):
+	def __init__ ( self, name, logger, directory, get_header, incremental ):
 		"""Initializes a PackageDir which contains ebuilds, metadata and
 		a Manifest file.
 
 		arguments:
-		* name      -- name of the directory (${PN} in ebuilds)
-		* logger    -- parent logger
-		* directory -- filesystem location of this PackageDir
+		* name       -- name of the directory (${PN} in ebuilds)
+		* logger     -- parent logger
+		* directory  -- filesystem location of this PackageDir
+		* get_header -- function that returns an ebuild header
 		"""
-		self.logger            = logger.getChild ( name )
-		self.name              = name
-		self._lock             = threading.RLock()
+		self.logger              = logger.getChild ( name )
+		self.name                = name
+		self._lock               = threading.RLock()
 		# { <version> : <PackageInfo> }
-		self._packages         = dict()
-		self.physical_location = directory
+		self._packages           = dict()
+		self.physical_location   = directory
+		self.get_header          = get_header
+		self.runtime_incremental = incremental
 
-		self._metadata         = MetadataJob (
+		self._metadata = MetadataJob (
 			filepath = self.physical_location + os.sep + 'metadata.xml',
 			logger   = self.logger
 		)
@@ -41,18 +42,12 @@ class PackageDir ( object ):
 			self.name + "-{PVR}" + self.__class__.EBUILD_SUFFIX
 
 		# used to track changes for this package dir
-		self.modified = False
+		self.modified          = False
 		self._manifest_package = None
-		self._need_manifest = False
-		self._need_metadata = False
+		self._need_manifest    = False
+		self._need_metadata    = False
 	# --- end of __init__ (...) ---
 
-	def new_ebuild ( self ):
-		"""Called when a new ebuild has been created for this PackageDir."""
-		self._need_manifest = True
-		self.modified       = True
-	# --- end of new_ebuild (...) ---
-
 	def add ( self, package_info, add_if_physical=False ):
 		"""Adds a package to this PackageDir.
 
@@ -66,7 +61,7 @@ class PackageDir ( object ):
 		raises: Exception when ebuild already exists.
 		"""
 		shortver = package_info ['ebuild_verstr']
-		_success = False
+		added = False
 		try:
 			self._lock.acquire()
 			if shortver in self._packages:
@@ -75,7 +70,8 @@ class PackageDir ( object ):
 					if not skip_if_physical:
 						# ignore ebuilds that exist as file
 						self._packages [shortver] = package_info
-						_success = True
+						added = True
+
 					else:
 						self.logger.debug (
 							"'{PN}-{PVR}.ebuild' exists as file, skipping.".format (
@@ -91,12 +87,20 @@ class PackageDir ( object ):
 					)
 			else:
 				self._packages [shortver] = package_info
-				_success = True
+				added = True
 
 		finally:
 			self._lock.release()
 
-		return _success
+		if added:
+			# add a link to this PackageDir into the package info,
+			# !! package_info <-> self (double-linked)
+			# FIXME: remove physical_only flag from PackageInfo if
+			#         overlay_package_ref can be used for that
+			package_info.overlay_package_ref = self
+			return True
+		else:
+			return False
 	# --- end of add (...) ---
 
 	def empty ( self ):
@@ -129,6 +133,17 @@ class PackageDir ( object ):
 		return self._packages.keys()
 	# --- end of list_versions (...) ---
 
+	def new_ebuild ( self, write=False ):
+		"""Called when a new ebuild has been created for this PackageDir."""
+		self._need_manifest = True
+		self._need_metadata = True
+		self.modified       = True
+		if self.runtime_incremental:
+			return self.write_incremental()
+		else:
+			return True
+	# --- end of new_ebuild (...) ---
+
 	def scan ( self, **kw ):
 		"""Scans the filesystem location of this package for existing
 		ebuilds and adds them.
@@ -166,13 +181,15 @@ class PackageDir ( object ):
 					)
 		# --- end of scan_ebuilds (...) ---
 
-		for pvr in scan_ebuilds():
-			if pvr not in self._packages:
-				p = PackageInfo ( physical_only=True, pvr=pvr )
-				self._packages [ p ['ebuild_verstr'] ] = p
+		# ignore directories without a Manifest file
+		if os.path.isfile ( self.physical_location + os.sep + 'Manifest' ):
+			for pvr in scan_ebuilds():
+				if pvr not in self._packages:
+					p = PackageInfo ( physical_only=True, pvr=pvr )
+					self._packages [ p ['ebuild_verstr'] ] = p
 	# --- end of scan (...) ---
 
-	def show ( self, stream=sys.stderr, default_header=None ):
+	def show ( self, stream=sys.stderr ):
 		"""Prints this dir (the ebuilds and the metadata) into a stream.
 
 		arguments:
@@ -183,18 +200,16 @@ class PackageDir ( object ):
 		raises:
 		* IOError
 		"""
-		return self.write ( default_header=default_header, shared_fh=stream )
+		return self.write ( shared_fh=stream )
 	# --- end of show (...) ---
 
 	def write ( self,
-		default_header=None, shared_fh=None,
-		write_ebuilds=True, write_manifest=True, write_metadata=True,
-		overwrite_ebuilds=True
+		shared_fh=None, overwrite_ebuilds=True,
+		write_ebuilds=True, write_manifest=True, write_metadata=True
 	):
 		"""Writes this directory to its (existent!) filesystem location.
 
 		arguments:
-		* default_header    -- ebuild header to write
 		* shared_fh         -- if set and not None: write everyting into <fh>
 		* write_ebuilds     -- if set and False: don't write ebuilds
 		* write_manifest    -- if set and False: don't write the Manifest file
@@ -212,8 +227,7 @@ class PackageDir ( object ):
 			# write ebuilds
 			if write_ebuilds:
 				self.write_ebuilds (
-					default_header=default_header, shared_fh=shared_fh,
-					overwrite=overwrite_ebuilds
+					overwrite=overwrite_ebuilds, shared_fh=shared_fh
 				)
 
 			# write metadata
@@ -225,30 +239,30 @@ class PackageDir ( object ):
 				self.write_manifest()
 	# --- end of write (...) ---
 
-	def write_ebuilds ( self, default_header, overwrite, shared_fh=None ):
+	def write_ebuilds ( self, overwrite, shared_fh=None ):
 		"""Writes all ebuilds.
 
 		arguments:
-		* default_header -- ebuild header
 		* shared_fh      -- if set and not None: don't use own file handles
 		                     (i.e. write files), write everything into shared_fh
 		* overwrite      -- write ebuilds that have been written before,
 		                     defaults to True
 		"""
+		ebuild_header = self.get_header()
+
 		def write_ebuild ( efile, ebuild ):
 			"""Writes an ebuild.
 
 			arguments:
 			* efile  -- file to write
 			* ebuild -- ebuild object to write (has to have a __str__ method)
-			* (default_header from write_ebuilds())
 			* (shared_fh from write_ebuilds())
 			"""
 			_success = False
 			try:
 				fh = open ( efile, 'w' ) if shared_fh is None else shared_fh
-				if default_header is not None:
-					fh.write ( str ( default_header ) )
+				if ebuild_header is not None:
+					fh.write ( str ( ebuild_header ) )
 					fh.write ( '\n\n' )
 				fh.write ( str ( ebuild ) )
 				fh.write ( '\n' )
@@ -304,11 +318,12 @@ class PackageDir ( object ):
 				)
 
 		self.modified = not all_ebuilds_written
+		return all_ebuilds_written
 	# --- end of write_ebuilds (...) ---
 
-	def write_incremental ( self, default_header ):
+	def write_incremental ( self ):
 		with self._lock:
-			self.write_ebuilds ( default_header=default_header, overwrite=False )
+			return self.write_ebuilds ( overwrite=False )
 	# --- end of write_incremental (...) ---
 
 	def write_manifest ( self ):
@@ -330,8 +345,14 @@ class PackageDir ( object ):
 		#  'cause DISTDIR would be unknown
 		#
 
-		if self._manifest_package is not None:
-			manifest.create_manifest ( self._manifest_package, nofail=False )
+		# collect suitable PackageInfo instances
+		pkgs_for_manifest = tuple (
+			p for p in self._packages.values() \
+				if p.has ( 'distdir', 'ebuild_file' )
+		)
+
+		if pkgs_for_manifest:
+			manifest.create_manifest ( pkgs_for_manifest, nofail=False )
 			self._need_manifest = False
 		else:
 			raise Exception (
@@ -341,28 +362,24 @@ class PackageDir ( object ):
 
 	def write_metadata ( self, shared_fh=None ):
 		"""Writes metadata for this package."""
-		if self._manifest_package is None and not shared_fh:
-			self.logger.error (
-				'write_metadata() requested, but no ebuild written so far! '
-				'This will most likely result in a corrupt Manifest.'
-			)
-
 		try:
 			self.generate_metadata ( skip_if_existent=True )
 
-			if self._metadata.write() \
-				if shared_fh is None else self._metadata.show ( shared_fh ) \
-			:
-				self._need_metadata = False
-				self._need_manifest = True
-				return True
-			else:
-				self.logger.error (
-					"Failed to write metadata file {}.".format (
-						self._metadata.filepath
+			if shared_fh is None:
+				if self._metadata.write():
+					self._need_metadata = False
+					self._need_manifest = True
+					return True
+				else:
+					self.logger.error (
+						"Failed to write metadata file {}.".format (
+							self._metadata.filepath
+						)
 					)
-				)
-				return False
+					return False
+			else:
+				self._metadata.show ( shared_fh )
+				return True
 		except:
 			# already logged
 			return False

diff --git a/roverlay/overlay/root.py b/roverlay/overlay/root.py
index e0446df..4766b02 100644
--- a/roverlay/overlay/root.py
+++ b/roverlay/overlay/root.py
@@ -10,50 +10,13 @@ import os
 from roverlay import config, util
 
 from roverlay.overlay.category import Category
+from roverlay.overlay.header   import EbuildHeader
 
 DEFAULT_USE_DESC = '\n'.join ( [
 	'byte-compile - enable byte compiling',
 	'R_suggests - install recommended packages'
 ] )
 
-class EbuildHeader ( object ):
-	def __init__ ( self, default_header ):
-		self.default_header = default_header
-		self.eclasses       = ()
-
-		self._cached_header = None
-	# --- end of __init__ (...) ---
-
-	def set_eclasses ( self, eclass_names ):
-		self.eclasses = eclass_names
-	# --- end of set_eclasses (...) ---
-
-	def get ( self, use_cached=True ):
-		if self._cached_header is None or not use_cached:
-			self._cached_header = self._make()
-		return self._cached_header
-	# --- end of get (...) ---
-
-	def _make ( self ):
-		if self.eclasses:
-			inherit = 'inherit ' + ' '.join ( self.eclasses )
-		else:
-			inherit = None
-
-		# header and inherit is expected and therefore the first condition here
-		if inherit and self.default_header:
-			return self.default_header + '\n' + inherit
-
-		elif inherit:
-			return inherit
-
-		elif self.default_header:
-			return self.default_header
-
-		else:
-			return None
-	# --- end of _make (...) ---
-
 
 class Overlay ( object ):
 
@@ -61,7 +24,8 @@ class Overlay ( object ):
 		self,
 		name, logger, directory,
 		default_category, eclass_files,
-		ebuild_header
+		ebuild_header,
+		incremental
 	):
 		if directory is None:
 			raise Exception (
@@ -80,20 +44,23 @@ class Overlay ( object ):
 		self._catlock          = threading.Lock()
 		self._categories       = dict()
 		self._header           = EbuildHeader ( ebuild_header )
-		self._get_header       = self._header.get
 
 		# fixme or ignore: calculating eclass names twice,
 		# once here and another time when calling _init_overlay
-		self._header.set_eclasses ( tuple (
+		self._header.set_eclasses ( set (
 			self._get_eclass_import_info ( only_eclass_names=True )
 		) )
 
-		self._incremental_write_lock = threading.Lock()
+		self.incremental = incremental
+		if self.incremental:
 
+			self._incremental_write_lock = threading.Lock()
+			self.scan()
+			self._init_overlay ( reimport_eclass=True, make_profiles_dir=True )
 
+			for c in self.list_packages ( for_deprules=True ):
+				print ( str ( c ) )
 
-		#self.scan()
-		#raise Exception ( "^" )
 	# --- end of __init__ (...) ---
 
 	def scan ( self, **kw ):
@@ -107,12 +74,18 @@ class Overlay ( object ):
 #			print ( package )
 	# --- end of scan (...) ---
 
-	def list_packages ( self ):
+	def list_packages ( self, for_deprules=True ):
 		for cat in self._categories.values():
-			for package in cat.list_packages():
+			for package in cat.list_packages ( for_deprules=True ):
 				yield package
 	# --- end of list_packages (...) ---
 
+	def list_rule_kwargs ( self ):
+		for cat in self._categories.values():
+			for kwargs in cat.list_packages ( for_deprules=True ):
+				yield kwargs
+	# --- end of list_rule_kwargs (...) ---
+
 	def has_dir ( self, _dir ):
 		return os.path.isdir ( self.physical_location + os.sep + _dir )
 	# --- end of has_category (...) ---
@@ -134,11 +107,16 @@ class Overlay ( object ):
 			self._catlock.acquire()
 			try:
 				if not category in self._categories:
-					self._categories [category] = Category (
+					newcat = Category (
 						category,
 						self.logger,
-						self.physical_location + os.sep + category
+						self.physical_location + os.sep + category,
+						get_header=self._header.get,
+						incremental=self.incremental
 					)
+					self._categories [category] = newcat
+					if self.incremental:
+						util.dodir ( newcat.physical_location )
 			finally:
 				self._catlock.release()
 
@@ -158,18 +136,14 @@ class Overlay ( object ):
 		cat = self._get_category (
 			self.default_category if category is None else category
 		)
-
-		if write_after_add:
-			raise Exception ( "add~write_after_add: to be removed." )
-
-		return cat.add ( package_info, write_after_add=False )
+		return cat.add ( package_info )
 	# --- end of add (...) ---
 
 	def show ( self, **show_kw ):
 		"""Presents the ebuilds/metadata stored in this overlay.
 
 		arguments:
-		* **show_kw -- ignored! (keywords for package.PackageDir.show(...))
+		* **show_kw -- keywords for package.PackageDir.show(...)
 
 		returns: None (implicit)
 		"""
@@ -177,7 +151,7 @@ class Overlay ( object ):
 			tuple ( self._get_eclass_import_info ( only_eclass_names=True ) )
 		)
 		for cat in self._categories.values():
-			cat.show ( default_header=self._get_header() )
+			cat.show ( **show_kw )
 	# --- end of show (...) ---
 
 	def write ( self, **write_kw ):
@@ -185,7 +159,7 @@ class Overlay ( object ):
 		metadata and Manifest files.
 
 		arguments:
-		* **write_kw -- ignored! (keywords for package.PackageDir.write(...))
+		* **write_kw -- keywords for package.PackageDir.write(...)
 
 		returns: None (implicit)
 
@@ -194,13 +168,14 @@ class Overlay ( object ):
 		! TODO/FIXME/DOC: This is not thread-safe, it's expected to be called
 		when ebuild creation is done.
 		"""
+		raise Exception ( "^,^" )
 		# writing profiles/ here, rewriting categories/ later
 		self._init_overlay ( reimport_eclass=True, make_profiles_dir=True )
 
 		for cat in self._categories.values():
 			if not cat.empty():
 				util.dodir ( cat.physical_location )
-				cat.write ( default_header=self._get_header() )
+				cat.write ( **write_kw )
 
 		self._write_categories ( only_active=True )
 	# --- end of write (...) ---
@@ -222,18 +197,15 @@ class Overlay ( object ):
 			cats = tuple ( self._categories.values() )
 			for cat in cats:
 				util.dodir ( cat.physical_location )
-				cat.write_incremental ( default_header=self._get_header() )
+				cat.write_incremental ( **write_kw )
 		finally:
 			self._incremental_write_lock.release()
 	# --- end of write_incremental (...) ---
 
 	def finalize_write_incremental ( self ):
 		"""Writes metadata + Manifest for all packages."""
-		self._init_overlay ( reimport_eclass=True, make_profiles_dir=True )
-
 		for cat in self._categories.values():
 			cat.finalize_write_incremental()
-
 		self._write_categories ( only_active=True )
 	# --- end of finalize_incremental (...) ---
 
@@ -394,7 +366,7 @@ class Overlay ( object ):
 				raise
 	# --- end of _import_eclass (...) ---
 
-	def _init_overlay ( self, reimport_eclass=False, make_profiles_dir=False ):
+	def _init_overlay ( self, reimport_eclass, make_profiles_dir ):
 		"""Initializes the overlay at its physical/filesystem location.
 
 		arguments:

diff --git a/roverlay/packageinfo.py b/roverlay/packageinfo.py
index f9d26d1..829b798 100644
--- a/roverlay/packageinfo.py
+++ b/roverlay/packageinfo.py
@@ -45,9 +45,10 @@ class PackageInfo ( object ):
 		arguments:
 		* **initial_info -- passed to update ( **kw )
 		"""
-		self._info        = dict()
-		self.readonly     = False
-		self._update_lock = threading.RLock()
+		self._info               = dict()
+		self.readonly            = False
+		self._update_lock        = threading.RLock()
+		self.overlay_package_ref = None
 
 		self.update ( **initial_info )
 	# --- end of __init__ (...) ---