[gentoo-commits] proj/R_overlay:master commit in: roverlay/overlay/metadata/, roverlay/overlay/

[André Erdmann]

commit:     b60d7f137f1f99f4b4afe85d1f82ace045d6b211
Author:     André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Mon Jul 23 16:31:50 2012 +0000
Commit:     André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Mon Jul 23 16:31:50 2012 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=b60d7f13

incremental overlay writing: keep n ebuilds

Added keep_nth_latest(n) which keeps the n-th latest ebuilds
of a PackageDir and removes all others.
Also added fs_cleanup() that removes "empty" (no ebuilds) package dirs from
the filesystem.

	geändert:   roverlay/overlay/category.py
	geändert:   roverlay/overlay/creator.py
	geändert:   roverlay/overlay/metadata/abstractnodes.py
	geändert:   roverlay/overlay/metadata/nodes.py
	geändert:   roverlay/overlay/package.py
	geändert:   roverlay/overlay/root.py

---
 roverlay/overlay/category.py               |    7 +
 roverlay/overlay/creator.py                |    1 +
 roverlay/overlay/metadata/abstractnodes.py |    2 +-
 roverlay/overlay/metadata/nodes.py         |    3 +-
 roverlay/overlay/package.py                |  153 +++++++++-
 roverlay/overlay/root.py                   |  428 +++++++++++++---------------
 6 files changed, 352 insertions(+), 242 deletions(-)

diff --git a/roverlay/overlay/category.py b/roverlay/overlay/category.py
index 1d0bf97..33149c4 100644
--- a/roverlay/overlay/category.py
+++ b/roverlay/overlay/category.py
@@ -104,6 +104,13 @@ class Category ( object ):
 		return os.path.isdir ( self.physical_location + os.sep + _dir )
 	# --- end of has_category (...) ---
 
+	def keep_nth_latest ( self, *args, **kwargs ):
+		"""See package.py:PackageDir:keep_nth_latest."""
+		for subdir in self._subdirs.values():
+			subdir.keep_nth_latest ( *args, **kwargs )
+			subdir.fs_cleanup()
+	# --- end of keep_nth_latest (...) ---
+
 	def list_packages ( self, for_deprules=False ):
 		"""Lists all packages in this category.
 		Yields <category>/<package name> or a dict (see for_deprules below).

diff --git a/roverlay/overlay/creator.py b/roverlay/overlay/creator.py
index 114f12d..9de063c 100644
--- a/roverlay/overlay/creator.py
+++ b/roverlay/overlay/creator.py
@@ -304,6 +304,7 @@ class OverlayCreator ( object ):
 
 		self._close_workers()
 		close_resolver()
+		self.overlay.keep_nth_latest ( n=1 )
 		self.closed = True
 	# --- end of close (...) ---
 

diff --git a/roverlay/overlay/metadata/abstractnodes.py b/roverlay/overlay/metadata/abstractnodes.py
index a344fc1..284ea20 100644
--- a/roverlay/overlay/metadata/abstractnodes.py
+++ b/roverlay/overlay/metadata/abstractnodes.py
@@ -242,7 +242,7 @@ class MetadataLeaf ( _MetadataBasicNode ):
 		self._do_verify()
 		if self.print_node_name:
 			return "{indent}<{name}{flags}>{value}</{name}>".format (
-				indent=self._indent,
+				indent=self.indent,
 				name=self.name,
 				flags=self._flagstr(),
 				value=self._value_str(),

diff --git a/roverlay/overlay/metadata/nodes.py b/roverlay/overlay/metadata/nodes.py
index 02d5482..0002267 100644
--- a/roverlay/overlay/metadata/nodes.py
+++ b/roverlay/overlay/metadata/nodes.py
@@ -22,7 +22,8 @@ class MetadataRoot ( MetadataNodeNamedAccess ):
 
 	def __init__ ( self ):
 		super ( MetadataRoot, self ) . __init__ ( 'pkgmetadata' )
-		self.priority = 0
+		self.priority    = 0
+		self.allow_empty = True
 	# --- end of __init__ (...) ---
 
 	def empty ( self ):

diff --git a/roverlay/overlay/package.py b/roverlay/overlay/package.py
index 2ea4cb2..a2e4d9b 100644
--- a/roverlay/overlay/package.py
+++ b/roverlay/overlay/package.py
@@ -1,14 +1,15 @@
 import os
 import sys
 import threading
+import shutil
 
+from roverlay                  import util
 from roverlay.overlay          import manifest
 from roverlay.packageinfo      import PackageInfo
 from roverlay.overlay.metadata import MetadataJob
 
 SUPPRESS_EXCEPTIONS = True
 
-
 class PackageDir ( object ):
 	EBUILD_SUFFIX = '.ebuild'
 
@@ -48,6 +49,22 @@ class PackageDir ( object ):
 		self._need_metadata    = False
 	# --- end of __init__ (...) ---
 
+	def _remove_ebuild_file ( self, pkg_info ):
+		"""Removes the ebuild file of a pkg_info object.
+		Returns True on success, else False.
+		"""
+		try:
+			efile = pkg_info ['ebuild_file']
+			if efile is not None:
+				os.unlink ( efile )
+				# Manifest file has to be updated
+				self._need_manifest = True
+			return True
+		except Exception as e:
+			self.logger.exception ( e )
+			return False
+	# --- end of remove_ebuild_file (...) ---
+
 	def add ( self, package_info, add_if_physical=False ):
 		"""Adds a package to this PackageDir.
 
@@ -105,19 +122,51 @@ class PackageDir ( object ):
 
 	def empty ( self ):
 		"""Returns True if no ebuilds stored, else False.
-		Note that "not empty" does not mean "have ebuilds to write".
+		Note that "not empty" doesn't mean "has ebuilds to write" or "has
+		ebuilds written", use the modified attribute for the former, and the
+		has_ebuilds() function for the latter one.
 		"""
 		return len ( self._packages ) == 0
 	# --- end of empty (...) ---
 
 	def finalize_write_incremental ( self ):
+		"""Method that finalizes incremental writing, i.e. write outstanding
+		ebuilds and write metadata.xml, Manifest.
+		"""
 		with self._lock:
-			if self._need_metadata:
-				self.write_metadata()
-			if self._need_manifest:
-				self.write_manifest()
+			if self.has_ebuilds():
+				if self.modified:
+					self.write_ebuilds ( overwrite=False )
+				if self._need_metadata:
+					self.write_metadata()
+				if self._need_manifest:
+					self.write_manifest()
+			else:
+				self.logger.critical (
+					"<<todo>>: please clean up this dir: {}.".format (
+						self.physical_location
+				) )
 	# --- end of finalize_write_incremental (...) ---
 
+	def fs_cleanup ( self ):
+		"""Cleans up the filesystem location of this package dir.
+		To be called after keep_nth_latest, calls finalize_write_incremental().
+		"""
+		def rmtree_error ( function, path, excinfo ):
+			"""rmtree onerror function that simply logs the exception"""
+			self.logger.exception ( excinfo )
+		# --- end of rmtree_error (...) ---
+
+		with self._lock:
+			if self.has_ebuilds():
+				# !!! FIXME this doesn't work if no ebuilds written, but
+				# old ones removed -> DISTDIR unknown during Manifest creation
+				self.finalize_write_incremental()
+			elif os.path.isdir ( self.physical_location ):
+				# destroy self.physical_location
+				shutil.rmtree ( self.physical_location, onerror=rmtree_error )
+	# --- end of fs_cleanup (...) ---
+
 	def generate_metadata ( self, skip_if_existent, **ignored_kw ):
 		"""Creates metadata for this package.
 
@@ -129,6 +178,70 @@ class PackageDir ( object ):
 				self._metadata.update_using_iterable ( self._packages.values() )
 	# --- end of generate_metadata (...) ---
 
+	def has_ebuilds ( self ):
+		"""Returns True if this PackageDir has any ebuild files (filesystem)."""
+		for p in self._packages.values():
+			if p ['physical_only'] or p.has ( 'ebuild' ):
+				return True
+		return False
+	# --- end of has_ebuilds (...) ---
+
+	def keep_nth_latest ( self, n, cautious=True ):
+		"""Keeps the n-th latest ebuild files, removing all other packages,
+		physically (filesystem) as well as from this PackageDir object.
+
+		arguments:
+		* n        -- # of packages/ebuilds to keep
+		* cautious -- if True: be extra careful, verify that ebuilds exist
+		"""
+
+		# create the list of packages to iterate over,
+		# * package has to have an ebuild_file
+		# * sort them by version in reverse order (latest package gets index 0)
+		packages = reversed ( sorted (
+			filter (
+				lambda p : p [1] ['ebuild_file'] is not None,
+				self._packages.items()
+			),
+			key=lambda p : p [1] ['version']
+		) )
+
+		kept   = 0
+		ecount = 0
+		if not cautious:
+			# could use a slice here, too
+			for pvr, pkg in packages:
+				ecount += 1
+				if kept < n:
+					printself.logger.debug ( "Keeping {pvr}.".format ( pvr=pvr ) )
+					kept += 1
+				else:
+					self.logger.debug ( "Removing {pvr}.".format ( pvr=pvr ) )
+					self.purge_package ( pvr )
+		else:
+			for pvr, pkg in packages:
+				ecount += 1
+				if os.path.isfile ( pkg ['ebuild_file'] ):
+					if kept < n:
+						self.logger.debug ( "Keeping {pvr}.".format ( pvr=pvr ) )
+						kept += 1
+					else:
+						self.logger.debug ( "Removing {pvr}.".format ( pvr=pvr ) )
+						self.purge_package ( pvr )
+				else:
+					self.logger.error (
+						"{efile} is assumed to exist as file but doesn't!".format (
+							efile=pkg ['ebuild_file']
+					) )
+
+		# FIXME/IGNORE: this doesn't count inexistent files as kept
+		self.logger.debug (
+			"Kept {kept}/{total} ebuilds.".format ( kept=kept, total=ecount )
+		)
+
+		# FIXME: Manifest is now invalid and dir could be "empty" (no ebuilds)
+	# --- end of keep_nth_latest (...) ---
+
 	def list_versions ( self ):
 		return self._packages.keys()
 	# --- end of list_versions (...) ---
@@ -144,6 +257,21 @@ class PackageDir ( object ):
 			return True
 	# --- end of new_ebuild (...) ---
 
+	def purge_package ( self, pvr ):
+		"""Removes the PackageInfo with key pvr entirely from this PackageDir,
+		including its ebuild file.
+		Returns: removed PackageInfo object or None.
+		"""
+		try:
+			p = self._packages [pvr]
+			del self._packages [pvr]
+			self._remove_ebuild_file ( p )
+			return p
+		except Exception as e:
+			self.logger.exception ( e )
+			return None
+	# --- end of purge_package (...) ---
+
 	def scan ( self, **kw ):
 		"""Scans the filesystem location of this package for existing
 		ebuilds and adds them.
@@ -168,7 +296,7 @@ class PackageDir ( object ):
 						# not an ebuild
 						pass
 					elif pn == self.name:
-						yield pvr
+						yield ( pvr, self.physical_location + os.sep + f )
 					else:
 						# $PN does not match directory name, warn about that
 						self.logger.warning (
@@ -183,9 +311,11 @@ class PackageDir ( object ):
 
 		# ignore directories without a Manifest file
 		if os.path.isfile ( self.physical_location + os.sep + 'Manifest' ):
-			for pvr in scan_ebuilds():
+			for pvr, efile in scan_ebuilds():
 				if pvr not in self._packages:
-					p = PackageInfo ( physical_only=True, pvr=pvr )
+					p = PackageInfo (
+						physical_only=True, pvr=pvr, ebuild_file=efile
+					)
 					self._packages [ p ['ebuild_verstr'] ] = p
 	# --- end of scan (...) ---
 
@@ -260,6 +390,7 @@ class PackageDir ( object ):
 			"""
 			_success = False
 			try:
+				util.dodir ( self.physical_location )
 				fh = open ( efile, 'w' ) if shared_fh is None else shared_fh
 				if ebuild_header is not None:
 					fh.write ( str ( ebuild_header ) )
@@ -380,7 +511,7 @@ class PackageDir ( object ):
 			else:
 				self._metadata.show ( shared_fh )
 				return True
-		except:
-			# already logged
+		except Exception as e:
+			self.logger.exception ( e )
 			return False
 	# --- end of write_metadata (...) ---

diff --git a/roverlay/overlay/root.py b/roverlay/overlay/root.py
index 51239e2..363bdb5 100644
--- a/roverlay/overlay/root.py
+++ b/roverlay/overlay/root.py
@@ -27,11 +27,6 @@ class Overlay ( object ):
 		ebuild_header,
 		incremental
 	):
-		if directory is None:
-			raise Exception (
-				"support for overlays without filesystem location has been dropped"
-			)
-
 		self.name              = name
 		self.logger            = logger.getChild ( 'overlay' )
 		self.physical_location = directory
@@ -47,7 +42,7 @@ class Overlay ( object ):
 
 		# fixme or ignore: calculating eclass names twice,
 		# once here and another time when calling _init_overlay
-		self._header.set_eclasses ( set (
+		self._header.set_eclasses ( frozenset (
 			self._get_eclass_import_info ( only_eclass_names=True )
 		) )
 
@@ -56,44 +51,10 @@ class Overlay ( object ):
 
 			self._incremental_write_lock = threading.Lock()
 			self.scan()
-			self._init_overlay ( reimport_eclass=True, make_profiles_dir=True )
+			self._init_overlay ( reimport_eclass=True )
 
 	# --- end of __init__ (...) ---
 
-	def scan ( self, **kw ):
-		if os.path.isdir ( self.physical_location ):
-			for cat in self._scan_categories():
-				try:
-					cat.scan ( **kw )
-				except Exception as e:
-					self.logger.exception ( e )
-#		for package in sorted ( self.list_packages() ):
-#			print ( package )
-	# --- end of scan (...) ---
-
-	def list_packages ( self, for_deprules=True ):
-		for cat in self._categories.values():
-			for package in cat.list_packages ( for_deprules=True ):
-				yield package
-	# --- end of list_packages (...) ---
-
-	def list_rule_kwargs ( self ):
-		for cat in self._categories.values():
-			for kwargs in cat.list_packages ( for_deprules=True ):
-				yield kwargs
-	# --- end of list_rule_kwargs (...) ---
-
-	def has_dir ( self, _dir ):
-		return os.path.isdir ( self.physical_location + os.sep + _dir )
-	# --- end of has_category (...) ---
-
-	def _scan_categories ( self ):
-		for x in os.listdir ( self.physical_location ):
-			# FIXME could use a better check here
-			if '-' in x and self.has_dir ( x ):
-				yield self._get_category ( x )
-	# --- end of _scan_categories (...) ---
-
 	def _get_category ( self, category ):
 		"""Returns a reference to the given category. Creates it if necessary.
 
@@ -120,191 +81,6 @@ class Overlay ( object ):
 		return self._categories [category]
 	# --- end of _get_category (...) ---
 
-	def add ( self, package_info, category=None ):
-		"""Adds a package to this overlay.
-
-		arguments:
-		* package_info -- PackageInfo of the package to add
-		* category     -- category where the pkg should be put in, defaults to
-		                   self.default_category
-
-		returns: True if successfully added else False
-		"""
-		cat = self._get_category (
-			self.default_category if category is None else category
-		)
-		return cat.add ( package_info )
-	# --- end of add (...) ---
-
-	def show ( self, **show_kw ):
-		"""Presents the ebuilds/metadata stored in this overlay.
-
-		arguments:
-		* **show_kw -- keywords for package.PackageDir.show(...)
-
-		returns: None (implicit)
-		"""
-		if not self._header.eclasses: self._header.set_eclasses (
-			tuple ( self._get_eclass_import_info ( only_eclass_names=True ) )
-		)
-		for cat in self._categories.values():
-			cat.show ( **show_kw )
-	# --- end of show (...) ---
-
-	def write ( self, **write_kw ):
-		"""Writes the overlay to its physical location (filesystem), including
-		metadata and Manifest files.
-
-		arguments:
-		* **write_kw -- keywords for package.PackageDir.write(...)
-
-		returns: None (implicit)
-
-		raises: IOError
-
-		! TODO/FIXME/DOC: This is not thread-safe, it's expected to be called
-		when ebuild creation is done.
-		"""
-		raise Exception ( "^,^" )
-		# writing profiles/ here, rewriting categories/ later
-		self._init_overlay ( reimport_eclass=True, make_profiles_dir=True )
-
-		for cat in self._categories.values():
-			if not cat.empty():
-				util.dodir ( cat.physical_location )
-				cat.write ( **write_kw )
-
-		self._write_categories ( only_active=True )
-	# --- end of write (...) ---
-
-	def write_incremental ( self, **write_kw ):
-		"""Writes all ebuilds that have been added since the last
-		write_incremental call.
-		TODO:
-		* This could be useful to save some mem by removing already written
-		package infos.
-		* This has to be thread safe
-		"""
-		if not self._incremental_write_lock.acquire():
-			# another incremental write is running, drop this request
-			return
-
-		try:
-			util.dodir ( self.physical_location )
-			cats = tuple ( self._categories.values() )
-			for cat in cats:
-				util.dodir ( cat.physical_location )
-				cat.write_incremental ( **write_kw )
-		finally:
-			self._incremental_write_lock.release()
-	# --- end of write_incremental (...) ---
-
-	def finalize_write_incremental ( self ):
-		"""Writes metadata + Manifest for all packages."""
-		self._write_categories ( only_active=True )
-		for cat in self._categories.values():
-			cat.finalize_write_incremental()
-	# --- end of finalize_incremental (...) ---
-
-	def generate_metadata ( self, **metadata_kw ):
-		"""Tells the overlay's categories to create metadata.
-		You don't have to call this before write()/show() unless you want to use
-		special metadata options.
-
-		arguments:
-		* **metadata_kw -- keywords for package.PackageDir.generate_metadata(...)
-
-		returns: None (implicit)
-		"""
-		for cat in self._categories.values():
-			cat.generate_metadata ( **metadata_kw )
-	# --- end of generate_metadata (...) ---
-
-	def generate_manifest ( self, **manifest_kw ):
-		"""Generates Manifest files for all ebuilds in this overlay that exist
-		physically/in filesystem.
-		Manifest files are automatically created when calling write().
-
-		arguments:
-		* **manifest_kw -- see PackageDir.generate_manifest(...)
-
-		returns: None (implicit)
-		"""
-		for cat in self._categories.values():
-			cat.generate_manifest ( **manifest_kw )
-	# --- end of generate_manifest (...) ---
-
-	def _write_profiles_dir ( self, only_active_categories=True ):
-		"""Creates and updates the profiles/ dir.
-
-		arguments:
-		* only_active_categories -- if True: do not list categories without
-		                                      ebuilds in profiles/categories
-		"""
-		# profiles/
-		util.dodir ( self._profiles_dir )
-		self._write_repo_name()
-		self._write_categories ( only_active=only_active_categories )
-		self._write_usedesc()
-	# --- end of _write_profiles_dir (...) ---
-
-	def _write_profiles_file ( self, filename, to_write ):
-		"""Writes a file in profiles/.
-
-		arguments:
-		* filename -- name of the file to write (including file extension)
-		* to_write -- string to write (don't forget newline at the end)
-		"""
-		fh = None
-		try:
-			fh = open ( self._profiles_dir + os.sep + filename, 'w' )
-			if to_write:
-				# else touch file
-				fh.write ( to_write )
-		except IOError as e:
-			self.logger.exception ( e )
-			raise
-		finally:
-			if fh: fh.close()
-	# --- end of _write_profiles_file (...) ---
-
-	def _write_repo_name ( self ):
-		"""Writes profiles/repo_name."""
-		self._write_profiles_file ( 'repo_name', self.name + '\n' )
-	# --- end of _write_repo_name (...) ---
-
-	def _write_categories ( self, only_active=True ):
-		"""Writes profiles/categories.
-
-		arguments:
-		* only_active -- exclude categories without ebuilds
-		"""
-		cats = None
-		if only_active:
-			cats = [
-				name for name, category
-					in self._categories.items() if not category.empty()
-			]
-		else:
-			cats = list ( self._categories.keys() )
-
-		if cats:
-			self._write_profiles_file (
-				'categories',
-				'\n'.join ( cats ) + '\n'
-			)
-	# --- end of _write_categories (...) ---
-
-	def _write_usedesc ( self ):
-		"""Writes profiles/use.desc."""
-		use_desc = config.get (
-			'OVERLAY.use_desc',
-			fallback_value=DEFAULT_USE_DESC
-		)
-		if use_desc:
-			self._write_profiles_file ( 'use.desc', use_desc + '\n' )
-	# --- end of _write_usedesc (...) ---
-
 	def _get_eclass_import_info ( self, only_eclass_names=False ):
 		"""Yields eclass import information (eclass names and files).
 
@@ -363,7 +139,7 @@ class Overlay ( object ):
 				raise
 	# --- end of _import_eclass (...) ---
 
-	def _init_overlay ( self, reimport_eclass, make_profiles_dir ):
+	def _init_overlay ( self, reimport_eclass ):
 		"""Initializes the overlay at its physical/filesystem location.
 
 		arguments:
@@ -374,14 +150,58 @@ class Overlay ( object ):
 		raises:
 		* IOError
 		"""
+		def write_profiles_dir():
+			"""Creates and updates the profiles/ dir."""
+			def write_profiles_file ( filename, to_write ):
+				"""Writes a file in profiles/.
+
+				arguments:
+				* filename -- name of the file to write (including file extension)
+				* to_write -- string to write (don't forget newline at the end)
+				"""
+				fh = None
+				try:
+					fh = open ( self._profiles_dir + os.sep + filename, 'w' )
+					if to_write:
+						# else touch file
+						fh.write ( to_write )
+				except IOError as e:
+					self.logger.exception ( e )
+					raise
+				finally:
+					if fh: fh.close()
+			# --- end of write_profiles_file (...) ---
+
+			# always use the default category (+write it into profiles/categories)
+			self._get_category ( self.default_category )
+
+			# profiles/
+			util.dodir ( self._profiles_dir )
+
+			# profiless/repo_name
+			write_profiles_file ( 'repo_name', self.name + '\n' )
+
+			# profiles/categories
+			cats = '\n'.join ( self._categories.keys() )
+			if cats:
+				write_profiles_file ( 'categories', cats + '\n' )
+
+			# profiles/use.desc
+			use_desc = config.get (
+				'OVERLAY.use_desc',
+				fallback_value=DEFAULT_USE_DESC
+			)
+			if use_desc:
+				write_profiles_file ( 'use.desc', use_desc + '\n' )
+		# --- end of write_profiles_dir (...) ---
+
 		try:
 			# mkdir overlay root
 			util.dodir ( self.physical_location, mkdir_p=True )
 
 			self._import_eclass ( reimport_eclass )
 
-			if make_profiles_dir:
-				self._write_profiles_dir ( only_active_categories=False )
+			write_profiles_dir()
 
 		except IOError as e:
 
@@ -389,3 +209,153 @@ class Overlay ( object ):
 			self.logger.critical ( "^failed to init overlay" )
 			raise
 	# --- end of _init_overlay (...) ---
+
+	def add ( self, package_info, category=None ):
+		"""Adds a package to this overlay.
+
+		arguments:
+		* package_info -- PackageInfo of the package to add
+		* category     -- category where the pkg should be put in, defaults to
+		                   self.default_category
+
+		returns: True if successfully added else False
+		"""
+		cat = self._get_category (
+			self.default_category if category is None else category
+		)
+		return cat.add ( package_info )
+	# --- end of add (...) ---
+
+	def finalize_write_incremental ( self ):
+		"""Writes metadata + Manifest for all packages."""
+		for cat in self._categories.values():
+			cat.finalize_write_incremental()
+	# --- end of finalize_incremental (...) ---
+
+	def generate_manifest ( self, **manifest_kw ):
+		"""Generates Manifest files for all ebuilds in this overlay that exist
+		physically/in filesystem.
+		Manifest files are automatically created when calling write().
+
+		arguments:
+		* **manifest_kw -- see PackageDir.generate_manifest(...)
+
+		returns: None (implicit)
+		"""
+		for cat in self._categories.values():
+			cat.generate_manifest ( **manifest_kw )
+	# --- end of generate_manifest (...) ---
+
+	def generate_metadata ( self, **metadata_kw ):
+		"""Tells the overlay's categories to create metadata.
+		You don't have to call this before write()/show() unless you want to use
+		special metadata options.
+
+		arguments:
+		* **metadata_kw -- keywords for package.PackageDir.generate_metadata(...)
+
+		returns: None (implicit)
+		"""
+		for cat in self._categories.values():
+			cat.generate_metadata ( **metadata_kw )
+	# --- end of generate_metadata (...) ---
+
+	def has_dir ( self, _dir ):
+		return os.path.isdir ( self.physical_location + os.sep + _dir )
+	# --- end of has_category (...) ---
+
+	def keep_nth_latest ( self, *args, **kwargs ):
+		"""See package.py:PackageDir:keep_nth_latest."""
+		for cat in self._categories.values():
+			cat.keep_nth_latest ( *args, **kwargs )
+	# --- end of keep_nth_latest (...) ---
+
+	def list_packages ( self, for_deprules=True ):
+		for cat in self._categories.values():
+			for package in cat.list_packages ( for_deprules=True ):
+				yield package
+	# --- end of list_packages (...) ---
+
+	def list_rule_kwargs ( self ):
+		for cat in self._categories.values():
+			for kwargs in cat.list_packages ( for_deprules=True ):
+				yield kwargs
+	# --- end of list_rule_kwargs (...) ---
+
+	def scan ( self, **kw ):
+		def scan_categories():
+			for x in os.listdir ( self.physical_location ):
+				# FIXME could use a better check here
+				if '-' in x and self.has_dir ( x ):
+					yield self._get_category ( x )
+		# --- end of scan_categories (...) ---
+
+		if os.path.isdir ( self.physical_location ):
+			for cat in scan_categories():
+				try:
+					cat.scan ( **kw )
+				except Exception as e:
+					self.logger.exception ( e )
+	# --- end of scan (...) ---
+
+	def show ( self, **show_kw ):
+		"""Presents the ebuilds/metadata stored in this overlay.
+
+		arguments:
+		* **show_kw -- keywords for package.PackageDir.show(...)
+
+		returns: None (implicit)
+		"""
+		if not self._header.eclasses: self._header.set_eclasses (
+			tuple ( self._get_eclass_import_info ( only_eclass_names=True ) )
+		)
+		for cat in self._categories.values():
+			cat.show ( **show_kw )
+	# --- end of show (...) ---
+
+	def write ( self, **write_kw ):
+		"""Writes the overlay to its physical location (filesystem), including
+		metadata and Manifest files.
+
+		arguments:
+		* **write_kw -- keywords for package.PackageDir.write(...)
+
+		returns: None (implicit)
+
+		raises: IOError
+
+		! TODO/FIXME/DOC: This is not thread-safe, it's expected to be called
+		when ebuild creation is done.
+		"""
+		raise Exception ( "to be removed/replaced" )
+		# writing profiles/ here, rewriting categories/ later
+		self._init_overlay ( reimport_eclass=True )
+
+		for cat in self._categories.values():
+			if not cat.empty():
+				util.dodir ( cat.physical_location )
+				cat.write ( **write_kw )
+	# --- end of write (...) ---
+
+	def write_incremental ( self, **write_kw ):
+		"""Writes all ebuilds that have been modified since the last write call.
+		Note that there are currently two modes of incremental writing:
+		(a) per-PackageDir incremental writing triggered by the new_ebuild()
+		event method and (b) "batched" incremental writing (this method) which
+		writes all modified PackageDirs.
+		"""
+		# FIXME merge with write(), making incremental writing the only option
+		# FIXME finalize_write_incremental?
+		if not self._incremental_write_lock.acquire():
+			# another incremental write is running, drop this request
+			return
+
+		try:
+			util.dodir ( self.physical_location )
+			cats = tuple ( self._categories.values() )
+			for cat in cats:
+				util.dodir ( cat.physical_location )
+				cat.write_incremental ( **write_kw )
+		finally:
+			self._incremental_write_lock.release()
+	# --- end of write_incremental (...) ---