[gentoo-commits] proj/R_overlay:master commit in: roverlay/, roverlay/overlay/

[André Erdmann]

commit:     bd0d4d01b9b32f82040f0782fab40b957f70eb94
Author:     André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Wed Jul 18 16:46:49 2012 +0000
Commit:     André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Wed Jul 18 16:46:49 2012 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=bd0d4d01

incremental overlay writing (at runtime)

Packages can now be written directly after adding them to the overlay.
Also removed unused code / code duplicates, e.g. show() and write() in
package.py.

	geändert:   roverlay/overlay/category.py
	geändert:   roverlay/overlay/creator.py
	geändert:   roverlay/overlay/package.py
	geändert:   roverlay/overlay/root.py
	geändert:   roverlay/packageinfo.py

---
 roverlay/overlay/category.py |   85 +++++++++++----
 roverlay/overlay/creator.py  |   21 ++--
 roverlay/overlay/package.py  |  252 ++++++++++++++++++++++++------------------
 roverlay/overlay/root.py     |  141 ++++++++++++++++--------
 roverlay/packageinfo.py      |   26 +++--
 5 files changed, 328 insertions(+), 197 deletions(-)

diff --git a/roverlay/overlay/category.py b/roverlay/overlay/category.py
index 34864b9..afa2456 100644
--- a/roverlay/overlay/category.py
+++ b/roverlay/overlay/category.py
@@ -33,6 +33,19 @@ class Category ( object ):
 		self.physical_location = directory
 	# --- end of __init__ (...) ---
 
+	def has ( self, subdir ):
+		return subdir in self._subdirs
+	# --- end of has (...) ---
+
+	def list_packages ( self, print_category=True ):
+		if print_category:
+			for package in self._subdirs.keys():
+				yield self.name + os.sep + package
+		else:
+			for package in self._subdirs.keys():
+				yield package
+	# --- end of list_packages (...) ---
+
 	def has_dir ( self, _dir ):
 		return os.path.isdir ( self.physical_location + os.sep + _dir )
 	# --- end of has_category (...) ---
@@ -43,10 +56,9 @@ class Category ( object ):
 				yield self._get_package_dir ( x )
 	# --- end of _scan_packages (...) ---
 
-	def scan ( self ):
+	def scan ( self, **kw ):
 		for pkg in self._scan_packages():
-			print ( pkg.name )
-			pkg.scan()
+			pkg.scan ( **kw )
 	# --- end of scan (...) ---
 
 	def empty ( self ):
@@ -64,8 +76,7 @@ class Category ( object ):
 					self._subdirs [pkg_name] = PackageDir (
 						pkg_name,
 						self.logger,
-						None if self.physical_location is None else \
-							os.path.join ( self.physical_location, pkg_name )
+						self.physical_location + os.sep + pkg_name
 					)
 			finally:
 				self._lock.release()
@@ -73,7 +84,7 @@ class Category ( object ):
 		return self._subdirs [pkg_name]
 	# --- end of _get_package_dir (...) ---
 
-	def add ( self, package_info ):
+	def add ( self, package_info, write_after_add=False, header=None ):
 		"""Adds a package to this category.
 
 		arguments:
@@ -81,9 +92,14 @@ class Category ( object ):
 
 		returns: success
 		"""
-		return self._get_package_dir (
-			package_info ['name']
-		).add ( package_info )
+		subdir = self._get_package_dir ( package_info ['name'] )
+		if subdir.add ( package_info ):
+			if write_after_add:
+				roverlay.util.dodir ( subdir.physical_location )
+				subdir.write_incremental ( default_header=header )
+			return True
+		else:
+			return False
 	# --- end of add (...) ---
 
 	def generate_metadata ( self, **metadata_kw ):
@@ -99,18 +115,18 @@ class Category ( object ):
 			package.generate_metadata ( **metadata_kw )
 	# --- end of generate_metadata (...) ---
 
-	def generate_manifest ( self, **manifest_kw ):
+	def write_manifest ( self, **manifest_kw ):
 		"""Generates Manifest files for all packages in this category.
 		Manifest files are automatically created when calling write().
 
 		arguments:
-		* **manifest_kw -- see PackageDir.generate_manifest(...)
+		* **manifest_kw -- see PackageDir.write_manifest(...)
 
 		returns: None (implicit)
 		"""
 		for package in self._subdirs.values():
-			package.generate_manifest ( **manifest_kw )
-	# --- end of generate_manifest (...) ---
+			package.write_manifest ( **manifest_kw )
+	# --- end of write_manifest (...) ---
 
 	def show ( self, **show_kw ):
 		"""Prints this category (its ebuild and metadata files).
@@ -122,10 +138,16 @@ class Category ( object ):
 	# --- end of show (...) ---
 
 	def _run_write_queue ( self, q, write_kw ):
+		"""Calls <package>.write for every <package> received from the queue.
+
+		arguments:
+		* q        -- queue
+		* write_kw --
+		"""
 		try:
 			while not q.empty():
 				pkg = q.get_nowait()
-				pkg.write ( **write_kw )
+				pkg.write ( write_manifest=False, **write_kw )
 
 		except queue.Empty:
 			pass
@@ -134,6 +156,21 @@ class Category ( object ):
 
 	# --- end of _run_write_queue (...) ---
 
+	def write_incremental ( self, **write_kw ):
+		"""Writes this category incrementally."""
+		try:
+			with self._lock:
+				# new package dirs could be added during overlay writing,
+				# so collect the list of package dirs before iterating over it
+				subdirs = tuple ( self._subdirs.values() )
+
+			for package in subdirs:
+				roverlay.util.dodir ( package.physical_location )
+				package.write_incremental ( **write_kw )
+		except Exception as e:
+			self.logger.exception ( e )
+	# --- end of write_incremental (...) ---
+
 	def write ( self, **write_kw ):
 		"""Writes this category to its filesystem location.
 
@@ -148,14 +185,15 @@ class Category ( object ):
 
 			# writing 1..self.__class__.WRITE_JOBCOUNT package dirs at once
 
-			write_queue = queue.Queue()
-			for package in self._subdirs.values():
-				if package.physical_location and not package.empty():
+			modified_packages = tuple (
+				p for p in self._subdirs.values() if p.modified
+			)
+			if len ( modified_packages ) > 0:
+				write_queue = queue.Queue()
+				for package in modified_packages:
 					roverlay.util.dodir ( package.physical_location )
 					write_queue.put_nowait ( package )
 
-
-			if not write_queue.empty():
 				workers = (
 					threading.Thread (
 						target=self._run_write_queue,
@@ -168,9 +206,14 @@ class Category ( object ):
 
 				if hasattr ( self, 'RERAISE_EXCEPTION' ):
 					raise self.RERAISE_EXCEPTION
+
+				# write manifest files
+				for package in modified_packages:
+					package.write_manifest()
+
 		else:
 			for package in self._subdirs.values():
-				if package.physical_location and not package.empty():
+				if package.modified:
 					roverlay.util.dodir ( package.physical_location )
-					package.write ( **write_kw )
+					package.write ( write_manifest=True, **write_kw )
 	# --- end of write (...) ---

diff --git a/roverlay/overlay/creator.py b/roverlay/overlay/creator.py
index 600a078..3a6e0ed 100644
--- a/roverlay/overlay/creator.py
+++ b/roverlay/overlay/creator.py
@@ -108,6 +108,7 @@ class OverlayCreator ( object ):
 		self._runlock   = threading.RLock()
 
 		self.can_write_overlay = OVERLAY_WRITE_ALLOWED
+		self.write_incremental = True
 
 		self.closed = False
 
@@ -223,7 +224,6 @@ class OverlayCreator ( object ):
 		self.package_added.inc()
 	# --- end of add_packages (...) ---
 
-
 	def write_overlay ( self, incremental=False ):
 		"""Writes the overlay.
 
@@ -232,12 +232,7 @@ class OverlayCreator ( object ):
 		"""
 		if self.can_write_overlay:
 			start = time.time()
-			if incremental:
-				# this will fail 'cause not implemented
-				self.overlay.write_incremental()
-			else:
-				self.overlay.write()
-
+			self.overlay.write()
 			self._timestamp ( "overlay written", start )
 		else:
 			self.logger.warning ( "Not allowed to write overlay!" )
@@ -394,13 +389,17 @@ class OverlayCreator ( object ):
 		#  * request an incremental write to save memory etc.
 
 		# if <>:
-		if package_info ['ebuild'] is None:
-			self.create_fail.inc()
-		else:
+		if package_info ['ebuild'] is not None:
 			self.create_success.inc()
-			if self.overlay.add ( package_info ):
+			if self.overlay.add (
+				package_info,
+				write_after_add=self.write_incremental and self.can_write_overlay
+			):
 				self.overlay_added.inc()
 
+		else:
+			self.create_fail.inc()
+
 	# --- end of _add_to_overlay (...) ---
 
 	def _get_worker ( self, start_now=False, use_threads=True ):

diff --git a/roverlay/overlay/package.py b/roverlay/overlay/package.py
index ceee371..c0b4294 100644
--- a/roverlay/overlay/package.py
+++ b/roverlay/overlay/package.py
@@ -6,9 +6,9 @@ import threading
 import os
 import sys
 
-from roverlay.metadata    import MetadataJob
-from roverlay             import manifest
-from roverlay.packageinfo import PackageInfo
+from roverlay                  import manifest
+from roverlay.packageinfo      import PackageInfo
+from roverlay.overlay.metadata import MetadataJob
 
 SUPPRESS_EXCEPTIONS = True
 EBUILD_SUFFIX = '.ebuild'
@@ -33,10 +33,16 @@ class PackageDir ( object ):
 		self._metadata         = None
 		self.physical_location = directory
 
+		self._package_for_manifest = None
+
 		# used to track changes for this package dir
 		self.modified          = False
 	# --- end of __init__ (...) ---
 
+	def list_versions ( self ):
+		return self._packages.keys()
+	# --- end of list_versions (...) ---
+
 	def has_manifest ( self ):
 		return os.path.isfile (
 			self.physical_location + os.sep + 'Manifest'
@@ -56,6 +62,7 @@ class PackageDir ( object ):
 	# --- end of get_ebuilds (...) ---
 
 	def _scan_ebuilds ( self ):
+		"""Searches for ebuilds in self.physical_location."""
 		elen = len ( EBUILD_SUFFIX )
 
 		for f in os.listdir ( self.physical_location ):
@@ -79,11 +86,13 @@ class PackageDir ( object ):
 
 	# --- end of _scan_ebuilds (...) ---
 
-	def scan ( self ):
+	def scan ( self, **kw ):
+		"""Scans the filesystem location of this package for existing
+		ebuilds and adds them.
+		"""
 		for pvr in self._scan_ebuilds():
 			if pvr not in self._packages:
-				print ( pvr )
-				p = PackageInfo ( physical=True, pvr=pvr )
+				p = PackageInfo ( physical_only=True, pvr=pvr )
 				self._packages [ p ['ebuild_verstr'] ] = p
 	# --- end of scan (...) ---
 
@@ -92,104 +101,124 @@ class PackageDir ( object ):
 		return len ( self._packages ) == 0
 	# --- end of empty (...) ---
 
-	def _get_metadata_filepath ( self ):
-		"""Returns the path to the metadata file."""
-		return os.path.join (
-			'??' if self.physical_location is None else self.physical_location,
-			self._metadata.filename
-		)
-	# --- end of _get_metadata_filepath (...) ---
-
 	def _get_ebuild_filepath ( self, pvr ):
 		"""Returns the path to the ebuild file.
 
 		arguments:
 		* pvr -- version number with the revision (${PVR} in ebuilds)
 		"""
-		filename = "%s-%s.ebuild" % ( self.name, pvr )
-		return os.path.join (
-			'??' if self.physical_location is None else self.physical_location,
-			filename
+		return "{root}{sep}{PN}-{PVR}{EBUILD_SUFFIX}".format (
+			root=self.physical_location, sep=os.sep,
+			PN=self.name, PVR=pvr, EBUILD_SUFFIX=EBUILD_SUFFIX
 		)
 	# --- end of _get_ebuild_filepath (...) ---
 
-	def write ( self, default_header=None ):
-		"""Writes this directory to its (existent!) filesystem location.
-
-		returns: None (implicit)
-
-		raises:
-		* Exception if no directory assigned
-		* IOError
-		"""
-		if self.physical_location is None:
-			raise Exception ( "cannot write - no directory assigned!" )
+	def write_incremental ( self, default_header ):
+		self.write_ebuilds ( header=default_header, overwrite=False )
+	# --- end of write_incremental (...) ---
 
-		self._lock.acquire()
+	def write_metadata ( self, shared_fh=None ):
+		"""Writes metadata for this package."""
 		try:
-			self._regen_metadata()
 
-			# mkdir not required here, overlay.Category does this
+			self._regen_metadata()
 
-			# write ebuilds
-			for ver, p_info in self._packages.items():
-				fh = None
-				try:
-					efile  = self._get_ebuild_filepath ( ver )
+			if shared_fh is None:
+				fh = open (
+					self.physical_location + os.sep + self._metadata.filename, 'w'
+				)
+			else:
+				fh = shared_fh
 
-					ebuild = p_info ['ebuild']
+			self._metadata.write ( fh )
 
-					fh = open ( efile, 'w' )
-					fh.write ( default_header )
-					fh.write ( '\n\n' )
-					fh.write ( str ( ebuild ) )
-					fh.write ( '\n' )
+		except IOError as e:
 
-					if fh: fh.close()
+			self.logger.error (
+				"Failed to write metadata file {}.".format ( mfile )
+			)
+			self.logger.exception ( e )
 
-					# adjust owner/perm? TODO
-					# chmod 0644 or 0444
-					# chown 250.250
+		finally:
+			if shared_fh is None and 'fh' in locals() and fh:
+				fh.close()
+	# --- end of write_metadata (...) ---
 
-					# this marks the package as 'written to fs'
-					p_info.set_writeable()
-					p_info ['ebuild_file'] = efile
-					p_info.set_readonly()
+	def write_ebuild ( self, efile, ebuild, header, shared_fh=None ):
+		"""Writes an ebuild.
 
-					self.logger.info ( "Wrote ebuild %s." % efile )
-				except IOError as e:
-					if fh: fh.close()
-					self.logger.error ( "Couldn't write ebuild %s." % efile )
-					self.logger.exception ( e )
+		arguments:
+		* efile     -- file to write
+		* ebuild    -- ebuild object to write (has to have a __str__ method)
+		* header    -- ebuild header to write (^)
+		* shared_fh -- optional, see write_ebuilds()
+		"""
+		_success = False
+		try:
+			fh = open ( efile, 'w' ) if shared_fh is None else shared_fh
+			if header is not None:
+				fh.write ( str ( header ) )
+				fh.write ( '\n\n' )
+			fh.write ( str ( ebuild ) )
+			fh.write ( '\n' )
+
+			# adjust owner/perm? TODO
+			#if shared_fh is None:
+			#	chmod 0644 or 0444
+			#	chown 250.250
+			_success = True
+		except IOError as e:
+			self.logger.exception ( e )
+		finally:
+			if shared_fh is None and 'fh' in locals() and fh:
+				fh.close()
 
-			# write metadata
-			fh = None
-			try:
-				mfile = self._get_metadata_filepath()
+		return _success
+	# --- end of write_ebuild (...) ---
 
-				fh    = open ( mfile, 'w' )
-				self._metadata.write ( fh )
-				if fh: fh.close()
+	def write_ebuilds ( self, header, shared_fh=None, overwrite=True ):
+		"""Writes all ebuilds.
 
-			except IOError as e:
-				if fh: fh.close()
-				self.logger.error ( "Failed to write metadata at %s." % mfile )
-				self.logger.exception ( e )
+		arguments:
+		* header    -- ebuild header
+		* shared_fh -- if set and not None: don't use own file handles (i.e.
+		               write files), write everything into shared_fh
+		"""
+		for ver, p_info in self._packages.items():
+			if not p_info ['physical_only'] and p_info ['ebuild']:
+				efile = self._get_ebuild_filepath ( ver )
+
+				if not overwrite and efile == p_info ['ebuild_file']:
+					print ( efile + " exists, skipping write()." )
+
+
+				elif self.write_ebuild (
+					efile, p_info ['ebuild'], header, shared_fh
+				):
+					if shared_fh is None:
+						# this marks the package as 'written to fs'
+						p_info.update_now (
+							ebuild_file=efile,
+							remove_auto='ebuild_written'
+						)
 
-			# FIXME cannot write manifest here when using threads,
-			# multiprocessed manifest writing is not supported, gives errors like
-			# 'OSError: [Errno 17] File exists: '/var/cache/edb/dep/tmp/<overlay>'
-			#self.generate_manifest()
+						self._package_for_manifest = p_info
 
-		finally:
-			self._lock.release()
-	# --- end of write (...) ---
+						self.logger.info ( "Wrote ebuild {}.".format ( efile ) )
+				else:
+					self.logger.error (
+						"Couldn't write ebuild {}.".format ( efile )
+					)
+	# --- end of write_ebuilds (...) ---
 
-	def show ( self, stream=sys.stderr, default_header=None ):
-		"""Prints this dir (the ebuilds and the metadata) into a stream.
+	def write ( self,
+		default_header=None, write_manifest=True, shared_fh=None
+	):
+		"""Writes this directory to its (existent!) filesystem location.
 
 		arguments:
-		* stream -- stream to use, defaults to sys.stderr
+		* default_header    -- ebuild header to write
+		* write_manifest -- if set and False: don't write the Manifest file
 
 		returns: None (implicit)
 
@@ -198,30 +227,35 @@ class PackageDir ( object ):
 		"""
 		self._lock.acquire()
 		try:
-			self._regen_metadata()
+			# mkdir not required here, overlay.Category does this
 
+			# write ebuilds
+			self.write_ebuilds ( header=default_header, shared_fh=shared_fh )
 
-			for ver, p_info in self._packages.items():
-				efile  = self._get_ebuild_filepath ( ver )
-				ebuild = p_info ['ebuild']
+			# write metadata
+			self.write_metadata ( shared_fh=shared_fh )
 
-				stream.write ( "[BEGIN ebuild %s]\n" % efile )
+			if write_manifest and shared_fh is not None:
+				self.write_manifest()
 
-				stream.write ( default_header )
-				stream.write ( '\n\n' )
-				stream.write ( str ( ebuild ) )
-				stream.write ( '\n' )
+		finally:
+			self._lock.release()
+	# --- end of write (...) ---
 
-				stream.write ( "[END ebuild %s]\n" % efile )
+	def show ( self, stream=sys.stderr, default_header=None ):
+		"""Prints this dir (the ebuilds and the metadata) into a stream.
 
-			mfile = self._get_metadata_filepath()
+		arguments:
+		* stream -- stream to use, defaults to sys.stderr
 
-			stream.write ( "[BEGIN %s]\n" % mfile )
-			self._metadata.write ( stream )
-			stream.write ( "[END %s]\n" % mfile )
+		returns: None (implicit)
 
-		finally:
-			self._lock.release()
+		raises:
+		* IOError
+		"""
+		self.write (
+			default_header=default_header, shared_fh=stream, write_manifest=False
+		)
 	# --- end of show (...) ---
 
 	def _latest_package ( self, pkg_filter=None, use_lock=False ):
@@ -267,7 +301,7 @@ class PackageDir ( object ):
 			if shortver in self._packages:
 				self.logger.info (
 					"'{PN}-{PVR}.ebuild' already exists, cannot add it!".format (
-						{ 'PN' : self.name, 'PVR' : shortver }
+						PN=self.name, PVR=shortver
 					)
 				)
 				return True
@@ -282,6 +316,7 @@ class PackageDir ( object ):
 				self._lock.acquire()
 				if not already_exists():
 					self._packages [shortver] = package_info
+					self.modified = True
 					_success = True
 			finally:
 				self._lock.release()
@@ -310,10 +345,9 @@ class PackageDir ( object ):
 		* use_old_metadata -- TODO in metadata
 		"""
 		if use_old_metadata or use_all_packages:
-				raise Exception ( "using >1 package for metadata.xml is TODO!" )
+			raise Exception ( "using >1 package for metadata.xml is TODO!" )
 
-		if skip_if_existent and not self._metadata is None:
-			return
+		if skip_if_existent and not self._metadata is None: return
 
 		self._lock.acquire()
 		try:
@@ -332,19 +366,16 @@ class PackageDir ( object ):
 			self._lock.release()
 	# --- end of generate_metadata (...) ---
 
-	def generate_manifest ( self ):
-		"""Generates the Manifest file for this package.
+	def write_manifest ( self ):
+		"""Generates and writes the Manifest file for this package.
 
 		expects: called in self.write(), after writing metadata/ebuilds
 
 		returns: None (implicit)
 
 		raises:
-		* Exception if not physical
 		* Exception if no ebuild exists
 		"""
-		if self.physical_location is None:
-			raise Exception ( "no directory assigned." )
 
 		# it should be sufficient to call create_manifest for one ebuild,
 		#  choosing the latest one here that exists in self.physical_location.
@@ -352,12 +383,12 @@ class PackageDir ( object ):
 		# metadata.xml's full path cannot be used for manifest creation here
 		#  'cause DISTDIR would be unknown
 		#
-		pkg_info_for_manifest = self._latest_package (
-			pkg_filter=lambda pkg : not pkg ['ebuild_file'] is None,
-			use_lock=True
-		)
+#		pkg_info_for_manifest = self._latest_package (
+#			pkg_filter=lambda pkg : pkg ['ebuild_file'] is not None,
+#			use_lock=True
+#		)
 
-		if pkg_info_for_manifest is None:
+		if self._package_for_manifest is None:
 			# ? FIXME
 			raise Exception (
 				"No ebuild written so far! I really don't know what do to!"
@@ -365,6 +396,9 @@ class PackageDir ( object ):
 		else:
 			# TODO: manifest creation interface is single threaded,
 			#        may want to 'fix' this later
-			manifest.create_manifest ( pkg_info_for_manifest, nofail=False )
+			manifest.create_manifest (
+				self._package_for_manifest, nofail=False,
+				#ebuild_file=...
+			)
 
-	# --- end of generate_manifest (...) ---
+	# --- end of write_manifest (...) ---

diff --git a/roverlay/overlay/root.py b/roverlay/overlay/root.py
index 77036cb..26be02a 100644
--- a/roverlay/overlay/root.py
+++ b/roverlay/overlay/root.py
@@ -16,6 +16,45 @@ DEFAULT_USE_DESC = '\n'.join ( [
 	'R_suggests - install recommended packages'
 ] )
 
+class EbuildHeader ( object ):
+	def __init__ ( self, default_header ):
+		self.default_header = default_header
+		self.eclasses       = ()
+
+		self._cached_header = None
+	# --- end of __init__ (...) ---
+
+	def set_eclasses ( self, eclass_names ):
+		self.eclasses = eclass_names
+	# --- end of set_eclasses (...) ---
+
+	def get ( self, use_cached=True ):
+		if self._cached_header is None or not use_cached:
+			self._cached_header = self._make()
+		return self._cached_header
+	# --- end of get (...) ---
+
+	def _make ( self ):
+		if self.eclasses:
+			inherit = 'inherit ' + ' '.join ( self.eclasses )
+		else:
+			inherit = None
+
+		# header and inherit is expected and therefore the first condition here
+		if inherit and self.default_header:
+			return self.default_header + '\n' + inherit
+
+		elif inherit:
+			return inherit
+
+		elif self.default_header:
+			return self.default_header
+
+		else:
+			return None
+	# --- end of _make (...) ---
+
+
 class Overlay ( object ):
 
 	def __init__ (
@@ -24,6 +63,10 @@ class Overlay ( object ):
 		default_category, eclass_files,
 		ebuild_header
 	):
+		if directory is None:
+			raise Exception (
+				"support for overlays without filesystem location has been dropped"
+			)
 
 		self.name              = name
 		self.logger            = logger.getChild ( 'overlay' )
@@ -31,27 +74,43 @@ class Overlay ( object ):
 		self.default_category  = default_category
 		self.eclass_files      = eclass_files
 
-		self.eclass_names      = None
 		self._profiles_dir     = self.physical_location + os.sep + 'profiles'
 		self._catlock          = threading.Lock()
 		self._categories       = dict()
-		self._default_header   = ebuild_header
+		self._header           = EbuildHeader ( ebuild_header )
+		self._get_header       = self._header.get
+
+		# fixme or ignore: calculating eclass names twice,
+		# once here and another time when calling _init_overlay
+		self._header.set_eclasses ( tuple (
+			self._get_eclass_import_info ( only_eclass_names=True )
+		) )
+
+		self._incremental_write_lock = threading.Lock()
+
 
 
 		#self.scan()
 		#raise Exception ( "^" )
 	# --- end of __init__ (...) ---
 
-	def scan ( self ):
+	def scan ( self, **kw ):
 		if os.path.isdir ( self.physical_location ):
 			for cat in self._scan_categories():
 				try:
-					print ( cat.name )
-					cat.scan()
+					cat.scan ( **kw )
 				except Exception as e:
 					self.logger.exception ( e )
+#		for package in sorted ( self.list_packages() ):
+#			print ( package )
 	# --- end of scan (...) ---
 
+	def list_packages ( self ):
+		for cat in self._categories.values():
+			for package in cat.list_packages():
+				yield package
+	# --- end of list_packages (...) ---
+
 	def has_dir ( self, _dir ):
 		return os.path.isdir ( self.physical_location + os.sep + _dir )
 	# --- end of has_category (...) ---
@@ -76,8 +135,7 @@ class Overlay ( object ):
 					self._categories [category] = Category (
 						category,
 						self.logger,
-						None if self.physical_location is None else \
-							self.physical_location + os.sep + category
+						self.physical_location + os.sep + category
 					)
 			finally:
 				self._catlock.release()
@@ -85,7 +143,7 @@ class Overlay ( object ):
 		return self._categories [category]
 	# --- end of _get_category (...) ---
 
-	def add ( self, package_info, category=None ):
+	def add ( self, package_info, write_after_add=False, category=None ):
 		"""Adds a package to this overlay.
 
 		arguments:
@@ -95,9 +153,17 @@ class Overlay ( object ):
 
 		returns: True if successfully added else False
 		"""
-		return self._get_category (
+		cat = self._get_category (
 			self.default_category if category is None else category
-		) . add ( package_info )
+		)
+
+		if write_after_add:
+			util.dodir ( cat.physical_location, mkdir_p=True )
+			return cat.add (
+				package_info, write_after_add=True, header = self._get_header()
+			)
+		else:
+			return cat.add ( package_info, write_after_add=False )
 	# --- end of add (...) ---
 
 	def show ( self, **show_kw ):
@@ -108,6 +174,9 @@ class Overlay ( object ):
 
 		returns: None (implicit)
 		"""
+		if not self._header.eclasses: self._header.set_eclasses (
+			tuple ( self._get_eclass_import_info ( only_eclass_names=True ) )
+		)
 		for cat in self._categories.values():
 			cat.show ( default_header=self._get_header() )
 	# --- end of show (...) ---
@@ -130,7 +199,7 @@ class Overlay ( object ):
 		self._init_overlay ( reimport_eclass=True, make_profiles_dir=True )
 
 		for cat in self._categories.values():
-			if cat.physical_location and not cat.empty():
+			if not cat.empty():
 				util.dodir ( cat.physical_location )
 				cat.write ( default_header=self._get_header() )
 
@@ -145,7 +214,18 @@ class Overlay ( object ):
 		package infos.
 		* This has to be thread safe
 		"""
-		raise Exception ( "method stub" )
+		if not self._incremental_write_lock.acquire():
+			# another incremental write is running, drop this request
+			return
+
+		try:
+			util.dodir ( self.physical_location )
+			cats = tuple ( self._categories.values() )
+			for cat in cats:
+				util.dodir ( cat.physical_location )
+				cat.write_incremental ( default_header=self._get_header() )
+		finally:
+			self._incremental_write_lock.release()
 	# --- end of write_incremental (...) ---
 
 	def generate_metadata ( self, **metadata_kw ):
@@ -199,7 +279,7 @@ class Overlay ( object ):
 		"""
 		fh = None
 		try:
-			fh = open ( os.path.join ( self._profiles_dir, filename ), 'w' )
+			fh = open ( self._profiles_dir + os.sep + filename, 'w' )
 			if to_write:
 				# else touch file
 				fh.write ( to_write )
@@ -286,19 +366,19 @@ class Overlay ( object ):
 
 		if self.eclass_files:
 			# import eclass files
-			eclass_dir = os.path.join ( self.physical_location, 'eclass' )
+			eclass_dir = self.physical_location + os.sep +  'eclass'
 			try:
 				eclass_names = list()
 				util.dodir ( eclass_dir )
 
 				for destname, eclass in self._get_eclass_import_info ( False ):
-					dest = os.path.join ( eclass_dir, destname + '.eclass' )
+					dest = eclass_dir + os.sep +  destname + '.eclass'
 					if reimport_eclass or not os.path.isfile ( dest ):
 						shutil.copyfile ( eclass, dest )
 
 					eclass_names.append ( destname )
 
-				self.eclass_names = frozenset ( eclass_names )
+				self._header.set_eclasses ( frozenset ( eclass_names ) )
 
 			except Exception as e:
 				self.logger.critical ( "Cannot import eclass files!" )
@@ -314,12 +394,8 @@ class Overlay ( object ):
 		* make_profiles_dir -- if True: create the profiles/ dir now
 
 		raises:
-		* Exception if no physical location assigned
 		* IOError
 		"""
-		if self.physical_location is None:
-			raise Exception ( "no directory assigned." )
-
 		try:
 			# mkdir overlay root
 			util.dodir ( self.physical_location, mkdir_p=True )
@@ -335,28 +411,3 @@ class Overlay ( object ):
 			self.logger.critical ( "^failed to init overlay" )
 			raise
 	# --- end of _init_overlay (...) ---
-
-	def _get_header ( self ):
-		"""Returns the ebuild header (including inherit <eclasses>)."""
-		if self.eclass_names is None:
-				# writing is possibly disabled since eclass files have not been
-				# imported (or show() used before write())
-			inherit = ' '.join ( self._get_eclass_import_info ( True ) )
-		else:
-			inherit = ' '.join ( self.eclass_names )
-
-		inherit = "inherit " + inherit if inherit else None
-
-		# header and inherit is expected and therefore the first condition here
-		if inherit and self._default_header:
-			return '\n'.join (( self._default_header, '', inherit ))
-
-		elif inherit:
-			return inherit
-
-		elif self._default_header:
-			return self._default_header
-
-		else:
-			return None
-	# --- end of _get_header (...) ---

diff --git a/roverlay/packageinfo.py b/roverlay/packageinfo.py
index 96cdb37..a2910c5 100644
--- a/roverlay/packageinfo.py
+++ b/roverlay/packageinfo.py
@@ -157,8 +157,8 @@ class PackageInfo ( object ):
 			# 'has_suggests' not in self._info -> assume False
 			return False
 
-		elif key_low == 'physical':
-			# 'physical' not in self._info -> assume False
+		elif key_low == 'physical_only':
+			# 'physical_only' not in self._info -> assume False
 			return False
 
 		elif key_low == 'src_uri':
@@ -240,8 +240,11 @@ class PackageInfo ( object ):
 			elif key == 'ebuild':
 				self ['ebuild'] = value
 
-			elif key == 'physical':
-				self ['physical'] = value
+			elif key == 'ebuild_file':
+				self ['ebuild_file'] = value
+
+			elif key == 'physical_only':
+				self ['physical_only'] = value
 
 			elif key == 'pvr':
 				self._use_pvr ( value )
@@ -266,7 +269,7 @@ class PackageInfo ( object ):
 				self._remove_auto ( value )
 
 			else:
-				LOGGER.error ( "unknown info key %s!" % key )
+				LOGGER.error ( "unknown info key {}!".format ( key ) )
 
 		self._update_lock.release()
 	# --- end of update (**kw) ---
@@ -288,8 +291,8 @@ class PackageInfo ( object ):
 
 		if not sepa:
 			# file name unexpected, tarball extraction will (probably) fail
-			LOGGER.error    ( "unexpected file name '%s'." % filename )
-			raise Exception ( "cannot use file '%s'." % filename )
+			LOGGER.error    ( "unexpected file name {!r}.".format ( filename ) )
+			raise Exception ( "cannot use file {!r}.".format ( filename ) )
 			return
 
 		version_str = PackageInfo.EBUILDVER_REGEX.sub ( '.', package_version )
@@ -338,7 +341,7 @@ class PackageInfo ( object ):
 		after entering status 'ebuild_status' (like ebuild in overlay and
 		written -> don't need the ebuild string etc.)
 		"""
-		raise Exception ( "method stub" )
+		print ( "PackageInfo._remove_auto: method stub, request ignored." )
 	# --- end of _remove_auto (...) ---
 
 	def _use_filepath ( self, _filepath ):
@@ -356,7 +359,8 @@ class PackageInfo ( object ):
 	# --- end of _use_filepath (...) ---
 
 	def __str__ ( self ):
-		return "<PackageInfo for %s>" % self.get (
-			'package_file', fallback_value='[unknown file]', do_fallback=True
-		)
+		return "<PackageInfo for {pkg}>".format (
+			pkg=self.get (
+				'package_file', fallback_value='[unknown file]', do_fallback=True
+		) )
 	# --- end of __str__ (...) ---