[gentoo-commits] proj/R_overlay:master commit in: roverlay/manifest/, roverlay/overlay/manifest/

[André Erdmann]

commit:     5af2290c757997b706406b4dc5a83967c0bc26f7
Author:     André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Fri Jul 20 16:10:47 2012 +0000
Commit:     André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Fri Jul 20 16:10:47 2012 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=5af2290c

ManifestCreation is broken!

* Just for a complete commit history

This commit tries to fix problems with multiple distdirs (e.g. when creating a
Manifest for ebuilds from both CRAN and BIOC) using PORTAGE_RO_DISTDIRS.
In short, this does not work ("src file not found").

	gelöscht:    roverlay/manifest/helpers.py
	umbenannt:    roverlay/manifest/__init__.py -> roverlay/overlay/manifest/__init__.py
	neue Datei:   roverlay/overlay/manifest/env.py
	neue Datei:   roverlay/overlay/manifest/helpers.py

---
 roverlay/manifest/helpers.py                |  221 ---------------------------
 roverlay/{ => overlay}/manifest/__init__.py |   19 ++-
 roverlay/overlay/manifest/env.py            |  112 ++++++++++++++
 roverlay/overlay/manifest/helpers.py        |   98 ++++++++++++
 4 files changed, 224 insertions(+), 226 deletions(-)

diff --git a/roverlay/manifest/helpers.py b/roverlay/manifest/helpers.py
deleted file mode 100644
index 47e1e20..0000000
--- a/roverlay/manifest/helpers.py
+++ /dev/null
@@ -1,221 +0,0 @@
-# R Overlay -- Manifest creation for ebuilds
-# Copyright 2006-2012 Gentoo Foundation
-# Distributed under the terms of the GNU General Public License v2
-
-
-# TODO (in future): could use portage api directly, namely
-#  '/usr/lib/portage/pym/portage/package/ebuild/doebuild.py'
-# instead of using '/usr/bin/ebuild'
-
-import os
-import re
-import copy
-import logging
-import subprocess
-import threading
-
-from roverlay import config, util
-
-class _ManifestCreation ( object ):
-	"""This is the base class for Manifest file creation."""
-
-	def __init__ ( self ):
-		self.logger = logging.getLogger ( 'ManifestCreation' )
-	# --- end of __init__ (...) ---
-
-	def create_for ( self, package_info ):
-		"""Creates a Manifest file for the ebuild of the given package_info."""
-		raise Exception ( "method stub" )
-	# --- end of create_for (...) ---
-
-	@classmethod
-	def do ( cls, package_info ):
-		"""Class access to Manifest creation."""
-		return cls().create_for ( package_info )
-	# --- end of do (...) ---
-
-
-class ExternalManifestCreation ( _ManifestCreation ):
-	"""This class implements Manifest creation using the low level ebuild
-	interface, ebuild(1), which is called in a filtered environment.
-	"""
-
-	envlock = threading.Lock()
-	manifest_env = None
-
-	@classmethod
-	def get_env ( cls, repo_dir ):
-		cls.envlock.acquire()
-		try:
-			if cls.manifest_env is None:
-				cls.manifest_env = ManifestEnv ( filter_env=True )
-			ret = cls.manifest_env [repo_dir]
-		finally:
-			cls.envlock.release()
-
-		return ret
-
-
-	def __init__ ( self ):
-		super ( ExternalManifestCreation, self ) . __init__ ()
-		# ebuild <ebuild_file> <target>, where target is:
-		self.ebuild_tgt   = config.get ( 'TOOLS.EBUILD.target', 'manifest' )
-		self.ebuild_prog  = config.get ( 'TOOLS.EBUILD.prog', '/usr/bin/ebuild' )
-
-	# --- end of __init__ (...) ---
-
-	def create_for ( self, package_info ):
-		"""See ManifestCreation.create_for.
-		Calls ebuild, returns True on success else False.
-
-		raises: *passes Exceptions from failed config lookups
-		"""
-
-		my_env = self.__class__.get_env ( package_info ['distdir'] )
-
-		ebuild_file = package_info ['ebuild_file']
-
-		ebuild_call = subprocess.Popen (
-			(
-				self.ebuild_prog,
-				ebuild_file,
-				self.ebuild_tgt
-			),
-			stdin=None,
-			stdout=subprocess.PIPE,
-			stderr=subprocess.PIPE,
-			env=my_env
-		)
-
-		output = ebuild_call.communicate()
-
-		# log stdout?
-		#for line in util.pipe_lines ( output [0] ):
-		#	LOGGER.debug ( line )
-		#for line in util.pipe_lines ( output [0] ): print ( line )
-
-		# log stderr
-		for line in util.pipe_lines ( output [1], use_filter=True ):
-			self.logger.warning ( line )
-
-		if ebuild_call.returncode == 0:
-			self.logger.debug ( "Manifest written." )
-			return True
-		else:
-			self.logger.error (
-				"Couldn't create Manifest for %s! Return code was %i."
-					% ( ebuild_file, ebuild_call.returncode )
-			)
-			return False
-	# --- end of create_for (...) ---
-
-
-class ManifestEnv ( object ):
-	"""per-repo environment container for Manifest creation using ebuild."""
-
-	def __init__ ( self, filter_env=True ):
-		"""Initializes a ManifestEnv.
-
-		arguments:
-		* filter_env -- if True: start with an empty env and copy vars
-										 from os.environ selectively
-							 else   : start with os.environ as env
-		"""
-		self.filter_env  = filter_env
-		self._manenv     = dict()
-		self.logger      = logging.getLogger ( 'ManifestEnv' )
-		self._common_env = None
-	# --- end of __init__ (...) ---
-
-	def get_env ( self, repo_dir ):
-		"""Returns an env dict for repo_dir.
-
-		arguments:
-		* repo_dir --
-		"""
-		if not repo_dir in self._manenv:
-			repo_env                = self._get_common_manifest_env()
-			repo_env ['DISTDIR']    = repo_dir
-			self._manenv [repo_dir] = repo_env
-
-		return self._manenv [repo_dir]
-	# --- end of get_env (...) ---
-
-	# x = ManifestEnv(); env = x [<sth>] etc.
-	__getitem__ = get_env
-	# ---
-
-	def _get_common_manifest_env ( self, noret=False ):
-		"""Creates an environment suitable for an
-		"ebuild <ebuild> digest|manifest" call (or uses an already existing env).
-		Returns a shallow copy of this env which can then be locally modified
-		(setting DISTDIR).
-
-		arguments:
-		* noret -- do not return copied env if True
-		"""
-
-		if self._common_env is None:
-
-			if self.filter_env:
-
-				# selectively import os.environ
-				# FIXME: keep EBUILD_DEFAULT_OPTS?
-				our_env = util.keepenv (
-					( 'PATH', '' ),
-					'LANG',
-					'PWD',
-					'EBUILD_DEFAULT_OPTS'
-				)
-			else:
-				# copy os.environ
-				our_env = dict ( os.environ )
-
-			# -- common env part
-
-			# set FEATURES
-			# * digest -- needed? (works without it)
-			# * assume-digests --
-			# * unknown-features-warn -- should FEATURES ever change
-			#
-			# * noauto -- should prevent ebuild from adding additional actions,
-			#   it still tries to download source packages, which is just wrong
-			#   here 'cause it is expected that the R package file exists when
-			#   calling this function, so FETCHCOMMAND/RESUMECOMMAND will be set
-			#   to /bin/true if possible.
-			#
-			# * distlocks -- disabled if FETCHCOMMAND/RESUMECOMMAND set to no-op
-			#
-			our_env ['FEATURES'] = \
-				"noauto digest assume-digests unknown-features-warn"
-
-			# try to prevent src fetching
-			fetch_nop = util.sysnop (
-				nop_returns_success=True,
-				format_str="%s \${DISTDIR} \${FILE} \${URI}"
-			)
-
-			if not fetch_nop is None:
-				self.logger.debug (
-					"%s disables/replaces FETCHCOMMAND,RESUMECOMMAND."
-					% fetch_nop [0]
-				)
-				our_env ['FETCHCOMMAND']  = fetch_nop [1]
-				our_env ['RESUMECOMMAND'] = fetch_nop [1]
-				our_env ['FEATURES']     += " -distlocks"
-
-
-
-			# set PORDIR_OVERLAY
-			# ! FIXME the overlay module does not enforce usage of this config key
-			our_env ['PORTDIR_OVERLAY'] = config.get_or_fail (
-				[ 'OVERLAY', 'dir' ]
-			)
-
-			self._common_env = our_env
-		# -- end if
-		if noret:
-			return None
-		else:
-			return copy.copy ( self._common_env )
-	# --- end of _get_common_manifest_env (...) ---

diff --git a/roverlay/manifest/__init__.py b/roverlay/overlay/manifest/__init__.py
similarity index 53%
rename from roverlay/manifest/__init__.py
rename to roverlay/overlay/manifest/__init__.py
index b95a855..9a49c7f 100644
--- a/roverlay/manifest/__init__.py
+++ b/roverlay/overlay/manifest/__init__.py
@@ -2,13 +2,18 @@
 # Copyright 2006-2012 Gentoo Foundation
 # Distributed under the terms of the GNU General Public License v2
 import logging
+import threading
 
-from roverlay.manifest import helpers
+from roverlay.overlay.manifest import helpers
 
-_MANIFEST_IMPLEMENTATION = helpers.ExternalManifestCreation
+_manifest_creation = helpers.ExternalManifestCreation()
+# ExternalManifestCreation does not support threads (-> multiprocesses)
+# for one directory/overlay
+_manifest_lock = threading.Lock()
 
 
-def create_manifest ( package_info, nofail=False ):
+
+def create_manifest ( package_info_list, nofail=False ):
 	"""Creates a Manifest for package_info, using the <<best>> implementation
 	available.
 
@@ -18,12 +23,16 @@ def create_manifest ( package_info, nofail=False ):
 	* package_info --
 	* nofail -- catch exceptions and return False
 	"""
+	ret = False
 	try:
-		return _MANIFEST_IMPLEMENTATION.do ( package_info )
+		_manifest_lock.acquire()
+		ret = _manifest_creation.create_for ( package_info_list )
 	except Exception as e:
 		logging.exception ( e )
 		if nofail:
-			return False
+			ret = False
 		else:
 			raise
+	finally:
+		_manifest_lock.release()
 # --- end of create_manifest (...) ---

diff --git a/roverlay/overlay/manifest/env.py b/roverlay/overlay/manifest/env.py
new file mode 100644
index 0000000..58548a4
--- /dev/null
+++ b/roverlay/overlay/manifest/env.py
@@ -0,0 +1,112 @@
+# R Overlay -- Manifest creation for ebuilds
+# Copyright 2006-2012 Gentoo Foundation
+# Distributed under the terms of the GNU General Public License v2
+
+import os
+import copy
+import logging
+
+from roverlay import util
+
+class ManifestEnv ( object ):
+	"""per-repo environment container for Manifest creation using ebuild."""
+
+	@classmethod
+	def get_new ( cls ):
+		return cls ( filter_env=True ).get_env (
+			distdir="", portage_ro_distdirs=""
+		)
+	# --- end of get_new (...) ---
+
+	def __init__ ( self, filter_env=True ):
+		"""Initializes a ManifestEnv.
+
+		arguments:
+		* filter_env -- if True: start with an empty env and copy vars
+										 from os.environ selectively
+							 else   : start with os.environ as env
+		"""
+		self.filter_env  = filter_env
+		self._manenv     = dict()
+		self.logger      = logging.getLogger ( 'ManifestEnv' )
+		self._common_env = None
+	# --- end of __init__ (...) ---
+
+	def get_env ( self, distdir, portage_ro_distdirs ):
+		"""Returns an env dict for repo_dir.
+
+		arguments:
+		* repo_dir --
+		"""
+		env                         = self._get_common_manifest_env()
+		env ['DISTDIR']             = distdir
+		env ['PORTAGE_RO_DISTDIRS'] = portage_ro_distdirs
+		return env
+	# --- end of get_env (...) ---
+
+	def _get_common_manifest_env ( self, noret=False ):
+		"""Creates an environment suitable for an
+		"ebuild <ebuild> digest|manifest" call (or uses an already existing env).
+		Returns a shallow copy of this env which can then be locally modified
+		(setting DISTDIR, PORTAGE_RO_DISTDIRS).
+
+		arguments:
+		* noret -- do not return copied env if True
+		"""
+
+		if self._common_env is None:
+
+			if self.filter_env:
+
+				# selectively import os.environ
+				# FIXME: keep EBUILD_DEFAULT_OPTS?
+				our_env = util.keepenv (
+					( 'PATH', '' ),
+					'LANG',
+					'PWD',
+					'EBUILD_DEFAULT_OPTS'
+				)
+			else:
+				# copy os.environ
+				our_env = dict ( os.environ )
+
+			# -- common env part
+
+			# set FEATURES
+			# * digest -- needed? (works without it)
+			# * assume-digests --
+			# * unknown-features-warn -- should FEATURES ever change
+			#
+			# * noauto -- should prevent ebuild from adding additional actions,
+			#   it still tries to download source packages, which is just wrong
+			#   here 'cause it is expected that the R package file exists when
+			#   calling this function, so FETCHCOMMAND/RESUMECOMMAND will be set
+			#   to /bin/true if possible.
+			#
+			# * distlocks -- disabled if FETCHCOMMAND/RESUMECOMMAND set to no-op
+			#
+			our_env ['FEATURES'] = \
+				"noauto digest assume-digests unknown-features-warn"
+
+			# try to prevent src fetching
+			fetch_nop = util.sysnop (
+				nop_returns_success=True,
+				format_str="%s \${DISTDIR} \${FILE} \${URI}"
+			)
+
+			if not fetch_nop is None:
+				self.logger.debug (
+					fetch_nop [0] + " disables/replaces FETCHCOMMAND,RESUMECOMMAND."
+				)
+				our_env ['FETCHCOMMAND']  = fetch_nop [1]
+				our_env ['RESUMECOMMAND'] = fetch_nop [1]
+				our_env ['FEATURES']     += " -distlocks"
+
+
+			self._common_env = our_env
+		# -- end if
+		if noret:
+			return None
+		else:
+			return copy.copy ( self._common_env )
+	# --- end of _get_common_manifest_env (...) ---

diff --git a/roverlay/overlay/manifest/helpers.py b/roverlay/overlay/manifest/helpers.py
new file mode 100644
index 0000000..168a119
--- /dev/null
+++ b/roverlay/overlay/manifest/helpers.py
@@ -0,0 +1,98 @@
+# R Overlay -- Manifest creation for ebuilds
+# Copyright 2006-2012 Gentoo Foundation
+# Distributed under the terms of the GNU General Public License v2
+
+# TODO (in future): could use portage api directly, namely
+#  '/usr/lib/portage/pym/portage/package/ebuild/doebuild.py'
+# instead of using '/usr/bin/ebuild'
+
+import os.path
+import copy
+import logging
+import subprocess
+
+
+from roverlay import config, util
+
+from roverlay.overlay.manifest.env import ManifestEnv
+
+class ExternalManifestCreation ( object ):
+	"""This class implements Manifest creation using the low level ebuild
+	interface, ebuild(1), which is called in a filtered environment.
+	"""
+	# NOTE:
+	# ebuild <ebuild> digest does not support multiprocesses for one overlay,
+
+	def __init__ ( self ):
+		self.logger       = logging.getLogger ( 'ManifestCreation' )
+		self.manifest_env = ManifestEnv.get_new()
+		self.ebuild_tgt   = config.get ( 'TOOLS.EBUILD.target', 'manifest' )
+		self.ebuild_prog  = config.get ( 'TOOLS.EBUILD.prog', '/usr/bin/ebuild' )
+
+		# set PORDIR_OVERLAY and DISTDIR
+		self.manifest_env ['PORTDIR_OVERLAY'] = config.get_or_fail (
+			'OVERLAY.dir'
+		)
+		# !! FIXME: tell the <others> that __tmp__ is a reserved directory
+		self.manifest_env ['DISTDIR'] = \
+		config.get_or_fail ( 'DISTFILES.ROOT' ) + os.path.sep + '__tmp__'
+
+
+	# --- end of __init__ (...) ---
+
+	def create_for ( self, package_info_list ):
+		"""See ManifestCreation.create_for.
+		Calls ebuild, returns True on success else False.
+
+		raises: *passes Exceptions from failed config lookups
+		"""
+		self.logger.critical (
+			"Manifest creation is broken! PORTAGE_RO_DISTDIRS does not work."
+		)
+		return False
+
+		distdirs    = ' '.join ( set (
+			p ['distdir'] for p in package_info_list
+		) )
+		ebuild_file = package_info_list [0] ['ebuild_file']
+
+
+		self.manifest_env ['PORTAGE_RO_DISTDIRS'] = distdirs
+
+		#util.dodir ( self.manifest_env ['DISTDIR'] )
+
+		ebuild_call = subprocess.Popen (
+			(
+				self.ebuild_prog,
+				ebuild_file,
+				self.ebuild_tgt
+			),
+			stdin=None,
+			stdout=subprocess.PIPE,
+			stderr=subprocess.PIPE,
+			env=self.manifest_env
+		)
+
+		output = ebuild_call.communicate()
+
+		# log stdout?
+		#for line in util.pipe_lines ( output [0] ):
+		#	LOGGER.debug ( line )
+		#for line in util.pipe_lines ( output [0] ): print ( line )
+
+		# log stderr
+		for line in util.pipe_lines ( output [1], use_filter=True ):
+			self.logger.warning ( line )
+
+		if ebuild_call.returncode == 0:
+			self.logger.debug ( "Manifest written." )
+			return True
+		else:
+			self.logger.error (
+				'Couldn\'t create Manifest for {ebuild}! '
+				'Return code was {ret}.'.format (
+					ebuild=ebuild_file, ret=ebuild_call.returncode
+				)
+			)
+			return False
+	# --- end of create_for (...) ---