[gentoo-commits] portage r13462 - in main/branches/2.1.6: man pym/_emerge pym/portage pym/portage/cache pym/portage/dbapi

[Zac Medico (zmedico)]

Author: zmedico
Date: 2009-04-30 06:58:28 +0000 (Thu, 30 Apr 2009)
New Revision: 13462

Modified:
   main/branches/2.1.6/man/portage.5
   main/branches/2.1.6/pym/_emerge/__init__.py
   main/branches/2.1.6/pym/portage/__init__.py
   main/branches/2.1.6/pym/portage/cache/metadata.py
   main/branches/2.1.6/pym/portage/cache/util.py
   main/branches/2.1.6/pym/portage/dbapi/porttree.py
   main/branches/2.1.6/pym/portage/eclass_cache.py
Log:
Add portdbapi support for a metadata/layout.conf file which
specifies information about the repository layout. Currently,
only a single "masters" attribute is supported, which is used
to specify names of repositories which satisfy dependencies
on eclasses and/or ebuilds. Each repository name should
correspond the value of a repo_name entry from one of the
repositories that is configured via the PORTDIR or
PORTDIR_OVERLAY variables.

Since layout.conf is now used to control eclass inheritance,
it is now safer to use overlays which contain forked eclasses
have names identical to those from the main tree. Such
eclasses will only apply to their containing repository and
any other repositories which reference their containing
repository via layout.conf. This solves bug #124041 by
containing eclass overrides so that they don't necessarily
apply to all ebuilds.

Thanks to Alistair Bush <ali_bush@g.o> for his initial patch
for layout.conf support in repoman (will be merged later).
See the "QA Overlay Layout support" thread on the gentoo-dev
mailing list for more information:

http://archives.gentoo.org/gentoo-dev/msg_33c61550b4ed2b7b25dd5a4110e1ec81.xml
(trunk r13291)

Modified: main/branches/2.1.6/man/portage.5
===================================================================
--- main/branches/2.1.6/man/portage.5	2009-04-30 06:58:14 UTC (rev 13461)
+++ main/branches/2.1.6/man/portage.5	2009-04-30 06:58:28 UTC (rev 13462)
@@ -59,6 +59,11 @@
 .BR /etc/portage/profile/
 site-specific overrides of \fB/etc/make.profile/\fR
 .TP
+.BR /usr/portage/metadata/
+.nf
+layout.conf
+.fi
+.TP
 .BR /usr/portage/profiles/
 .nf
 arch.list
@@ -542,6 +547,23 @@
 .fi
 .RE
 .TP
+.BR /usr/portage/metadata/
+.RS
+.TP
+.BR layout.conf
+Specifies information about the repository layout. Currently, only a single
+"masters" attribute is supported, which is used to specify names of
+repositories which satisfy dependencies on eclasses and/or ebuilds. Each
+repository name should correspond the value of a \fBrepo_name\fR entry
+from one of the repositories that is configured via the \fBPORTDIR\fR or
+\fBPORTDIR_OVERLAY\fR variables (see \fBmake.conf\fR(5)).
+
+.I Example:
+.nf
+masters = gentoo java-overlay
+.fi
+.RE
+.TP
 .BR /usr/portage/profiles/
 Global Gentoo settings that are controlled by the developers.  To override 
 these settings, you can use the files in \fB/etc/portage/\fR.

Modified: main/branches/2.1.6/pym/_emerge/__init__.py
===================================================================
--- main/branches/2.1.6/pym/_emerge/__init__.py	2009-04-30 06:58:14 UTC (rev 13461)
+++ main/branches/2.1.6/pym/_emerge/__init__.py	2009-04-30 06:58:28 UTC (rev 13462)
@@ -14977,41 +14977,12 @@
 		#repo_name_check(trees)
 		config_protect_check(trees)
 
-	eclasses_overridden = {}
 	for mytrees in trees.itervalues():
 		mydb = mytrees["porttree"].dbapi
 		# Freeze the portdbapi for performance (memoize all xmatch results).
 		mydb.freeze()
-		eclasses_overridden.update(mydb.eclassdb._master_eclasses_overridden)
 	del mytrees, mydb
 
-	if eclasses_overridden and \
-		settings.get("PORTAGE_ECLASS_WARNING_ENABLE") != "0":
-		prefix = bad(" * ")
-		if len(eclasses_overridden) == 1:
-			writemsg(prefix + "Overlay eclass overrides " + \
-				"eclass from PORTDIR:\n", noiselevel=-1)
-		else:
-			writemsg(prefix + "Overlay eclasses override " + \
-				"eclasses from PORTDIR:\n", noiselevel=-1)
-		writemsg(prefix + "\n", noiselevel=-1)
-		for eclass_name in sorted(eclasses_overridden):
-			writemsg(prefix + "  '%s/%s.eclass'\n" % \
-				(eclasses_overridden[eclass_name], eclass_name),
-				noiselevel=-1)
-		writemsg(prefix + "\n", noiselevel=-1)
-		msg = "It is best to avoid overriding eclasses from PORTDIR " + \
-		"because it will trigger invalidation of cached ebuild metadata " + \
-		"that is distributed with the portage tree. If you must " + \
-		"override eclasses from PORTDIR then you are advised to add " + \
-		"FEATURES=\"metadata-transfer\" to /etc/make.conf and to run " + \
-		"`emerge --regen` after each time that you run `emerge --sync`. " + \
-		"Set PORTAGE_ECLASS_WARNING_ENABLE=\"0\" in /etc/make.conf if " + \
-		"you would like to disable this warning."
-		from textwrap import wrap
-		for line in wrap(msg, 72):
-			writemsg("%s%s\n" % (prefix, line), noiselevel=-1)
-
 	if "moo" in myfiles:
 		print """
 

Modified: main/branches/2.1.6/pym/portage/__init__.py
===================================================================
--- main/branches/2.1.6/pym/portage/__init__.py	2009-04-30 06:58:14 UTC (rev 13461)
+++ main/branches/2.1.6/pym/portage/__init__.py	2009-04-30 06:58:28 UTC (rev 13462)
@@ -1105,7 +1105,7 @@
 		"PORTAGE_BACKGROUND",
 		"PORTAGE_BINHOST_CHUNKSIZE", "PORTAGE_CALLER",
 		"PORTAGE_COUNTER_HASH",
-		"PORTAGE_ECLASS_WARNING_ENABLE", "PORTAGE_ELOG_CLASSES",
+		"PORTAGE_ELOG_CLASSES",
 		"PORTAGE_ELOG_MAILFROM", "PORTAGE_ELOG_MAILSUBJECT",
 		"PORTAGE_ELOG_MAILURI", "PORTAGE_ELOG_SYSTEM",
 		"PORTAGE_FETCH_CHECKSUM_TRY_MIRRORS", "PORTAGE_FETCH_RESUME_MIN_SIZE",
@@ -5150,6 +5150,12 @@
 	mysettings["FILESDIR"] = pkg_dir+"/files"
 	mysettings["PF"]       = mypv
 
+	if hasattr(mydbapi, '_repo_info'):
+		mytree = os.path.dirname(os.path.dirname(pkg_dir))
+		repo_info = mydbapi._repo_info[mytree]
+		mysettings['PORTDIR'] = repo_info.portdir
+		mysettings['PORTDIR_OVERLAY'] = repo_info.portdir_overlay
+
 	mysettings["PORTDIR"] = os.path.realpath(mysettings["PORTDIR"])
 	mysettings["DISTDIR"] = os.path.realpath(mysettings["DISTDIR"])
 	mysettings["RPMDIR"]  = os.path.realpath(mysettings["RPMDIR"])

Modified: main/branches/2.1.6/pym/portage/cache/metadata.py
===================================================================
--- main/branches/2.1.6/pym/portage/cache/metadata.py	2009-04-30 06:58:14 UTC (rev 13461)
+++ main/branches/2.1.6/pym/portage/cache/metadata.py	2009-04-30 06:58:28 UTC (rev 13462)
@@ -29,7 +29,7 @@
 		loc = location
 		super(database, self).__init__(location, *args, **config)
 		self.location = os.path.join(loc, "metadata","cache")
-		self.ec = portage.eclass_cache.cache(loc)
+		self.ec = None
 		self.raise_stat_collision = False
 
 	def _parse_data(self, data, cpv):
@@ -53,9 +53,11 @@
 
 		if "_eclasses_" not in d:
 			if "INHERITED" in d:
+				if self.ec is None:
+					self.ec = portage.eclass_cache.cache(self.location)
 				try:
 					d["_eclasses_"] = self.ec.get_eclass_data(
-						d["INHERITED"].split(), from_master_only=True)
+						d["INHERITED"].split())
 				except KeyError, e:
 					# INHERITED contains a non-existent eclass.
 					raise cache_errors.CacheCorruption(cpv, e)

Modified: main/branches/2.1.6/pym/portage/cache/util.py
===================================================================
--- main/branches/2.1.6/pym/portage/cache/util.py	2009-04-30 06:58:14 UTC (rev 13461)
+++ main/branches/2.1.6/pym/portage/cache/util.py	2009-04-30 06:58:28 UTC (rev 13462)
@@ -102,8 +102,7 @@
 				# Even if _eclasses_ already exists, replace it with data from
 				# eclass_cache, in order to insert local eclass paths.
 				try:
-					eclasses = eclass_cache.get_eclass_data(inherited,
-						from_master_only=True)
+					eclasses = eclass_cache.get_eclass_data(inherited)
 				except KeyError:
 					# INHERITED contains a non-existent eclass.
 					noise.eclass_stale(x)

Modified: main/branches/2.1.6/pym/portage/dbapi/porttree.py
===================================================================
--- main/branches/2.1.6/pym/portage/dbapi/porttree.py	2009-04-30 06:58:14 UTC (rev 13461)
+++ main/branches/2.1.6/pym/portage/dbapi/porttree.py	2009-04-30 06:58:28 UTC (rev 13462)
@@ -8,7 +8,8 @@
 portage.proxy.lazyimport.lazyimport(globals(),
 	'portage.checksum',
 	'portage.dep:dep_getkey,match_from_list,paren_reduce,use_reduce',
-	'portage.util:ensure_dirs,writemsg',
+	'portage.env.loaders:KeyValuePairFileLoader',
+	'portage.util:ensure_dirs,writemsg,writemsg_level',
 	'portage.versions:best,catpkgsplit,pkgsplit,ver_regexp',
 )
 
@@ -25,7 +26,7 @@
 	listdir, dep_expand, eapi_is_supported, key_expand, dep_check, \
 	_eapi_is_deprecated
 
-import codecs, os, stat
+import codecs, logging, os, stat
 from itertools import izip
 
 def _src_uri_validate(cpv, eapi, src_uri):
@@ -95,6 +96,15 @@
 			"getFetchMap(): '%s' SRC_URI arrow missing right operand" % \
 			(cpv,))
 
+class _repo_info(object):
+	__slots__ = ('name', 'path', 'eclass_db', 'portdir', 'portdir_overlay')
+	def __init__(self, name, path, eclass_db):
+		self.name = name
+		self.path = path
+		self.eclass_db = eclass_db
+		self.portdir = eclass_db.porttrees[0]
+		self.portdir_overlay = ' '.join(eclass_db.porttrees[1:])
+
 class portdbapi(dbapi):
 	"""this tree will scan a portage directory located at root (passed to init)"""
 	portdbapi_instances = []
@@ -116,10 +126,8 @@
 		# instance that is passed in.
 		self.doebuild_settings = config(clone=self.mysettings)
 
-		#self.root=settings["PORTDIR"]
+		porttree_root = os.path.realpath(porttree_root)
 		self.porttree_root = porttree_root
-		if porttree_root:
-			self.porttree_root = os.path.realpath(porttree_root)
 
 		self.depcachedir = os.path.realpath(self.mysettings.depcachedir)
 
@@ -132,8 +140,7 @@
 				os.environ["SANDBOX_WRITE"] = \
 					":".join(filter(None, sandbox_write))
 
-		self.eclassdb = eclass_cache.cache(self.porttree_root,
-			overlays=self.mysettings["PORTDIR_OVERLAY"].split())
+		self.eclassdb = eclass_cache.cache(porttree_root)
 
 		# This is used as sanity check for aux_get(). If there is no
 		# root eclass dir, we assume that PORTDIR is invalid or
@@ -163,7 +170,43 @@
 				# don't want to see a warning every time the portage module is
 				# imported.
 				pass
-		
+
+		self._repo_info = {}
+		eclass_dbs = {porttree_root : self.eclassdb}
+		for path in self.porttrees:
+			if path in self._repo_info:
+				continue
+
+			layout_filename = os.path.join(path, "metadata/layout.conf")
+			layout_file = KeyValuePairFileLoader(layout_filename, None, None)
+			layout_data, layout_errors = layout_file.load()
+			porttrees = []
+			for master_name in layout_data.get('masters', '').split():
+				master_path = self.treemap.get(master_name)
+				if master_path is None:
+					writemsg_level(("Unavailable repository '%s' " + \
+						"referenced by masters entry in '%s'\n") % \
+						(master_name, layout_filename),
+						level=logging.ERROR, noiselevel=-1)
+				else:
+					porttrees.append(master_path)
+			if not porttrees:
+				porttrees.append(porttree_root)
+			porttrees.append(path)
+
+			eclass_db = None
+			for porttree in porttrees:
+				tree_db = eclass_dbs.get(porttree)
+				if tree_db is None:
+					tree_db = eclass_cache.cache(porttree)
+				if eclass_db is None:
+					eclass_db = tree_db.copy()
+				else:
+					eclass_db.append(tree_db)
+
+			self._repo_info[path] = _repo_info(self._repository_map.get(path),
+				path, eclass_db)
+
 		self.auxdbmodule = self.mysettings.load_best_module("portdbapi.auxdbmodule")
 		self.auxdb = {}
 		self._pregen_auxdb = {}
@@ -191,6 +234,10 @@
 				if os.path.isdir(os.path.join(x, "metadata", "cache")):
 					self._pregen_auxdb[x] = self.metadbmodule(
 						x, "metadata/cache", filtered_auxdbkeys, readonly=True)
+					try:
+						self._pregen_auxdb[x].ec = self._repo_info[x].eclass_db
+					except AttributeError:
+						pass
 		# Selectively cache metadata in order to optimize dep matching.
 		self._aux_cache_keys = set(
 			["DEPEND", "EAPI", "INHERITED", "IUSE", "KEYWORDS", "LICENSE",

Modified: main/branches/2.1.6/pym/portage/eclass_cache.py
===================================================================
--- main/branches/2.1.6/pym/portage/eclass_cache.py	2009-04-30 06:58:14 UTC (rev 13461)
+++ main/branches/2.1.6/pym/portage/eclass_cache.py	2009-04-30 06:58:28 UTC (rev 13462)
@@ -5,6 +5,7 @@
 
 __all__ = ["cache"]
 
+import warnings
 from portage.util import normalize_path, writemsg
 import errno, os, sys
 from portage.data import portage_gid
@@ -15,19 +16,48 @@
 	Maintains the cache information about eclasses used in ebuild.
 	"""
 	def __init__(self, porttree_root, overlays=[]):
-		self.porttree_root = porttree_root
 
 		self.eclasses = {} # {"Name": ("location","_mtime_")}
 		self._eclass_locations = {}
 
 		# screw with the porttree ordering, w/out having bash inherit match it, and I'll hurt you.
 		# ~harring
-		self.porttrees = [self.porttree_root]+overlays
-		self.porttrees = tuple(map(normalize_path, self.porttrees))
-		self._master_eclass_root = os.path.join(self.porttrees[0],"eclass")
-		self._master_eclasses_overridden = {}
-		self.update_eclasses()
+		if porttree_root:
+			self.porttree_root = porttree_root
+			self.porttrees = [self.porttree_root] + overlays
+			self.porttrees = tuple(map(normalize_path, self.porttrees))
+			self._master_eclass_root = os.path.join(self.porttrees[0], "eclass")
+			self.update_eclasses()
+		else:
+			self.porttree_root = None
+			self.porttrees = ()
+			self._master_eclass_root = None
 
+	def copy(self):
+		return self.__copy__()
+
+	def __copy__(self):
+		result = self.__class__(None)
+		result.eclasses = self.eclasses.copy()
+		result._eclass_locations = self._eclass_locations.copy()
+		result.porttree_root = self.porttree_root
+		result.porttrees = self.porttrees
+		result._master_eclass_root = self._master_eclass_root
+		return result
+
+	def append(self, other):
+		"""
+		Append another instance to this instance. This will cause eclasses
+		from the other instance to override and eclases from this instance
+		that have the same name.
+		"""
+		if not isinstance(other, self.__class__):
+			raise TypeError(
+				"expected type %s, got %s" % (self.__class__, type(other)))
+		self.porttrees = self.porttrees + other.porttrees
+		self.eclasses.update(other.eclasses)
+		self._eclass_locations.update(other._eclass_locations)
+
 	def close_caches(self):
 		import traceback
 		traceback.print_stack()
@@ -77,8 +107,7 @@
 						# It appears to be identical to the master,
 						# so prefer the master entry.
 						continue
-					else:
-						self._master_eclasses_overridden[ys] = x
+
 				self.eclasses[ys] = (x, mtime)
 				self._eclass_locations[ys] = x
 
@@ -100,8 +129,10 @@
 		ec_dict = {}
 		for x in inherits:
 			ec_dict[x] = self.eclasses[x]
-			if from_master_only and \
-				self._eclass_locations[x] != self._master_eclass_root:
-				return None
 
+		if from_master_only is not False:
+			warnings.warn("portage.eclass_cache.cache.get_eclass_data(): " + \
+				"ignoring deprecated 'from_master_only' parameter",
+				DeprecationWarning)
+
 		return ec_dict