[gentoo-commits] portage r11302 - in main/branches/prefix: cnf doc/config pym/_emerge pym/portage pym/portage/sets

[Fabian Groffen (grobian)]

Author: grobian
Date: 2008-07-31 13:30:22 +0000 (Thu, 31 Jul 2008)
New Revision: 11302

Modified:
   main/branches/prefix/cnf/make.globals
   main/branches/prefix/cnf/sets.conf
   main/branches/prefix/doc/config/sets.docbook
   main/branches/prefix/pym/_emerge/__init__.py
   main/branches/prefix/pym/_emerge/help.py
   main/branches/prefix/pym/portage/__init__.py
   main/branches/prefix/pym/portage/sets/dbapi.py
Log:
   Merged from trunk 11287:11300

   | 11288   | Bug #233421 - Fix grammar, missing "be" in --update          |
   | zmedico | description. Thanks to Mikael Magnusson for this patch.      |
   
   | 11289   | Split out a _spawn_fetch() function that will be useful for  |
   | zmedico | implementing a userpriv testcase for bug #233303.            |
   
   | 11290   | Fixes in portage.fetch() for bugs #233303 and #94133: *      |
   | zmedico | Totally skip $DISTDIR creation if the fetch_to_ro feature is |
   |         | enabled. * Don't touch $DISTDIR permissions unless unless    |
   |         | usepriv and/or userfetch are enabled. * When usepriv and/or  |
   |         | userfetch are enabled, test whether or not a process that    |
   |         | has dropped privileges is able to create a file in the       |
   |         | directory, and only adjust permissions if the test fails. *  |
   |         | Completely                                                   |
   
   | 11291   | disable default IONICE command as it breaks for non-root,    |
   | genone  | and ionice might not always be available                     |
   
   | 11292   | Implement a new @live-ebuilds which is generated from        |
   | zmedico | installed packages that inherit from know live eclasses such |
   |         | as cvs, darcs, git, mercurial, and subversion. The list of   |
   |         | eclasses is controlled by an "inherits" attribute that is    |
   |         | configure in sets.conf for and instance of InheritSet. This  |
   |         | set serves a purpose similar to the -scm ebuild suffix that  |
   |         | has been proposed in GLEP 54.                                |
   
   | 11293   | Add a new @module-rebuild set which emulates the behavior of |
   | zmedico | the module-rebuild tool. The /lib/modules path is set in     |
   |         | sets.conf via a "files" attribute of an OwnerSet instance.   |
   |         | This can be easily used to define similar sets based on      |
   |         | paths on installed files.                                    |
   
   | 11294   | Remove quotes since the seem to cause incorrect results.     |
   | zmedico |                                                              |
   
   | 11295   | * Rename @live-ebuilds to @live-rebuild, for consistency     |
   | zmedico | with the other *-rebuild sets. * Document the new sets.      |
   
   | 11296   | Describe InheritSet and OwnerSet.                            |
   | zmedico |                                                              |
   
   | 11297   | Bug #233458 - Fix AsynchronousTask exit listener handling so |
   | zmedico | that an exit listener will never get called after it's been  |
   |         | passed into removeExitListener(), since the caller of        |
   |         | removeExitListener() needs to be able to be able to trust    |
   |         | that the given exit listener will not be called under any    |
   |         | circumstances.                                               |
   
   | 11298   | Always invalidate results from                               |
   | zmedico | _userpriv_test_write_file_cache when adjusting permissions   |
   |         | on a given directory.                                        |
   
   | 11299   | Bug #233253 - Implement a @downgrade set which selects       |
   | zmedico | packages for which the highest visible ebuild version is     |
   |         | lower than the currently installed version. This is useful   |
   |         | if you have installed packages from an overlay and you want  |
   |         | to downgrade to the highest visible after removing the       |
   |         | overlay, even though the packages that will be dowgraded are |
   |         | not necessarily masked in any way.                           |
   
   | 11300   | Fix DowngradeSet so it's safe for cases when no ebuild is    |
   | zmedico | available.                                                   |


Modified: main/branches/prefix/cnf/make.globals
===================================================================
--- main/branches/prefix/cnf/make.globals	2008-07-31 13:26:41 UTC (rev 11301)
+++ main/branches/prefix/cnf/make.globals	2008-07-31 13:30:22 UTC (rev 11302)
@@ -63,7 +63,8 @@
 PORTAGE_FETCH_RESUME_MIN_SIZE="350K"
 
 # Command called to adjust the io priority of portage and it's subprocesses.
-PORTAGE_IONICE_COMMAND="ionice -c 3 -p \${PID}"
+# Note: should be wrapped inside a uid check
+#PORTAGE_IONICE_COMMAND="ionice -c 3 -p \${PID}"
 
 # Number of times 'emerge --sync' will run before giving up.
 PORTAGE_RSYNC_RETRIES="3"

Modified: main/branches/prefix/cnf/sets.conf
===================================================================
--- main/branches/prefix/cnf/sets.conf	2008-07-31 13:26:41 UTC (rev 11301)
+++ main/branches/prefix/cnf/sets.conf	2008-07-31 13:30:22 UTC (rev 11302)
@@ -41,3 +41,21 @@
 [preserved-rebuild]
 class = portage.sets.libs.PreservedLibraryConsumerSet
 world-candidate = False
+
+# Installed ebuilds that inherit from known live eclasses.
+[live-rebuild]
+class = portage.sets.dbapi.InheritSet
+world-candidate = False
+inherits = cvs darcs git mercurial subversion
+
+# Installed packages that own files inside /lib/modules.
+[module-rebuild]
+class = portage.sets.dbapi.OwnerSet
+world-candidate = False
+files = /lib/modules
+
+# Installed packages for which the highest visible ebuild
+# version is lower than the currently installed version.
+[downgrade]
+class = portage.sets.dbapi.DowngradeSet
+world-candidate = False

Modified: main/branches/prefix/doc/config/sets.docbook
===================================================================
--- main/branches/prefix/doc/config/sets.docbook	2008-07-31 13:26:41 UTC (rev 11301)
+++ main/branches/prefix/doc/config/sets.docbook	2008-07-31 13:30:22 UTC (rev 11302)
@@ -453,6 +453,41 @@
 			</para>
 			</sect3>
 		</sect2>
+		<sect2 id='config-set-classes-InheritSet'>
+		<title>portage.sets.dbapi.InheritSet</title>
+		<para>
+		Package set which contains all packages
+		that inherit one or more specific eclasses.
+		This class supports the following options:
+		<itemizedlist>
+		<listitem><varname>inherits</varname>: Required. A list of eclass names
+		which should be used to create the package set.
+		</listitem>
+		</itemizedlist>
+		</para>
+		</sect2>
+		<sect2 id='config-set-classes-OwnerSet'>
+		<title>portage.sets.dbapi.OwnerSet</title>
+		<para>
+		Package set which contains all packages
+		that own one or more files.
+		This class supports the following options:
+		<itemizedlist>
+		<listitem><varname>files</varname>: Required. A list of file paths
+		that should be used to create the package set.
+		</listitem>
+		</itemizedlist>
+		</para>
+		</sect2>
+		<sect2 id='config-set-classes-DowngradeSet'>
+		<title>portage.sets.dbapi.DowngradeSet</title>
+		<para>
+		Package set which contains all packages
+		for which the highest visible ebuild version is lower than
+		the currently installed version.
+		This class doesn't support any extra options.
+		</para>
+		</sect2>
 		<sect2 id='config-set-classes-PreservedLibraryConsumerSet'>
 		<title>portage.sets.libs.PreservedLibraryConsumerSet</title>
 		<para>
@@ -490,6 +525,9 @@
 	<listitem><varname>security</varname>: uses <classname>NewAffectedSet</classname> with default options</listitem>
 	<listitem><varname>everything</varname>: uses <classname>EverythingSet</classname></listitem>
 	<listitem><varname>preserved-rebuild</varname>: uses <classname>PreservedLibraryConsumerSet</classname></listitem>
+	<listitem><varname>live-rebuild</varname>: uses <classname>InheritSet</classname></listitem>
+	<listitem><varname>module-rebuild</varname>: uses <classname>OwnerSet</classname></listitem>
+	<listitem><varname>downgrade</varname>: uses <classname>DowngradeSet</classname></listitem>
 	</itemizedlist>
 	Additionally the default configuration includes a multi set section based on
 	the <classname>StaticFileSet</classname> defaults that creates a set for each 

Modified: main/branches/prefix/pym/_emerge/__init__.py
===================================================================
--- main/branches/prefix/pym/_emerge/__init__.py	2008-07-31 13:26:41 UTC (rev 11301)
+++ main/branches/prefix/pym/_emerge/__init__.py	2008-07-31 13:30:22 UTC (rev 11302)
@@ -1626,7 +1626,7 @@
 	"""
 
 	__slots__ = ("background", "cancelled", "returncode") + \
-		("_exit_listeners", "_start_listeners")
+		("_exit_listeners", "_exit_listener_stack", "_start_listeners")
 
 	def start(self):
 		"""
@@ -1692,6 +1692,8 @@
 
 	def removeExitListener(self, f):
 		if self._exit_listeners is None:
+			if self._exit_listener_stack is not None:
+				self._exit_listener_stack.remove(f)
 			return
 		self._exit_listeners.remove(f)
 
@@ -1707,12 +1709,22 @@
 
 			# This prevents recursion, in case one of the
 			# exit handlers triggers this method again by
-			# calling wait().
-			exit_listeners = self._exit_listeners
+			# calling wait(). Use a stack that gives
+			# removeExitListener() an opportunity to consume
+			# listeners from the stack, before they can get
+			# called below. This is necessary because a call
+			# to one exit listener may result in a call to
+			# removeExitListener() for another listener on
+			# the stack. That listener needs to be removed
+			# from the stack since it would be inconsistent
+			# to call it after it has been been passed into
+			# removeExitListener().
+			self._exit_listener_stack = self._exit_listeners
 			self._exit_listeners = None
 
-			for f in exit_listeners:
-				f(self)
+			self._exit_listener_stack.reverse()
+			while self._exit_listener_stack:
+				self._exit_listener_stack.pop()(self)
 
 class PipeReader(AsynchronousTask):
 

Modified: main/branches/prefix/pym/_emerge/help.py
===================================================================
--- main/branches/prefix/pym/_emerge/help.py	2008-07-31 13:26:41 UTC (rev 11301)
+++ main/branches/prefix/pym/_emerge/help.py	2008-07-31 13:30:22 UTC (rev 11302)
@@ -189,8 +189,8 @@
 		print "              Updates packages to the best version available, which may not"
 		print "              always be the highest version number due to masking for testing"
 		print "              and development. This will also update direct dependencies which"
-		print "              may not what you want. Package atoms specified on the command line"
-		print "              are greedy, meaning that unspecific atoms may match multiple"
+		print "              may not be what you want. Package atoms specified on the command"
+		print "              line are greedy, meaning that unspecific atoms may match multiple"
 		print "              installed versions of slotted packages."
 		print
 		print "       "+green("--version")+" ("+green("-V")+" short option)"

Modified: main/branches/prefix/pym/portage/__init__.py
===================================================================
--- main/branches/prefix/pym/portage/__init__.py	2008-07-31 13:26:41 UTC (rev 11301)
+++ main/branches/prefix/pym/portage/__init__.py	2008-07-31 13:30:22 UTC (rev 11302)
@@ -3167,6 +3167,84 @@
 		return retval >> 8
 	return retval
 
+_userpriv_spawn_kwargs = (
+	("uid",    portage_uid),
+	("gid",    portage_gid),
+	("groups", userpriv_groups),
+	("umask",  002),
+)
+
+def _spawn_fetch(settings, args, **kwargs):
+	"""
+	Spawn a process with appropriate settings for fetching, including
+	userfetch and selinux support.
+	"""
+
+	global _userpriv_spawn_kwargs
+
+	# Redirect all output to stdout since some fetchers like
+	# wget pollute stderr (if portage detects a problem then it
+	# can send it's own message to stderr).
+	if "fd_pipes" not in kwargs:
+
+		kwargs["fd_pipes"] = {
+			0 : sys.stdin.fileno(),
+			1 : sys.stdout.fileno(),
+			2 : sys.stdout.fileno(),
+		}
+
+	if "userfetch" in settings.features and \
+		os.getuid() == 0 and portage_gid and portage_uid:
+		kwargs.update(_userpriv_spawn_kwargs)
+
+	try:
+
+		if settings.selinux_enabled():
+			con = selinux.getcontext()
+			con = con.replace(settings["PORTAGE_T"], settings["PORTAGE_FETCH_T"])
+			selinux.setexec(con)
+			# bash is an allowed entrypoint, while most binaries are not
+			if args[0] != BASH_BINARY:
+				args = [BASH_BINARY, "-c", "exec \"$@\"", args[0]] + args
+
+		rval = portage.process.spawn(args,
+			env=dict(settings.iteritems()), **kwargs)
+
+	finally:
+		if settings.selinux_enabled():
+			selinux.setexec(None)
+
+	return rval
+
+_userpriv_test_write_file_cache = {}
+_userpriv_test_write_cmd_script = "> %(file_path)s ; rval=$? ; " + \
+	"rm -f  %(file_path)s ; exit $rval"
+
+def _userpriv_test_write_file(settings, file_path):
+	"""
+	Drop privileges and try to open a file for writing. The file may or
+	may not exist, and the parent directory is assumed to exist. The file
+	is removed before returning.
+
+	@param settings: A config instance which is passed to _spawn_fetch()
+	@param file_path: A file path to open and write.
+	@return: True if write succeeds, False otherwise.
+	"""
+
+	global _userpriv_test_write_file_cache, _userpriv_test_write_cmd_script
+	rval = _userpriv_test_write_file_cache.get(file_path)
+	if rval is not None:
+		return rval
+
+	args = [BASH_BINARY, "-c", _userpriv_test_write_cmd_script % \
+		{"file_path" : _shell_quote(file_path)}]
+
+	returncode = _spawn_fetch(settings, args)
+
+	rval = returncode == os.EX_OK
+	_userpriv_test_write_file_cache[file_path] = rval
+	return rval
+
 def _checksum_failure_temp_file(distdir, basename):
 	"""
 	First try to find a duplicate temp file with the same checksum and return
@@ -3274,6 +3352,11 @@
 
 	features = mysettings.features
 	restrict = mysettings.get("PORTAGE_RESTRICT","").split()
+
+	from portage.data import secpass
+	userfetch = secpass >= 2 and "userfetch" in features
+	userpriv = secpass >= 2 and "userpriv" in features
+
 	# 'nomirror' is bad/negative logic. You Restrict mirroring, not no-mirroring.
 	if "mirror" in restrict or \
 	   "nomirror" in restrict:
@@ -3473,7 +3556,8 @@
 		if not mysettings.get(var_name, None):
 			can_fetch = False
 
-	if can_fetch:
+	if can_fetch and not fetch_to_ro:
+		global _userpriv_test_write_file_cache
 		dirmode  = 02070
 		filemode =   060
 		modemask =    02
@@ -3491,6 +3575,16 @@
 			
 			for x in distdir_dirs:
 				mydir = os.path.join(mysettings["DISTDIR"], x)
+				write_test_file = os.path.join(
+					mydir, ".__portage_test_write__")
+
+				if os.path.isdir(mydir):
+					if not (userfetch or userpriv):
+						continue
+					if _userpriv_test_write_file(mysettings, write_test_file):
+						continue
+
+				_userpriv_test_write_file_cache.pop(write_test_file, None)
 				if portage.util.ensure_dirs(mydir, gid=dir_gid, mode=dirmode, mask=modemask):
 					writemsg("Adjusting permissions recursively: '%s'\n" % mydir,
 						noiselevel=-1)
@@ -3840,39 +3934,11 @@
 					lexer = shlex.shlex(StringIO.StringIO(locfetch), posix=True)
 					lexer.whitespace_split = True
 					myfetch = [varexpand(x, mydict=variables) for x in lexer]
-
-					spawn_keywords = {}
-					# Redirect all output to stdout since some fetchers like
-					# wget pollute stderr (if portage detects a problem then it
-					# can send it's own message to stderr).
-					spawn_keywords["fd_pipes"] = {
-						0:sys.stdin.fileno(),
-						1:sys.stdout.fileno(),
-						2:sys.stdout.fileno()
-					}
-					if "userfetch" in mysettings.features and \
-						os.getuid() == 0 and portage_gid and portage_uid:
-						spawn_keywords.update({
-							"uid"    : portage_uid,
-							"gid"    : portage_gid,
-							"groups" : userpriv_groups,
-							"umask"  : 002})
 					myret = -1
 					try:
 
-						if mysettings.selinux_enabled():
-							con = selinux.getcontext()
-							con = con.replace(mysettings["PORTAGE_T"], mysettings["PORTAGE_FETCH_T"])
-							selinux.setexec(con)
-							# bash is an allowed entrypoint, while most binaries are not
-							myfetch = ["bash", "-c", "exec \"$@\"", myfetch[0]] + myfetch
+						myret = _spawn_fetch(mysettings, myfetch)
 
-						myret = portage.process.spawn(myfetch,
-							env=dict(mysettings.iteritems()), **spawn_keywords)
-
-						if mysettings.selinux_enabled():
-							selinux.setexec(None)
-
 					finally:
 						try:
 							apply_secpass_permissions(myfile_path,

Modified: main/branches/prefix/pym/portage/sets/dbapi.py
===================================================================
--- main/branches/prefix/pym/portage/sets/dbapi.py	2008-07-31 13:26:41 UTC (rev 11301)
+++ main/branches/prefix/pym/portage/sets/dbapi.py	2008-07-31 13:30:22 UTC (rev 11302)
@@ -2,11 +2,11 @@
 # Distributed under the terms of the GNU General Public License v2
 # $Id$
 
-from portage.versions import catsplit
+from portage.versions import catpkgsplit, catsplit, pkgcmp
 from portage.sets.base import PackageSet
 from portage.sets import SetConfigError, get_boolean
 
-__all__ = ["CategorySet", "EverythingSet"]
+__all__ = ["CategorySet", "EverythingSet", "InheritSet"]
 
 class EverythingSet(PackageSet):
 	_operations = ["merge", "unmerge"]
@@ -32,6 +32,119 @@
 		return EverythingSet(trees["vartree"].dbapi)
 	singleBuilder = classmethod(singleBuilder)
 
+class OwnerSet(PackageSet):
+
+	_operations = ["merge", "unmerge"]
+
+	description = "Package set which contains all packages " + \
+		"that own one or more files."
+
+	def __init__(self, vardb=None, files=None):
+		super(OwnerSet, self).__init__()
+		self._db = vardb
+		self._files = files
+
+	def mapPathsToAtoms(self, paths):
+		rValue = set()
+		vardb = self._db
+		aux_get = vardb.aux_get
+		aux_keys = ["SLOT"]
+		for link, p in vardb._owners.iter_owners(paths):
+			cat, pn = catpkgsplit(link.mycpv)[:2]
+			slot, = aux_get(link.mycpv, aux_keys)
+			rValue.add("%s/%s:%s" % (cat, pn, slot))
+		return rValue
+
+	def load(self):
+		self._setAtoms(self.mapPathsToAtoms(self._files))
+
+	def singleBuilder(cls, options, settings, trees):
+		if not "files" in options:
+			raise SetConfigError("no files given")
+
+		import shlex
+		return cls(vardb=trees["vartree"].dbapi,
+			files=frozenset(shlex.split(options["files"])))
+
+	singleBuilder = classmethod(singleBuilder)
+
+class InheritSet(PackageSet):
+
+	_operations = ["merge", "unmerge"]
+
+	description = "Package set which contains all packages " + \
+		"that inherit one or more specific eclasses."
+
+	def __init__(self, vardb=None, inherits=None):
+		super(InheritSet, self).__init__()
+		self._db = vardb
+		self._inherits = inherits
+
+	def load(self):
+		atoms = []
+		inherits = self._inherits
+		cp_list = self._db.cp_list
+		aux_get = self._db.aux_get
+		aux_keys = ["INHERITED", "SLOT"]
+		for cp in self._db.cp_all():
+			for cpv in cp_list(cp):
+				inherited, slot = aux_get(cpv, aux_keys)
+				inherited = inherited.split()
+				if inherits.intersection(inherited):
+					atoms.append("%s:%s" % (cp, slot))
+
+		self._setAtoms(atoms)
+
+	def singleBuilder(cls, options, settings, trees):
+		if not "inherits" in options:
+			raise SetConfigError("no inherits given")
+
+		inherits = options["inherits"]
+		return cls(vardb=trees["vartree"].dbapi,
+			inherits=frozenset(inherits.split()))
+
+	singleBuilder = classmethod(singleBuilder)
+
+class DowngradeSet(PackageSet):
+
+	_operations = ["merge", "unmerge"]
+
+	description = "Package set which contains all packages " + \
+		"for which the highest visible ebuild version is lower than " + \
+		"the currently installed version."
+
+	def __init__(self, portdb=None, vardb=None):
+		super(DowngradeSet, self).__init__()
+		self._portdb = portdb
+		self._vardb = vardb
+
+	def load(self):
+		atoms = []
+		xmatch = self._portdb.xmatch
+		xmatch_level = "bestmatch-visible"
+		cp_list = self._vardb.cp_list
+		aux_get = self._vardb.aux_get
+		aux_keys = ["SLOT"]
+		for cp in self._vardb.cp_all():
+			for cpv in cp_list(cp):
+				slot, = aux_get(cpv, aux_keys)
+				slot_atom = "%s:%s" % (cp, slot)
+				ebuild = xmatch(xmatch_level, slot_atom)
+				if not ebuild:
+					continue
+				ebuild_split = catpkgsplit(ebuild)[1:]
+				installed_split = catpkgsplit(cpv)[1:]
+				if pkgcmp(installed_split, ebuild_split) > 0:
+					atoms.append(slot_atom)
+
+		self._setAtoms(atoms)
+
+	def singleBuilder(cls, options, settings, trees):
+		return cls(portdb=trees["porttree"].dbapi,
+			vardb=trees["vartree"].dbapi)
+
+	singleBuilder = classmethod(singleBuilder)
+
 class CategorySet(PackageSet):
 	_operations = ["merge", "unmerge"]