[gentoo-commits] portage r14437 - in main/branches/prefix: bin pym/portage pym/portage/cache pym/portage/dbapi pym/portage/tests pym/portage/tests/ebuild pym/repoman

[Fabian Groffen (grobian)]

Author: grobian
Date: 2009-09-26 18:44:55 +0000 (Sat, 26 Sep 2009)
New Revision: 14437

Added:
   main/branches/prefix/pym/portage/tests/ebuild/test_pty_eof.py
Modified:
   main/branches/prefix/bin/dispatch-conf
   main/branches/prefix/bin/emaint
   main/branches/prefix/bin/regenworld
   main/branches/prefix/pym/portage/__init__.py
   main/branches/prefix/pym/portage/cache/flat_hash.py
   main/branches/prefix/pym/portage/dbapi/porttree.py
   main/branches/prefix/pym/portage/dbapi/vartree.py
   main/branches/prefix/pym/portage/glsa.py
   main/branches/prefix/pym/portage/tests/__init__.py
   main/branches/prefix/pym/portage/update.py
   main/branches/prefix/pym/portage/util.py
   main/branches/prefix/pym/repoman/utilities.py
Log:
   Merged from trunk -r14412:14422

   | 14413    | Use a runtime check to see if                               |
   | zmedico  | http://bugs.python.org/issue5380 is fixed.                  |
   
   | 14414    | Make _test_pty_eof() return None if openpty() fails.        |
   | zmedico  |                                                             |
   
   | 14415    | Add a test case for http://bugs.python.org/issue5380 and    |
   | zmedico  | mark it as 'todo' since it fails with current versions of   |
   |          | python 3.                                                   |
   
   | 14416    | Fix regression in _getitem() from r14398, since             |
   | zmedico  | myf.read().split("\n") yields an empty string at the end    |
   |          | which is causes _parse_data() to catch a ValueError and     |
   |          | raise CacheCorruption.                                      |
   
   | 14417    | Show an appropriate error message in _create_pty_or_pipe()  |
   | zmedico  | if openpty() fails inside _test_pty_eof().                  |
   
   | 14418    | Remove unnecessary dict.has_key() call (fix for python 3).  |
   | zmedico  |                                                             |
   
   | 14419    | Fix ValueError in digestcheck(), reported by Arfrever.      |
   | zmedico  |                                                             |
   
   | 14420    | Fix more.                                                   |
   | zmedico  |                                                             |
   
   | 14421    | Use list comprehensions instead of filter() or map() in     |
   | arfrever | some places for compatibility with Python 3.                |
   
   | 14422    | Use 'in' instead of has_key() in portage.glsa.Glsa.parse()  |
   | arfrever | when Python 3 is used.                                      |


Modified: main/branches/prefix/bin/dispatch-conf
===================================================================
--- main/branches/prefix/bin/dispatch-conf	2009-09-26 18:43:40 UTC (rev 14436)
+++ main/branches/prefix/bin/dispatch-conf	2009-09-26 18:44:55 UTC (rev 14437)
@@ -207,7 +207,7 @@
             else:
                 return True
 
-        confs = filter (f, confs)
+        confs = [x for x in confs if f(x)]
 
         #
         # Interactively process remaining

Modified: main/branches/prefix/bin/emaint
===================================================================
--- main/branches/prefix/bin/emaint	2009-09-26 18:43:40 UTC (rev 14436)
+++ main/branches/prefix/bin/emaint	2009-09-26 18:44:55 UTC (rev 14437)
@@ -82,9 +82,9 @@
 		self._check_world(onProgress)
 		errors = []
 		if self.found:
-			errors += map(lambda x: "'%s' is not a valid atom" % x, self.invalid)
-			errors += map(lambda x: "'%s' is not installed" % x, self.not_installed)
-			errors += map(lambda x: "'%s' has a category that is not listed in /etc/portage/categories" % x, self.invalid_category)
+			errors += ["'%s' is not a valid atom" % x for x in self.invalid]
+			errors += ["'%s' is not installed" % x for x in self.not_installed]
+			errors += ["'%s' has a category that is not listed in /etc/portage/categories" % x for x in self.invalid_category]
 		else:
 			errors.append(self.world_file + " could not be opened for reading")
 		return errors

Modified: main/branches/prefix/bin/regenworld
===================================================================
--- main/branches/prefix/bin/regenworld	2009-09-26 18:43:40 UTC (rev 14436)
+++ main/branches/prefix/bin/regenworld	2009-09-26 18:44:55 UTC (rev 14437)
@@ -59,18 +59,16 @@
 	sys.exit(0)
 
 worldlist = portage.grabfile(os.path.join(portage.const.EPREFIX, portage.WORLD_FILE))
-syslist = portage.settings.packages
-syslist = filter(issyspkg, syslist)
+syslist = [x for x in portage.settings.packages if issyspkg(x)]
 
 logfile = portage.grabfile(portage.const.EPREFIX+"/var/log/emerge.log")
-biglist = filter(iscandidate, logfile)
-biglist = map(getpkginfo, biglist)
+biglist = [getpkginfo(x) for x in logfile if iscandidate(x)]
 tmplist = []
 for l in biglist:
 	tmplist += l.split()
-biglist = filter(isunwanted, tmplist)
+biglist = [x for x in tmplist if isunwanted(x)]
 #for p in biglist:
-#	print p
+#	print(p)
 #sys.exit(0)
 
 # resolving virtuals
@@ -78,7 +76,7 @@
 for mykey in syslist:
 	# drop the asterix
 	mykey = mykey[1:]
-	#print "candidate:",mykey
+	#print("candidate:",mykey)
 	mylist=portage.db["/"]["vartree"].dbapi.match(mykey)
 	if mylist:
 		mykey=portage.cpv_getkey(mylist[0])
@@ -86,7 +84,7 @@
 			realsyslist.append(mykey)
 
 for mykey in biglist:
-	#print "checking:",mykey
+	#print("checking:",mykey)
 	try:
 		mylist=portage.db["/"]["vartree"].dbapi.match(mykey)
 	except (portage.exception.InvalidAtom, KeyError):

Modified: main/branches/prefix/pym/portage/__init__.py
===================================================================
--- main/branches/prefix/pym/portage/__init__.py	2009-09-26 18:43:40 UTC (rev 14436)
+++ main/branches/prefix/pym/portage/__init__.py	2009-09-26 18:44:55 UTC (rev 14437)
@@ -3755,6 +3755,60 @@
 		keys = __iter__
 		items = iteritems
 
+def _test_pty_eof():
+	"""
+	Returns True if this issues is fixed for the currently
+	running version of python: http://bugs.python.org/issue5380
+	Raises an EnvironmentError from openpty() if it fails.
+	"""
+
+	import array, pty, termios
+	test_string = 2 * "blah blah blah\n"
+	test_string = _unicode_decode(test_string,
+		encoding='utf_8', errors='strict')
+
+	# may raise EnvironmentError
+	master_fd, slave_fd = pty.openpty()
+
+	master_file = os.fdopen(master_fd, 'rb')
+	slave_file = os.fdopen(slave_fd, 'wb')
+
+	# Disable post-processing of output since otherwise weird
+	# things like \n -> \r\n transformations may occur.
+	mode = termios.tcgetattr(slave_fd)
+	mode[1] &= ~termios.OPOST
+	termios.tcsetattr(slave_fd, termios.TCSANOW, mode)
+
+	# Simulate a subprocess writing some data to the
+	# slave end of the pipe, and then exiting.
+	slave_file.write(_unicode_encode(test_string,
+		encoding='utf_8', errors='strict'))
+	slave_file.close()
+
+	eof = False
+	data = []
+
+	while not eof:
+
+		buf = array.array('B')
+		try:
+			buf.fromfile(master_file, 1024)
+		except EOFError:
+			eof = True
+		except IOError:
+			# This is where data loss occurs.
+			eof = True
+
+		if not buf:
+			eof = True
+		else:
+			data.append(_unicode_decode(buf.tostring(),
+				encoding='utf_8', errors='strict'))
+
+	master_file.close()
+
+	return test_string == ''.join(data)
+
 # In some cases, openpty can be slow when it fails. Therefore,
 # stop trying to use it after the first failure.
 if platform.system() in ["SunOS"]:
@@ -3764,11 +3818,8 @@
 	_disable_openpty = True
 else:
 	_disable_openpty = False
+_tested_pty = False
 
-if sys.hexversion >= 0x3000000:
-	# This is a temporary workaround for http://bugs.python.org/issue5380.
-	_disable_openpty = True
-
 def _create_pty_or_pipe(copy_term_size=None):
 	"""
 	Try to create a pty and if then fails then create a normal
@@ -3785,7 +3836,18 @@
 
 	got_pty = False
 
-	global _disable_openpty
+	global _disable_openpty, _tested_pty
+	if not (_tested_pty or _disable_openpty):
+		try:
+			if not _test_pty_eof():
+				_disable_openpty = True
+		except EnvironmentError as e:
+			_disable_openpty = True
+			writemsg("openpty failed: '%s'\n" % str(e),
+				noiselevel=-1)
+			del e
+		_tested_pty = True
+
 	if _disable_openpty:
 		master_fd, slave_fd = os.pipe()
 	else:
@@ -5379,6 +5441,7 @@
 				return 0
 			continue
 		for d in dirs:
+			d_bytes = d
 			try:
 				d = _unicode_decode(d,
 					encoding=_encodings['fs'], errors='strict')
@@ -5391,10 +5454,10 @@
 					noiselevel=-1)
 				if strict:
 					return 0
-				dirs.remove(d)
+				dirs.remove(d_bytes)
 				continue
 			if d.startswith(".") or d == "CVS":
-				dirs.remove(d)
+				dirs.remove(d_bytes)
 		for f in files:
 			try:
 				f = _unicode_decode(f,

Modified: main/branches/prefix/pym/portage/cache/flat_hash.py
===================================================================
--- main/branches/prefix/pym/portage/cache/flat_hash.py	2009-09-26 18:43:40 UTC (rev 14436)
+++ main/branches/prefix/pym/portage/cache/flat_hash.py	2009-09-26 18:44:55 UTC (rev 14437)
@@ -41,7 +41,10 @@
 				mode='r', encoding=_encodings['repo.content'],
 				errors='replace')
 			try:
-				d = self._parse_data(myf.read().split("\n"), cpv)
+				lines = myf.read().split("\n")
+				if not lines[-1]:
+					lines.pop()
+				d = self._parse_data(lines, cpv)
 				if '_mtime_' not in d:
 					# Backward compatibility with old cache
 					# that uses mtime mangling.

Modified: main/branches/prefix/pym/portage/dbapi/porttree.py
===================================================================
--- main/branches/prefix/pym/portage/dbapi/porttree.py	2009-09-26 18:43:40 UTC (rev 14436)
+++ main/branches/prefix/pym/portage/dbapi/porttree.py	2009-09-26 18:44:55 UTC (rev 14437)
@@ -342,7 +342,7 @@
 
 		# XXX: REMOVE THIS ONCE UNUSED_0 IS YANKED FROM auxdbkeys
 		# ~harring
-		filtered_auxdbkeys = list(filter(lambda x: not x.startswith("UNUSED_0"), auxdbkeys))
+		filtered_auxdbkeys = [x for x in auxdbkeys if not x.startswith("UNUSED_0")]
 		filtered_auxdbkeys.sort()
 		from portage.cache import metadata_overlay, volatile
 		if not depcachedir_w_ok:

Modified: main/branches/prefix/pym/portage/dbapi/vartree.py
===================================================================
--- main/branches/prefix/pym/portage/dbapi/vartree.py	2009-09-26 18:43:40 UTC (rev 14436)
+++ main/branches/prefix/pym/portage/dbapi/vartree.py	2009-09-26 18:44:55 UTC (rev 14437)
@@ -388,7 +388,7 @@
 				for x in filter(None, fields[3].replace(
 				"${ORIGIN}", os.path.dirname(obj)).replace(
 				"$ORIGIN", os.path.dirname(obj)).split(":"))])
-			needed = filter(None, fields[4].split(","))
+			needed = [x for x in fields[4].split(",") if x]
 
 			obj_key = self._obj_key(obj)
 			indexed = True

Modified: main/branches/prefix/pym/portage/glsa.py
===================================================================
--- main/branches/prefix/pym/portage/glsa.py	2009-09-26 18:43:40 UTC (rev 14436)
+++ main/branches/prefix/pym/portage/glsa.py	2009-09-26 18:44:55 UTC (rev 14437)
@@ -511,7 +511,8 @@
 		# <revised count="2">2007-12-30</revised>
 		revisedEl = myroot.getElementsByTagName("revised")[0]
 		self.revised = getText(revisedEl, format="strip")
-		if (revisedEl.attributes.has_key("count")):
+		if ((sys.hexversion >= 0x3000000 and "count" in revisedEl.attributes) or
+			(sys.hexversion < 0x3000000 and revisedEl.attributes.has_key("count"))):
 			count = revisedEl.getAttribute("count")
 		elif (self.revised.find(":") >= 0):
 			(self.revised, count) = self.revised.split(":")

Modified: main/branches/prefix/pym/portage/tests/__init__.py
===================================================================
--- main/branches/prefix/pym/portage/tests/__init__.py	2009-09-26 18:43:40 UTC (rev 14436)
+++ main/branches/prefix/pym/portage/tests/__init__.py	2009-09-26 18:44:55 UTC (rev 14437)
@@ -176,7 +176,8 @@
 		self.stream.writeln()
 		if not result.wasSuccessful():
 			self.stream.write("FAILED (")
-			failed, errored = map(len, (result.failures, result.errors))
+			failed = len(result.failures)
+			errored = len(result.errors)
 			if failed:
 				self.stream.write("failures=%d" % failed)
 			if errored:

Copied: main/branches/prefix/pym/portage/tests/ebuild/test_pty_eof.py (from rev 14422, main/trunk/pym/portage/tests/ebuild/test_pty_eof.py)
===================================================================
--- main/branches/prefix/pym/portage/tests/ebuild/test_pty_eof.py	                        (rev 0)
+++ main/branches/prefix/pym/portage/tests/ebuild/test_pty_eof.py	2009-09-26 18:44:55 UTC (rev 14437)
@@ -0,0 +1,22 @@
+# Copyright 1998-2007 Gentoo Foundation
+# Distributed under the terms of the GNU General Public License v2
+# $Id$
+
+import pty
+
+import portage
+from portage import os
+from portage.tests import TestCase
+
+class PtyEofTestCase(TestCase):
+
+	def testPtyEof(self):
+		# This tests if the following python issue is fixed yet:
+		#   http://bugs.python.org/issue5380
+		# Since it might not be fixed, mark as todo.
+		self.todo = True
+		# The result is only valid if openpty does not raise EnvironmentError.
+		try:
+			self.assertEqual(portage._test_pty_eof(), True)
+		except EnvironmentError:
+			pass

Modified: main/branches/prefix/pym/portage/update.py
===================================================================
--- main/branches/prefix/pym/portage/update.py	2009-09-26 18:43:40 UTC (rev 14436)
+++ main/branches/prefix/pym/portage/update.py	2009-09-26 18:44:55 UTC (rev 14437)
@@ -227,8 +227,6 @@
 				mode='r', encoding=_encodings['content'],
 				errors='replace').readlines()
 		except IOError:
-			if file_contents.has_key(x):
-				del file_contents[x]
 			continue
 
 	# update /etc/portage/packages.*

Modified: main/branches/prefix/pym/portage/util.py
===================================================================
--- main/branches/prefix/pym/portage/util.py	2009-09-26 18:43:40 UTC (rev 14436)
+++ main/branches/prefix/pym/portage/util.py	2009-09-26 18:44:55 UTC (rev 14437)
@@ -151,7 +151,7 @@
 	new_dl = {}
 	for key in myDict:
 		new_dl[key] = []
-		new_dl[key] = map(func,myDict[key])
+		new_dl[key] = [func(x) for x in myDict[key]]
 	return new_dl
 
 def stack_dictlist(original_dicts, incremental=0, incrementals=[], ignore_none=0):

Modified: main/branches/prefix/pym/repoman/utilities.py
===================================================================
--- main/branches/prefix/pym/repoman/utilities.py	2009-09-26 18:43:40 UTC (rev 14436)
+++ main/branches/prefix/pym/repoman/utilities.py	2009-09-26 18:44:55 UTC (rev 14437)
@@ -346,7 +346,7 @@
 	Args:
 		settings - portage.config instance, preferably repoman_settings
 	Returns:
-		tuple(portdir, portdir_overlay, location)
+		list(portdir, portdir_overlay, location)
 	"""
 
 	portdir = None
@@ -430,4 +430,4 @@
 	if not portdir.endswith('/'):
 		portdir += '/'
 
-	return map(normalize_path, (portdir, portdir_overlay, location))
+	return [normalize_path(x) for x in (portdir, portdir_overlay, location)]