[gentoo-commits] portage r13923 - in main/branches/prefix/pym: _emerge portage portage/dbapi

[Fabian Groffen (grobian)]

Author: grobian
Date: 2009-08-05 18:11:44 +0000 (Wed, 05 Aug 2009)
New Revision: 13923

Modified:
   main/branches/prefix/pym/_emerge/Binpkg.py
   main/branches/prefix/pym/_emerge/BinpkgVerifier.py
   main/branches/prefix/pym/_emerge/EbuildBuild.py
   main/branches/prefix/pym/_emerge/EbuildFetcher.py
   main/branches/prefix/pym/_emerge/EbuildPhase.py
   main/branches/prefix/pym/_emerge/PackageUninstall.py
   main/branches/prefix/pym/_emerge/changelog.py
   main/branches/prefix/pym/_emerge/depgraph.py
   main/branches/prefix/pym/_emerge/emergelog.py
   main/branches/prefix/pym/_emerge/main.py
   main/branches/prefix/pym/portage/__init__.py
   main/branches/prefix/pym/portage/dbapi/bintree.py
   main/branches/prefix/pym/portage/xpak.py
Log:
   Merged from trunk -r13909:13919

   | 13910   | Avoid UnicodeEncodeError with unicode package.mask comments. |
   | zmedico | Thanks to Thanks to Scott Moreau (soreau) for reporting.     |
   
   | 13911   | Fix bindbapi.aux_get and aux_update to work with             |
   | zmedico | py3k/unicode.                                                |
   
   | 13912   | Open all files in binary mode for py3k compatibility.        |
   | zmedico |                                                              |
   
   | 13913   | Open file in text mode (unicode) where appropriate.          |
   | zmedico |                                                              |
   
   | 13914   | In _check_build_log(), open the log in text mode (unicode).  |
   | zmedico |                                                              |
   
   | 13915   | Open ld.so.conf as text (unicode).                           |
   | zmedico |                                                              |
   
   | 13916   | Fix the FEATURES=parse-eapi-ebuild-head regex to handle      |
   | zmedico | comments on the same line, like this: EAPI=2 #foo Thanks to  |
   |         | Markus Meier <maekke@g.o> for reporting.              |
   
   | 13917   | s/utf8/utf_8/ for consistency                                |
   | zmedico |                                                              |
   
   | 13919   | Fix insert_optional_args() to properly handle thinks like    |
   | zmedico | -Dk which require multiple substitutions of default          |
   |         | arguments.                                                   |


Modified: main/branches/prefix/pym/_emerge/Binpkg.py
===================================================================
--- main/branches/prefix/pym/_emerge/Binpkg.py	2009-08-05 18:10:06 UTC (rev 13922)
+++ main/branches/prefix/pym/_emerge/Binpkg.py	2009-08-05 18:11:44 UTC (rev 13923)
@@ -11,6 +11,7 @@
 from _emerge.EbuildBuildDir import EbuildBuildDir
 from portage.util import writemsg
 # for an explanation on this logic, see pym/_emerge/__init__.py
+import codecs
 import os
 import sys
 if os.environ.__contains__("PORTAGE_PYTHONPATH"):
@@ -36,7 +37,8 @@
 
 		log_path = self.settings.get("PORTAGE_LOG_FILE")
 		if  log_path is not None:
-			f = open(log_path, 'a')
+			f = codecs.open(log_path, mode='a',
+				encoding='utf_8', errors='replace')
 			try:
 				f.write(msg)
 			finally:
@@ -229,7 +231,8 @@
 			else:
 				continue
 
-			f = open(os.path.join(infloc, k), 'wb')
+			f = codecs.open(os.path.join(infloc, k), mode='w',
+				encoding='utf_8', errors='replace')
 			try:
 				f.write(v + "\n")
 			finally:

Modified: main/branches/prefix/pym/_emerge/BinpkgVerifier.py
===================================================================
--- main/branches/prefix/pym/_emerge/BinpkgVerifier.py	2009-08-05 18:10:06 UTC (rev 13922)
+++ main/branches/prefix/pym/_emerge/BinpkgVerifier.py	2009-08-05 18:11:44 UTC (rev 13923)
@@ -6,6 +6,7 @@
 from portage.util import writemsg
 import sys
 # for an explanation on this logic, see pym/_emerge/__init__.py
+import codecs
 import os
 import sys
 if os.environ.__contains__("PORTAGE_PYTHONPATH"):
@@ -31,7 +32,8 @@
 		stderr_orig = sys.stderr
 		log_file = None
 		if self.background and self.logfile is not None:
-			log_file = open(self.logfile, 'a')
+			log_file = codecs.open(self.logfile, mode='a',
+				encoding='utf_8', errors='replace')
 		try:
 			if log_file is not None:
 				sys.stdout = log_file

Modified: main/branches/prefix/pym/_emerge/EbuildBuild.py
===================================================================
--- main/branches/prefix/pym/_emerge/EbuildBuild.py	2009-08-05 18:10:06 UTC (rev 13922)
+++ main/branches/prefix/pym/_emerge/EbuildBuild.py	2009-08-05 18:11:44 UTC (rev 13923)
@@ -12,6 +12,7 @@
 from _emerge.EbuildBuildDir import EbuildBuildDir
 from portage.util import writemsg
 # for an explanation on this logic, see pym/_emerge/__init__.py
+import codecs
 import os
 import sys
 if os.environ.__contains__("PORTAGE_PYTHONPATH"):
@@ -188,7 +189,8 @@
 
 			log_path = self.settings.get("PORTAGE_LOG_FILE")
 			if log_path is not None:
-				log_file = open(log_path, 'a')
+				log_file = codecs.open(log_path, mode='a',
+					encoding='utf_8', errors='replace')
 				try:
 					log_file.write(msg)
 				finally:

Modified: main/branches/prefix/pym/_emerge/EbuildFetcher.py
===================================================================
--- main/branches/prefix/pym/_emerge/EbuildFetcher.py	2009-08-05 18:10:06 UTC (rev 13922)
+++ main/branches/prefix/pym/_emerge/EbuildFetcher.py	2009-08-05 18:11:44 UTC (rev 13923)
@@ -12,6 +12,7 @@
 else:
 	sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(os.path.realpath(__file__))), "pym"))
 import portage
+import codecs
 import os
 from portage.elog.messages import eerror
 class EbuildFetcher(SpawnProcess):
@@ -92,7 +93,8 @@
 				elog_out = None
 				if self.logfile is not None:
 					if self.background:
-						elog_out = open(self.logfile, 'a')
+						elog_out = codecs.open(self.logfile, mode='a',
+							encoding='utf_8', errors='replace')
 				msg = "Fetch failed for '%s'" % (self.pkg.cpv,)
 				if self.logfile is not None:
 					msg += ", Log file:"

Modified: main/branches/prefix/pym/_emerge/EbuildPhase.py
===================================================================
--- main/branches/prefix/pym/_emerge/EbuildPhase.py	2009-08-05 18:10:06 UTC (rev 13922)
+++ main/branches/prefix/pym/_emerge/EbuildPhase.py	2009-08-05 18:11:44 UTC (rev 13923)
@@ -7,6 +7,7 @@
 from _emerge.CompositeTask import CompositeTask
 from portage.util import writemsg
 # for an explanation on this logic, see pym/_emerge/__init__.py
+import codecs
 import os
 import sys
 if os.environ.__contains__("PORTAGE_PYTHONPATH"):
@@ -36,7 +37,8 @@
 			log_path = self.settings.get("PORTAGE_LOG_FILE")
 			log_file = None
 			if self.background and log_path is not None:
-				log_file = open(log_path, 'a')
+				log_file = codecs.open(log_path, mode='a',
+					encoding='utf_8', errors='replace')
 				out = log_file
 			try:
 				portage._check_build_log(self.settings, out=out)

Modified: main/branches/prefix/pym/_emerge/PackageUninstall.py
===================================================================
--- main/branches/prefix/pym/_emerge/PackageUninstall.py	2009-08-05 18:10:06 UTC (rev 13922)
+++ main/branches/prefix/pym/_emerge/PackageUninstall.py	2009-08-05 18:11:44 UTC (rev 13923)
@@ -2,6 +2,7 @@
 # Distributed under the terms of the GNU General Public License v2
 # $Id$
 
+import codecs
 import logging
 # for an explanation on this logic, see pym/_emerge/__init__.py
 import os
@@ -46,7 +47,8 @@
 				portage.util.writemsg_level(msg,
 					level=level, noiselevel=noiselevel)
 
-			f = open(log_path, 'a')
+			f = codecs.open(log_path, mode='a',
+				encoding='utf_8', errors='replace')
 			try:
 				f.write(msg)
 			finally:

Modified: main/branches/prefix/pym/_emerge/changelog.py
===================================================================
--- main/branches/prefix/pym/_emerge/changelog.py	2009-08-05 18:10:06 UTC (rev 13922)
+++ main/branches/prefix/pym/_emerge/changelog.py	2009-08-05 18:11:44 UTC (rev 13923)
@@ -2,6 +2,7 @@
 # Distributed under the terms of the GNU General Public License v2
 # $Id$
 
+import codecs
 import os
 import re
 
@@ -25,7 +26,8 @@
 		next = next[:-3]
 	changelogpath = os.path.join(os.path.split(ebuildpath)[0],'ChangeLog')
 	try:
-		changelog = open(changelogpath).read()
+		changelog = codecs.open(changelogpath, mode='r',
+			encoding='utf_8', errors='replace').read()
 	except SystemExit, e:
 		raise # Needed else can't exit
 	except:

Modified: main/branches/prefix/pym/_emerge/depgraph.py
===================================================================
--- main/branches/prefix/pym/_emerge/depgraph.py	2009-08-05 18:10:06 UTC (rev 13922)
+++ main/branches/prefix/pym/_emerge/depgraph.py	2009-08-05 18:11:44 UTC (rev 13923)
@@ -4356,6 +4356,14 @@
 		if "--changelog" in self._frozen_config.myopts:
 			print
 			for revision,text in changelogs:
+
+				if sys.hexversion < 0x3000000:
+					# avoid potential UnicodeEncodeError
+					if isinstance(revision, unicode):
+						revision = revision.encode('utf_8', 'replace')
+					if isinstance(text, unicode):
+						text = text.encode('utf_8', 'replace')
+
 				print bold('*'+revision)
 				sys.stdout.write(text)
 
@@ -5113,6 +5121,11 @@
 				pass
 
 		print "- "+cpv+" (masked by: "+", ".join(mreasons)+")"
+
+		if sys.hexversion < 0x3000000 and isinstance(comment, unicode):
+			# avoid potential UnicodeEncodeError
+			comment = comment.encode('utf_8', 'replace')
+
 		if comment and comment not in shown_comments:
 			print filename+":"
 			print comment

Modified: main/branches/prefix/pym/_emerge/emergelog.py
===================================================================
--- main/branches/prefix/pym/_emerge/emergelog.py	2009-08-05 18:10:06 UTC (rev 13922)
+++ main/branches/prefix/pym/_emerge/emergelog.py	2009-08-05 18:11:44 UTC (rev 13923)
@@ -2,6 +2,7 @@
 # Distributed under the terms of the GNU General Public License v2
 # $Id$
 
+import codecs
 import os
 import sys
 import time
@@ -27,7 +28,8 @@
 		xtermTitle(short_msg)
 	try:
 		file_path = os.path.join(_emerge_log_dir, 'emerge.log')
-		mylogfile = open(file_path, "a")
+		mylogfile = codecs.open(file_path, mode='a',
+			encoding='utf_8', errors='replace')
 		portage.util.apply_secpass_permissions(file_path,
 			uid=portage.portage_uid, gid=portage.portage_gid,
 			mode=0660)

Modified: main/branches/prefix/pym/_emerge/main.py
===================================================================
--- main/branches/prefix/pym/_emerge/main.py	2009-08-05 18:10:06 UTC (rev 13922)
+++ main/branches/prefix/pym/_emerge/main.py	2009-08-05 18:11:44 UTC (rev 13923)
@@ -487,7 +487,9 @@
 			new_args.append(opt_arg)
 
 		if saved_opts is not None:
-			new_args.append("-" + saved_opts)
+			# Recycle these on arg_stack since they
+			# might contain another match.
+			arg_stack.append("-" + saved_opts)
 
 	return new_args
 

Modified: main/branches/prefix/pym/portage/__init__.py
===================================================================
--- main/branches/prefix/pym/portage/__init__.py	2009-08-05 18:10:06 UTC (rev 13922)
+++ main/branches/prefix/pym/portage/__init__.py	2009-08-05 18:11:44 UTC (rev 13923)
@@ -727,7 +727,8 @@
 
 	ldsoconf_path = os.path.join(target_root, EPREFIX_LSTRIP, "etc", "ld.so.conf")
 	try:
-		myld = open(ldsoconf_path)
+		myld = codecs.open(ldsoconf_path, mode='r',
+			encoding='utf_8', errors='replace')
 		myldlines=myld.readlines()
 		myld.close()
 		oldld=[]
@@ -5034,7 +5035,8 @@
 	if logfile is None:
 		return
 	try:
-		f = open(logfile)
+		f = codecs.open(logfile, mode='r',
+			encoding='utf_8', errors='replace')
 	except EnvironmentError:
 		return
 
@@ -5295,7 +5297,7 @@
 # the ebuild.
 _validate_cache_for_unsupported_eapis = True
 
-_parse_eapi_ebuild_head_re = re.compile(r'^EAPI=[\'"]?([^\'"]*)')
+_parse_eapi_ebuild_head_re = re.compile(r'^EAPI=[\'"]?([^\'"#]*)')
 _parse_eapi_ebuild_head_max_lines = 30
 
 def _parse_eapi_ebuild_head(f):

Modified: main/branches/prefix/pym/portage/dbapi/bintree.py
===================================================================
--- main/branches/prefix/pym/portage/dbapi/bintree.py	2009-08-05 18:10:06 UTC (rev 13922)
+++ main/branches/prefix/pym/portage/dbapi/bintree.py	2009-08-05 18:11:44 UTC (rev 13923)
@@ -25,6 +25,7 @@
 import codecs
 import os, errno, stat
 import re
+import sys
 from itertools import chain, izip
 
 class bindbapi(fakedbapi):
@@ -68,7 +69,12 @@
 			tbz2_path = self.bintree.getname(mycpv)
 			if not os.path.exists(tbz2_path):
 				raise KeyError(mycpv)
-			getitem = portage.xpak.tbz2(tbz2_path).getfile
+			tbz2 = portage.xpak.tbz2(tbz2_path)
+			def getitem(k):
+				v = tbz2.getfile(k)
+				if v is not None and not isinstance(v, unicode):
+					v = unicode(v, encoding='utf_8', errors='replace')
+				return v
 		else:
 			getitem = self.bintree._remotepkgs[mycpv].get
 		mydata = {}
@@ -81,15 +87,16 @@
 			# or the tbz2 is corrupt.
 			if myval:
 				mydata[x] = " ".join(myval.split())
-		if "EAPI" in mykeys:
-			if not mydata.setdefault("EAPI", "0"):
-				mydata["EAPI"] = "0"
+
+		if not mydata.setdefault('EAPI', u'0'):
+			mydata['EAPI'] = u'0'
+
 		if cache_me:
 			aux_cache = self._aux_cache_slot_dict()
 			for x in self._aux_cache_keys:
-				aux_cache[x] = mydata.get(x, "")
+				aux_cache[x] = mydata.get(x, u'')
 			self._aux_cache[mycpv] = aux_cache
-		return [mydata.get(x, "") for x in wants]
+		return [mydata.get(x, u'') for x in wants]
 
 	def aux_update(self, cpv, values):
 		if not self.bintree.populated:
@@ -99,7 +106,23 @@
 			raise KeyError(cpv)
 		mytbz2 = portage.xpak.tbz2(tbz2path)
 		mydata = mytbz2.get_data()
-		mydata.update(values)
+
+		if sys.hexversion < 0x3000000:
+			for k, v in values.iteritems():
+				if isinstance(k, unicode):
+					k = k.encode('utf_8', 'replace')
+				if isinstance(v, unicode):
+					v = v.encode('utf_8', 'replace')
+				mydata[k] = v
+
+		else:
+			for k, v in values.iteritems():
+				if isinstance(k, str):
+					k = k.encode('utf_8', 'replace')
+				if isinstance(v, str):
+					v = v.encode('utf_8', 'replace')
+				mydata[k] = v
+
 		for k, v in mydata.items():
 			if not v:
 				del mydata[k]
@@ -1083,7 +1106,7 @@
 		pkgindex = self._new_pkgindex()
 		try:
 			f = codecs.open(self._pkgindex_file, 
-				encoding='utf8', errors='replace')
+				encoding='utf_8', errors='replace')
 		except EnvironmentError:
 			pass
 		else:

Modified: main/branches/prefix/pym/portage/xpak.py
===================================================================
--- main/branches/prefix/pym/portage/xpak.py	2009-08-05 18:10:06 UTC (rev 13922)
+++ main/branches/prefix/pym/portage/xpak.py	2009-08-05 18:11:44 UTC (rev 13923)
@@ -68,14 +68,14 @@
 	mylist.sort()
 	mydata = {}
 	for x in mylist:
-		a = open(x, "r")
+		a = open(x, 'rb')
 		mydata[x] = a.read()
 		a.close()
 	os.chdir(origdir)
 
 	xpak_segment = xpak_mem(mydata)
 	if outfile:
-		outf = open(outfile, "w")
+		outf = open(outfile, 'wb')
 		outf.write(xpak_segment)
 		outf.close()
 	else:
@@ -104,7 +104,7 @@
 	"""(infile) -- Splits the infile into two files.
 	'infile.index' contains the index segment.
 	'infile.dat' contails the data segment."""
-	myfile=open(infile,"r")
+	myfile = open(infile, 'rb')
 	mydat=myfile.read()
 	myfile.close()
 	
@@ -112,10 +112,10 @@
 	if not splits:
 		return False
 	
-	myfile=open(infile+".index","w")
+	myfile = open(infile + '.index', 'wb')
 	myfile.write(splits[0])
 	myfile.close()
-	myfile=open(infile+".dat","w")
+	myfile = open(infile + '.dat', 'wb')
 	myfile.write(splits[1])
 	myfile.close()
 	return True
@@ -130,7 +130,7 @@
 
 def getindex(infile):
 	"""(infile) -- grabs the index segment from the infile and returns it."""
-	myfile=open(infile,"r")
+	myfile = open(infile, 'rb')
 	myheader=myfile.read(16)
 	if myheader[0:8]!="XPAKPACK":
 		myfile.close()
@@ -143,7 +143,7 @@
 def getboth(infile):
 	"""(infile) -- grabs the index and data segments from the infile.
 	Returns an array [indexSegment,dataSegment]"""
-	myfile=open(infile,"r")
+	myfile = open(infile, 'rb')
 	myheader=myfile.read(16)
 	if myheader[0:8]!="XPAKPACK":
 		myfile.close()
@@ -217,7 +217,7 @@
 		if dirname:
 			if not os.path.exists(dirname):
 				os.makedirs(dirname)
-		mydat=open(myname,"w")
+		mydat = open(myname, 'wb')
 		mydat.write(mydata[datapos:datapos+datalen])
 		mydat.close()
 		startpos=startpos+namelen+12
@@ -262,7 +262,7 @@
 
 	def recompose_mem(self, xpdata):
 		self.scan() # Don't care about condition... We'll rewrite the data anyway.
-		myfile=open(self.file,"a+")
+		myfile = open(self.file, 'ab+')
 		if not myfile:
 			raise IOError
 		myfile.seek(-self.xpaksize,2) # 0,2 or -0,2 just mean EOF.
@@ -298,7 +298,7 @@
 				if not changed:
 					return 1
 			self.filestat=mystat
-			a=open(self.file,"r")
+			a = open(self.file, 'rb')
 			a.seek(-16,2)
 			trailer=a.read()
 			self.infosize=0
@@ -341,7 +341,7 @@
 		myresult=searchindex(self.index,myfile)
 		if not myresult:
 			return mydefault
-		a=open(self.file,"r")
+		a = open(self.file, 'rb')
 		a.seek(self.datapos+myresult[0],0)
 		myreturn=a.read(myresult[1])
 		a.close()
@@ -365,7 +365,7 @@
 		except:
 			os.chdir("/")
 			origdir="/"
-		a=open(self.file,"r")
+		a = open(self.file, 'rb')
 		if not os.path.exists(mydest):
 			os.makedirs(mydest)
 		os.chdir(mydest)
@@ -379,7 +379,7 @@
 			if dirname:
 				if not os.path.exists(dirname):
 					os.makedirs(dirname)
-			mydat=open(myname,"w")
+			mydat = open(myname, 'wb')
 			a.seek(self.datapos+datapos)
 			mydat.write(a.read(datalen))
 			mydat.close()
@@ -392,7 +392,7 @@
 		"""Returns all the files from the dataSegment as a map object."""
 		if not self.scan():
 			return 0
-		a = open(self.file, "r")
+		a = open(self.file, 'rb')
 		mydata = {}
 		startpos=0
 		while ((startpos+8)<self.indexsize):
@@ -411,7 +411,7 @@
 		if not self.scan():
 			return None
 
-		a = open(self.file,"r")
+		a = open(self.file, 'rb')
 		a.seek(self.datapos)
 		mydata =a.read(self.datasize)
 		a.close()