[gentoo-commits] portage r14138 - in main/branches/prefix: bin bin/ebuild-helpers pym/_emerge pym/portage pym/portage/cache pym/portage/elog pym/portage/env pym/portage/sets pym/portage/tests pym/portage/tests/xpak

[Fabian Groffen (grobian)]

Author: grobian
Date: 2009-08-24 09:28:51 +0000 (Mon, 24 Aug 2009)
New Revision: 14138

Modified:
   main/branches/prefix/bin/ebuild-helpers/dohtml
   main/branches/prefix/bin/ebuild.sh
   main/branches/prefix/pym/_emerge/MergeListItem.py
   main/branches/prefix/pym/_emerge/PackageUninstall.py
   main/branches/prefix/pym/_emerge/Scheduler.py
   main/branches/prefix/pym/_emerge/unmerge.py
   main/branches/prefix/pym/portage/__init__.py
   main/branches/prefix/pym/portage/cache/ebuild_xattr.py
   main/branches/prefix/pym/portage/elog/__init__.py
   main/branches/prefix/pym/portage/env/loaders.py
   main/branches/prefix/pym/portage/sets/files.py
   main/branches/prefix/pym/portage/tests/__init__.py
   main/branches/prefix/pym/portage/tests/xpak/test_decodeint.py
   main/branches/prefix/pym/portage/update.py
   main/branches/prefix/pym/portage/util.py
   main/branches/prefix/pym/portage/xpak.py
Log:
   Merged from trunk -r14067:14077

   | 14068    | Use elog in _eapi0_pkg_nofetch().                           |
   | zmedico  |                                                             |
   
   | 14069    | sets/files.py cleanPackages function stop calling lock and  |
   | volkmar  | load and requires the caller to do that changing unmerge to |
   |          | reflect this change                                         |
   
   | 14070    | Scheduler is now able to clean world set when removing a    |
   | volkmar  | package. world_atom function has been updated and           |
   |          | PackageUninstall is calling it after unmerge.               |
   
   | 14071    | Use a clean listener system for portage.elog instead of     |
   | volkmar  | _emerge_elog_listener                                       |
   
   | 14072    | Use _content_encoding and _fs_encoding for unicode          |
   | zmedico  | encoding/decoding.                                          |
   
   | 14073    | When _unicode_func_wrapper() decodes a string in a returned |
   | zmedico  | list (typically from os.listdir), discard values with       |
   |          | invalid encoding. This insures that all names returned from |
   |          | all os.listdir() calls are valid.                           |
   
   | 14074    | Use portage.os and _fs_encoding where appropriate, and fix  |
   | zmedico  | binary string handling for py3k compat.                     |
   
   | 14075    | Print a warning when nonexistent files have been passed to  |
   | arfrever | dohtml.                                                     |
   
   | 14076    | Add 'return False' which was missing from the previous      |
   | arfrever | commit.                                                     |
   
   | 14077    | Test the edge case.                                         |
   | zmedico  |                                                             |


Modified: main/branches/prefix/bin/ebuild-helpers/dohtml
===================================================================
--- main/branches/prefix/bin/ebuild-helpers/dohtml	2009-08-24 09:27:07 UTC (rev 14137)
+++ main/branches/prefix/bin/ebuild-helpers/dohtml	2009-08-24 09:28:51 UTC (rev 14138)
@@ -59,7 +59,10 @@
 	else:
 		destdir = options.ED + "/usr/share/doc/" + options.PF + "/html/" + options.doc_prefix + "/" + prefix
 
-	if os.path.isfile(fullpath):
+	if not os.path.exists(fullpath):
+		sys.stderr.write("!!! dohtml: %s does not exist\n" % fullpath)
+		return False
+	elif os.path.isfile(fullpath):
 		ext = os.path.splitext(basename)[1]
 		if (len(ext) and ext[1:] in options.allowed_exts) or basename in options.allowed_files:
 			dodir(destdir)

Modified: main/branches/prefix/bin/ebuild.sh
===================================================================
--- main/branches/prefix/bin/ebuild.sh	2009-08-24 09:27:07 UTC (rev 14137)
+++ main/branches/prefix/bin/ebuild.sh	2009-08-24 09:28:51 UTC (rev 14138)
@@ -588,10 +588,10 @@
 _eapi0_pkg_nofetch() {
 	[ -z "${SRC_URI}" ] && return
 
-	echo "!!! The following are listed in SRC_URI for ${PN}:"
+	elog "The following are listed in SRC_URI for ${PN}:"
 	local x
 	for x in $(echo ${SRC_URI}); do
-		echo "!!!   ${x}"
+		elog "   ${x}"
 	done
 }
 

Modified: main/branches/prefix/pym/_emerge/MergeListItem.py
===================================================================
--- main/branches/prefix/pym/_emerge/MergeListItem.py	2009-08-24 09:27:07 UTC (rev 14137)
+++ main/branches/prefix/pym/_emerge/MergeListItem.py	2009-08-24 09:28:51 UTC (rev 14138)
@@ -132,7 +132,8 @@
 
 				uninstall = PackageUninstall(background=self.background,
 					ldpath_mtimes=ldpath_mtimes, opts=self.emerge_opts,
-					pkg=pkg, scheduler=scheduler, settings=settings)
+					pkg=pkg, scheduler=scheduler, settings=settings,
+					world_atom=world_atom)
 
 				uninstall.start()
 				retval = uninstall.wait()

Modified: main/branches/prefix/pym/_emerge/PackageUninstall.py
===================================================================
--- main/branches/prefix/pym/_emerge/PackageUninstall.py	2009-08-24 09:27:07 UTC (rev 14137)
+++ main/branches/prefix/pym/_emerge/PackageUninstall.py	2009-08-24 09:28:51 UTC (rev 14138)
@@ -12,11 +12,12 @@
 
 class PackageUninstall(AsynchronousTask):
 
-	__slots__ = ("ldpath_mtimes", "opts", "pkg", "scheduler", "settings")
+	__slots__ = ("world_atom", "ldpath_mtimes", "opts",
+			"pkg", "scheduler", "settings")
 
 	def _start(self):
 		try:
-			unmerge(self.pkg.root_config, self.opts, "unmerge",
+			retval = unmerge(self.pkg.root_config, self.opts, "unmerge",
 				[self.pkg.cpv], self.ldpath_mtimes, clean_world=0,
 				clean_delay=0, raise_on_error=1, scheduler=self.scheduler,
 				writemsg_level=self._writemsg_level)
@@ -24,6 +25,10 @@
 			self.returncode = e.status
 		else:
 			self.returncode = os.EX_OK
+
+		if retval == 1:
+			self.world_atom(self.pkg)
+
 		self.wait()
 
 	def _writemsg_level(self, msg, level=0, noiselevel=0):

Modified: main/branches/prefix/pym/_emerge/Scheduler.py
===================================================================
--- main/branches/prefix/pym/_emerge/Scheduler.py	2009-08-24 09:27:07 UTC (rev 14137)
+++ main/branches/prefix/pym/_emerge/Scheduler.py	2009-08-24 09:28:51 UTC (rev 14138)
@@ -1108,7 +1108,7 @@
 		pkg_queue = self._pkg_queue
 		failed_pkgs = self._failed_pkgs
 		portage.locks._quiet = self._background
-		portage.elog._emerge_elog_listener = self._elog_listener
+		portage.elog.add_listener(self._elog_listener)
 		rval = os.EX_OK
 
 		try:
@@ -1116,7 +1116,7 @@
 		finally:
 			self._main_loop_cleanup()
 			portage.locks._quiet = False
-			portage.elog._emerge_elog_listener = None
+			portage.elog.remove_listener(self._elog_listener)
 			if failed_pkgs:
 				rval = failed_pkgs[-1].returncode
 
@@ -1566,8 +1566,8 @@
 
 	def _world_atom(self, pkg):
 		"""
-		Add the package to the world file, but only if
-		it's supposed to be added. Otherwise, do nothing.
+		Add or remove the package to the world file, but only if
+		it's supposed to be added or removed. Otherwise, do nothing.
 		"""
 
 		if set(("--buildpkgonly", "--fetchonly",
@@ -1596,17 +1596,25 @@
 			if hasattr(world_set, "load"):
 				world_set.load() # maybe it's changed on disk
 
-			atom = create_world_atom(pkg, args_set, root_config)
-			if atom:
-				if hasattr(world_set, "add"):
-					self._status_msg(('Recording %s in "world" ' + \
-						'favorites file...') % atom)
-					logger.log(" === (%s of %s) Updating world file (%s)" % \
-						(pkg_count.curval, pkg_count.maxval, pkg.cpv))
-					world_set.add(atom)
-				else:
-					writemsg_level('\n!!! Unable to record %s in "world"\n' % \
-						(atom,), level=logging.WARN, noiselevel=-1)
+			if pkg.operation == "uninstall":
+				if hasattr(world_set, "cleanPackage"):
+					world_set.cleanPackage(pkg.root_config.trees["vartree"].dbapi,
+							pkg.cpv)
+				if hasattr(world_set, "remove"):
+					for s in pkg.root_config.setconfig.active:
+						world_set.remove(SETPREFIX+s)
+			else:
+				atom = create_world_atom(pkg, args_set, root_config)
+				if atom:
+					if hasattr(world_set, "add"):
+						self._status_msg(('Recording %s in "world" ' + \
+							'favorites file...') % atom)
+						logger.log(" === (%s of %s) Updating world file (%s)" % \
+							(pkg_count.curval, pkg_count.maxval, pkg.cpv))
+						world_set.add(atom)
+					else:
+						writemsg_level('\n!!! Unable to record %s in "world"\n' % \
+							(atom,), level=logging.WARN, noiselevel=-1)
 		finally:
 			if world_locked:
 				world_set.unlock()

Modified: main/branches/prefix/pym/_emerge/unmerge.py
===================================================================
--- main/branches/prefix/pym/_emerge/unmerge.py	2009-08-24 09:27:07 UTC (rev 14137)
+++ main/branches/prefix/pym/_emerge/unmerge.py	2009-08-24 09:28:51 UTC (rev 14138)
@@ -510,11 +510,22 @@
 					raise UninstallFailure(retval)
 				sys.exit(retval)
 			else:
-				if clean_world and hasattr(sets["world"], "cleanPackage"):
+				if clean_world and hasattr(sets["world"], "cleanPackage")\
+						and hasattr(sets["world"], "lock"):
+					sets["world"].lock()
+					if hasattr(sets["world"], "load"):
+						sets["world"].load()
 					sets["world"].cleanPackage(vartree.dbapi, y)
+					sets["world"].unlock()
 				emergelog(xterm_titles, " >>> unmerge success: "+y)
-	if clean_world and hasattr(sets["world"], "remove"):
+
+	if clean_world and hasattr(sets["world"], "remove")\
+			and hasattr(sets["world"], "lock"):
+		sets["world"].lock()
+		# load is called inside remove()
 		for s in root_config.setconfig.active:
 			sets["world"].remove(SETPREFIX+s)
+		sets["world"].unlock()
+
 	return 1
 

Modified: main/branches/prefix/pym/portage/__init__.py
===================================================================
--- main/branches/prefix/pym/portage/__init__.py	2009-08-24 09:27:07 UTC (rev 14137)
+++ main/branches/prefix/pym/portage/__init__.py	2009-08-24 09:28:51 UTC (rev 14138)
@@ -169,12 +169,21 @@
 		if isinstance(rval, (basestring, list, tuple)):
 			if isinstance(rval, basestring):
 				rval = _unicode_decode(rval, encoding=encoding)
-			elif isinstance(rval, list):
-				rval = [_unicode_decode(x, encoding=encoding) for x in rval]
-			elif isinstance(rval, tuple):
-				rval = tuple(_unicode_decode(x, encoding=encoding) \
-					for x in rval)
+			else:
+				decoded_rval = []
+				for x in rval:
+					try:
+						x = _unicode_decode(x, encoding=encoding, errors='strict')
+					except UnicodeDecodeError:
+						pass
+					else:
+						decoded_rval.append(x)
 
+				if isinstance(rval, tuple):
+					rval = tuple(decoded_rval)
+				else:
+					rval = decoded_rval
+
 		return rval
 
 class _unicode_module_wrapper(object):

Modified: main/branches/prefix/pym/portage/cache/ebuild_xattr.py
===================================================================
--- main/branches/prefix/pym/portage/cache/ebuild_xattr.py	2009-08-24 09:27:07 UTC (rev 14137)
+++ main/branches/prefix/pym/portage/cache/ebuild_xattr.py	2009-08-24 09:28:51 UTC (rev 14138)
@@ -10,6 +10,7 @@
 from portage.versions import catsplit
 from portage import cpv_getkey
 from portage import os
+from portage import _fs_encoding
 from portage import _unicode_decode
 import xattr
 from errno import ENODATA,ENOSPC,E2BIG
@@ -156,7 +157,11 @@
 
 		for root, dirs, files in os.walk(self.portdir):
 			for file in files:
-				file = _unicode_decode(file)
+				try:
+					file = _unicode_decode(file,
+						encoding=_fs_encoding, errors='strict')
+				except UnicodeDecodeError:
+					continue
 				if file[-7:] == '.ebuild':
 					cat = os.path.basename(os.path.dirname(root))
 					pn_pv = file[:-7]

Modified: main/branches/prefix/pym/portage/elog/__init__.py
===================================================================
--- main/branches/prefix/pym/portage/elog/__init__.py	2009-08-24 09:27:07 UTC (rev 14137)
+++ main/branches/prefix/pym/portage/elog/__init__.py	2009-08-24 09:28:51 UTC (rev 14138)
@@ -56,11 +56,23 @@
 		_elog_mod_imports[name] = m
 	return m
 
-_emerge_elog_listener = None
+_elog_listeners = []
+def add_listener(listener):
+	'''
+	Listeners should accept four arguments: settings, key, logentries and logtext
+	'''
+	_elog_listeners.append(listener)
+
+def remove_listener(listener):
+	'''
+	Remove previously added listener
+	'''
+	_elog_listeners.remove(listener)
+
 _elog_atexit_handlers = []
 _preserve_logentries = {}
 def elog_process(cpv, mysettings, phasefilter=None):
-	global _elog_atexit_handlers, _emerge_elog_listener, _preserve_logentries
+	global _elog_atexit_handlers, _preserve_logentries
 	
 	logsystems = mysettings.get("PORTAGE_ELOG_SYSTEM","").split()
 	for s in logsystems:
@@ -123,9 +135,9 @@
 
 		default_fulllog = _combine_logentries(default_logentries)
 
-		if _emerge_elog_listener is not None:
-			_emerge_elog_listener(mysettings, str(key),
-				default_logentries, default_fulllog)
+		# call listeners
+		for listener in _elog_listeners:
+			listener(mysettings, str(key), default_logentries, default_fulllog)
 
 		# pass the processing to the individual modules
 		for s, levels in logsystems.iteritems():

Modified: main/branches/prefix/pym/portage/env/loaders.py
===================================================================
--- main/branches/prefix/pym/portage/env/loaders.py	2009-08-24 09:27:07 UTC (rev 14137)
+++ main/branches/prefix/pym/portage/env/loaders.py	2009-08-24 09:28:51 UTC (rev 14138)
@@ -4,8 +4,13 @@
 # $Id$
 
 import codecs
-import os
+import errno
 import stat
+from portage import os
+from portage import _content_encoding
+from portage import _fs_encoding
+from portage import _unicode_decode
+from portage import _unicode_encode
 from portage.localization import _
 
 class LoaderError(Exception):
@@ -40,11 +45,6 @@
 	@returns: List of files to process
 	"""
 
-	if isinstance(filename, unicode):
-		# Avoid UnicodeDecodeError raised from
-		# os.path.join when called by os.walk.
-		filename = filename.encode('utf_8', 'replace')
-
 	try:
 		st = os.stat(filename)
 	except OSError:
@@ -55,6 +55,11 @@
 				if d[:1] == '.' or d == 'CVS':
 					dirs.remove(d)
 			for f in files:
+				try:
+					f = _unicode_decode(f,
+						encoding=_fs_encoding, errors='strict')
+				except UnicodeDecodeError:
+					continue
 				if f[:1] == '.' or f[-1:] == '~':
 					continue
 				yield os.path.join(root, f)
@@ -145,9 +150,18 @@
 		# once, which may be expensive due to digging in child classes.
 		func = self.lineParser
 		for fn in RecursiveFileLoader(self.fname):
-			f = codecs.open(fn, mode='r', encoding='utf_8', errors='replace')
+			try:
+				f = codecs.open(_unicode_encode(fn,
+					encoding=_fs_encoding, errors='strict'), mode='r',
+					encoding=_content_encoding, errors='replace')
+			except EnvironmentError, e:
+				if e.errno not in (errno.ENOENT, errno.ESTALE):
+					raise
+				del e
+				continue
 			for line_num, line in enumerate(f):
 				func(line, line_num, data, errors)
+			f.close()
 		return (data, errors)
 
 	def lineParser(self, line, line_num, data, errors):

Modified: main/branches/prefix/pym/portage/sets/files.py
===================================================================
--- main/branches/prefix/pym/portage/sets/files.py	2009-08-24 09:27:07 UTC (rev 14137)
+++ main/branches/prefix/pym/portage/sets/files.py	2009-08-24 09:28:51 UTC (rev 14138)
@@ -293,8 +293,13 @@
 		self._lock = None
 
 	def cleanPackage(self, vardb, cpv):
-		self.lock()
-		self._load() # loads latest from disk
+		'''
+		Before calling this function you should call lock and load.
+		After calling this function you should call unlock.
+		'''
+		if not self._lock:
+			raise AssertionError('cleanPackage needs the set to be locked')
+
 		worldlist = list(self._atoms)
 		mykey = cpv_getkey(cpv)
 		newworldlist = []
@@ -316,7 +321,6 @@
 
 		newworldlist.extend(self._nonatoms)
 		self.replace(newworldlist)
-		self.unlock()
 
 	def singleBuilder(self, options, settings, trees):
 		return WorldSet(settings["ROOT"])

Modified: main/branches/prefix/pym/portage/tests/__init__.py
===================================================================
--- main/branches/prefix/pym/portage/tests/__init__.py	2009-08-24 09:27:07 UTC (rev 14137)
+++ main/branches/prefix/pym/portage/tests/__init__.py	2009-08-24 09:28:51 UTC (rev 14138)
@@ -3,14 +3,21 @@
 # Distributed under the terms of the GNU General Public License v2
 # $Id$
 
-import os
 import sys
 import time
 import unittest
 
+from portage import os
+from portage import _fs_encoding
+from portage import _unicode_encode
+from portage import _unicode_decode
+
 def main():
 
-	TEST_FILE = '__test__'
+	TEST_FILE = _unicode_encode('__test__',
+		encoding=_fs_encoding, errors='strict')
+	svn_dirname = _unicode_encode('.svn',
+		encoding=_fs_encoding, errors='strict')
 	suite = unittest.TestSuite()
 	basedir = os.path.dirname(os.path.realpath(__file__))
 	testDirs = []
@@ -19,8 +26,14 @@
 	# I was tired of adding dirs to the list, so now we add __test__
 	# to each dir we want tested.
 	for root, dirs, files in os.walk(basedir):
-		if ".svn" in dirs:
-			dirs.remove('.svn')
+		if svn_dirname in dirs:
+			dirs.remove(svn_dirname)
+		try:
+			root = _unicode_decode(root,
+				encoding=_fs_encoding, errors='strict')
+		except UnicodeDecodeError:
+			continue
+
 		if TEST_FILE in files:
 			testDirs.append(root)
 

Modified: main/branches/prefix/pym/portage/tests/xpak/test_decodeint.py
===================================================================
--- main/branches/prefix/pym/portage/tests/xpak/test_decodeint.py	2009-08-24 09:27:07 UTC (rev 14137)
+++ main/branches/prefix/pym/portage/tests/xpak/test_decodeint.py	2009-08-24 09:28:51 UTC (rev 14138)
@@ -12,3 +12,6 @@
 		
 		for n in xrange(1000):
 			self.assertEqual(decodeint(encodeint(n)), n)
+
+		for n in (2 ** 32 - 1,):
+			self.assertEqual(decodeint(encodeint(n)), n)

Modified: main/branches/prefix/pym/portage/update.py
===================================================================
--- main/branches/prefix/pym/portage/update.py	2009-08-24 09:27:07 UTC (rev 14137)
+++ main/branches/prefix/pym/portage/update.py	2009-08-24 09:28:51 UTC (rev 14138)
@@ -2,8 +2,16 @@
 # Distributed under the terms of the GNU General Public License v2
 # $Id$
 
-import errno, os, re, sys
+import codecs
+import errno
+import re
+import sys
 
+from portage import os
+from portage import _content_encoding
+from portage import _fs_encoding
+from portage import _unicode_decode
+from portage import _unicode_encode
 import portage
 portage.proxy.lazyimport.lazyimport(globals(),
 	'portage.dep:dep_getkey,get_operator,isvalidatom,isjustname,remove_slot',
@@ -12,7 +20,7 @@
 	'portage.versions:ververify'
 )
 
-from portage.const import USER_CONFIG_PATH, WORLD_FILE
+from portage.const import USER_CONFIG_PATH
 from portage.exception import DirectoryNotFound, PortageException
 from portage.localization import _
 
@@ -63,9 +71,10 @@
 	mydata = {}
 	for myfile in [f for f in os.listdir(dbdir) if f not in ignored_dbentries]:
 		file_path = os.path.join(dbdir, myfile)
-		f = open(file_path, "r")
-		mydata[myfile] = f.read()
-		f.close()
+		mydata[myfile] = codecs.open(_unicode_encode(file_path,
+			encoding=_fs_encoding, errors='strict'),
+			mode='r', encoding=_content_encoding,
+			errors='replace').read()
 	updated_items = update_dbentries(update_iter, mydata)
 	for myfile, mycontent in updated_items.iteritems():
 		file_path = os.path.join(dbdir, myfile)
@@ -100,9 +109,9 @@
 		mystat = os.stat(file_path)
 		if file_path not in prev_mtimes or \
 		long(prev_mtimes[file_path]) != long(mystat.st_mtime):
-			f = open(file_path)
-			content = f.read()
-			f.close()
+			content = codecs.open(_unicode_encode(file_path,
+				encoding=_fs_encoding, errors='strict'),
+				mode='r', encoding=_content_encoding, errors='replace').read()
 			update_data.append((file_path, mystat, content))
 	return update_data
 
@@ -142,17 +151,12 @@
 	return myupd, errors
 
 def update_config_files(config_root, protect, protect_mask, update_iter):
-	"""Perform global updates on /etc/portage/package.* and the world file.
+	"""Perform global updates on /etc/portage/package.*.
 	config_root - location of files to update
 	protect - list of paths from CONFIG_PROTECT
 	protect_mask - list of paths from CONFIG_PROTECT_MASK
 	update_iter - list of update commands as returned from parse_updates()"""
 
-	if isinstance(config_root, unicode):
-		# Avoid UnicodeDecodeError raised from
-		# os.path.join when called by os.walk.
-		config_root = config_root.encode('utf_8', 'replace')
-
 	config_root = normalize_path(config_root)
 	update_files = {}
 	file_contents = {}
@@ -166,9 +170,20 @@
 		if os.path.isdir(config_file):
 			for parent, dirs, files in os.walk(config_file):
 				for y in dirs:
+					try:
+						y = _unicode_decode(y,
+							encoding=_fs_encoding, errors='strict')
+					except UnicodeDecodeError:
+						dirs.remove(y)
+						continue
 					if y.startswith("."):
 						dirs.remove(y)
 				for y in files:
+					try:
+						y = _unicode_decode(y,
+							encoding=_fs_encoding, errors='strict')
+					except UnicodeDecodeError:
+						continue
 					if y.startswith("."):
 						continue
 					recursivefiles.append(
@@ -178,9 +193,11 @@
 	myxfiles = recursivefiles
 	for x in myxfiles:
 		try:
-			myfile = open(os.path.join(abs_user_config, x),"r")
-			file_contents[x] = myfile.readlines()
-			myfile.close()
+			file_contents[x] = codecs.open(
+				_unicode_encode(os.path.join(abs_user_config, x),
+				encoding=_fs_encoding, errors='strict'),
+				mode='r', encoding=_content_encoding,
+				errors='replace').readlines()
 		except IOError:
 			if file_contents.has_key(x):
 				del file_contents[x]

Modified: main/branches/prefix/pym/portage/util.py
===================================================================
--- main/branches/prefix/pym/portage/util.py	2009-08-24 09:27:07 UTC (rev 14137)
+++ main/branches/prefix/pym/portage/util.py	2009-08-24 09:28:51 UTC (rev 14138)
@@ -14,7 +14,6 @@
 
 import commands
 import codecs
-import os
 import errno
 import logging
 import shlex
@@ -24,7 +23,8 @@
 
 import portage
 from portage import os
-from portage import _merge_encoding
+from portage import _content_encoding
+from portage import _fs_encoding
 from portage import _os_merge
 from portage import _unicode_encode
 from portage import _unicode_decode
@@ -328,8 +328,9 @@
 					os.path.join(myfilename, f), recursive))
 	else:
 		try:
-			myfile = codecs.open(_unicode_encode(myfilename),
-				mode='r', encoding='utf_8', errors='replace')
+			myfile = codecs.open(_unicode_encode(myfilename,
+				encoding=_fs_encoding, errors='strict'),
+				mode='r', encoding=_content_encoding, errors='replace')
 			mylines = myfile.readlines()
 			myfile.close()
 		except IOError, e:
@@ -395,10 +396,12 @@
 		# NOTE: shex doesn't seem to support unicode objects
 		# (produces spurious \0 characters with python-2.6.2)
 		if sys.hexversion < 0x3000000:
-			content = open(_unicode_encode(mycfg), 'rb').read()
+			content = open(_unicode_encode(mycfg,
+				encoding=_fs_encoding, errors='strict'), 'rb').read()
 		else:
-			content = open(_unicode_encode(mycfg), mode='r',
-				encoding='utf_8', errors='replace').read()
+			content = open(_unicode_encode(mycfg,
+				encoding=_fs_encoding, errors='strict'), mode='r',
+				encoding=_content_encoding, errors='replace').read()
 		if content and content[-1] != '\n':
 			content += '\n'
 	except IOError, e:
@@ -590,7 +593,8 @@
 		return default
 	data = None
 	try:
-		myf = open(_unicode_encode(filename), 'rb')
+		myf = open(_unicode_encode(filename,
+			encoding=_fs_encoding, errors='strict'), 'rb')
 		mypickle = pickle.Unpickler(myf)
 		data = mypickle.load()
 		myf.close()
@@ -905,7 +909,7 @@
 			open_func = open
 		else:
 			open_func = codecs.open
-			kargs.setdefault('encoding', 'utf_8')
+			kargs.setdefault('encoding', _content_encoding)
 			kargs.setdefault('errors', 'replace')
 
 		if follow_links:
@@ -914,7 +918,9 @@
 			tmp_name = "%s.%i" % (canonical_path, os.getpid())
 			try:
 				object.__setattr__(self, '_file',
-					open_func(_unicode_encode(tmp_name), mode=mode, **kargs))
+					open_func(_unicode_encode(tmp_name,
+						encoding=_fs_encoding, errors='strict'),
+						mode=mode, **kargs))
 				return
 			except IOError, e:
 				if canonical_path == filename:
@@ -926,7 +932,9 @@
 		object.__setattr__(self, '_real_name', filename)
 		tmp_name = "%s.%i" % (filename, os.getpid())
 		object.__setattr__(self, '_file',
-			open_func(_unicode_encode(tmp_name), mode=mode, **kargs))
+			open_func(_unicode_encode(tmp_name,
+				encoding=_fs_encoding, errors='strict'),
+				mode=mode, **kargs))
 
 	def _get_target(self):
 		return object.__getattribute__(self, '_file')

Modified: main/branches/prefix/pym/portage/xpak.py
===================================================================
--- main/branches/prefix/pym/portage/xpak.py	2009-08-24 09:27:07 UTC (rev 14137)
+++ main/branches/prefix/pym/portage/xpak.py	2009-08-24 09:28:51 UTC (rev 14138)
@@ -16,29 +16,50 @@
 # (integer) == encodeint(integer)  ===> 4 characters (big-endian copy)
 # '+' means concatenate the fields ===> All chunks are strings
 
-import sys,os,shutil,errno
-from stat import *
+import array
+import errno
+import shutil
 
-def addtolist(mylist,curdir):
+from portage import os
+from portage import normalize_path
+from portage import _fs_encoding
+from portage import _unicode_decode
+from portage import _unicode_encode
+
+def addtolist(mylist, curdir):
 	"""(list, dir) --- Takes an array(list) and appends all files from dir down
 	the directory tree. Returns nothing. list is modified."""
-	for x in os.listdir("."):
-		if os.path.isdir(x):
-			os.chdir(x)
-			addtolist(mylist,curdir+x+"/")
-			os.chdir("..")
-		else:
-			if curdir+x not in mylist:
-				mylist.append(curdir+x)
+	curdir = normalize_path(_unicode_decode(curdir,
+		encoding=_fs_encoding, errors='strict'))
+	for parent, dirs, files in os.walk(curdir):
 
+		parent = _unicode_decode(parent,
+			encoding=_fs_encoding, errors='strict')
+		if parent != curdir:
+			mylist.append(parent[len(curdir) + 1:] + os.sep)
+
+		for x in dirs:
+			try:
+				_unicode_decode(x, encoding=_fs_encoding, errors='strict')
+			except UnicodeDecodeError:
+				dirs.remove(x)
+
+		for x in files:
+			try:
+				x = _unicode_decode(x, encoding=_fs_encoding, errors='strict')
+			except UnicodeDecodeError:
+				continue
+			mylist.append(os.path.join(parent, x)[len(curdir) + 1:])
+
 def encodeint(myint):
 	"""Takes a 4 byte integer and converts it into a string of 4 characters.
 	Returns the characters in a string."""
-	part1=chr((myint >> 24 ) & 0x000000ff)
-	part2=chr((myint >> 16 ) & 0x000000ff)
-	part3=chr((myint >> 8 ) & 0x000000ff)
-	part4=chr(myint & 0x000000ff)
-	return part1+part2+part3+part4
+	a = array.array('B')
+	a.append((myint >> 24 ) & 0xff)
+	a.append((myint >> 16 ) & 0xff)
+	a.append((myint >> 8 ) & 0xff)
+	a.append(myint & 0xff)
+	return a.tostring()
 
 def decodeint(mystring):
 	"""Takes a 4 byte string and converts it into a 4 byte integer.
@@ -54,28 +75,20 @@
 	"""(rootdir,outfile) -- creates an xpak segment of the directory 'rootdir'
 	and under the name 'outfile' if it is specified. Otherwise it returns the
 	xpak segment."""
-	try:
-		origdir=os.getcwd()
-	except SystemExit, e:
-		raise
-	except:
-		os.chdir("/")
-		origdir="/"
-	os.chdir(rootdir)
+
 	mylist=[]
 
-	addtolist(mylist,"")
+	addtolist(mylist, rootdir)
 	mylist.sort()
 	mydata = {}
 	for x in mylist:
-		a = open(x, 'rb')
-		mydata[x] = a.read()
-		a.close()
-	os.chdir(origdir)
+		x = _unicode_encode(x, encoding=_fs_encoding, errors='strict')
+		mydata[x] = open(os.path.join(rootdir, x), 'rb').read()
 
 	xpak_segment = xpak_mem(mydata)
 	if outfile:
-		outf = open(outfile, 'wb')
+		outf = open(_unicode_encode(outfile,
+			encoding=_fs_encoding, errors='strict'), 'wb')
 		outf.write(xpak_segment)
 		outf.close()
 	else:
@@ -83,9 +96,9 @@
 
 def xpak_mem(mydata):
 	"""Create an xpack segement from a map object."""
-	indexglob=""
+	indexglob = _unicode_encode('')
 	indexpos=0
-	dataglob=""
+	dataglob = _unicode_encode('')
 	datapos=0
 	for x, newglob in mydata.iteritems():
 		mydatasize=len(newglob)
@@ -93,18 +106,21 @@
 		indexpos=indexpos+4+len(x)+4+4
 		dataglob=dataglob+newglob
 		datapos=datapos+mydatasize
-	return "XPAKPACK" \
+	return _unicode_encode('XPAKPACK') \
 	+ encodeint(len(indexglob)) \
 	+ encodeint(len(dataglob)) \
 	+ indexglob \
 	+ dataglob \
-	+ "XPAKSTOP"
+	+ _unicode_encode('XPAKSTOP')
 
 def xsplit(infile):
 	"""(infile) -- Splits the infile into two files.
 	'infile.index' contains the index segment.
 	'infile.dat' contails the data segment."""
-	myfile = open(infile, 'rb')
+	infile = _unicode_decode(infile,
+		encoding=_fs_encoding, errors='strict')
+	myfile = open(_unicode_encode(infile,
+		encoding=_fs_encoding, errors='strict'), 'rb')
 	mydat=myfile.read()
 	myfile.close()
 	
@@ -112,27 +128,30 @@
 	if not splits:
 		return False
 	
-	myfile = open(infile + '.index', 'wb')
+	myfile = open(_unicode_encode(infile + '.index',
+		encoding=_fs_encoding, errors='strict'), 'wb')
 	myfile.write(splits[0])
 	myfile.close()
-	myfile = open(infile + '.dat', 'wb')
+	myfile = open(_unicode_encode(infile + '.dat',
+		encoding=_fs_encoding, errors='strict'), 'wb')
 	myfile.write(splits[1])
 	myfile.close()
 	return True
 
 def xsplit_mem(mydat):
-	if mydat[0:8]!="XPAKPACK":
+	if mydat[0:8] != _unicode_encode('XPAKPACK'):
 		return None
-	if mydat[-8:]!="XPAKSTOP":
+	if mydat[-8:] != _unicode_encode('XPAKSTOP'):
 		return None
 	indexsize=decodeint(mydat[8:12])
 	return (mydat[16:indexsize+16], mydat[indexsize+16:-8])
 
 def getindex(infile):
 	"""(infile) -- grabs the index segment from the infile and returns it."""
-	myfile = open(infile, 'rb')
+	myfile = open(_unicode_encode(infile,
+		encoding=_fs_encoding, errors='strict'), 'rb')
 	myheader=myfile.read(16)
-	if myheader[0:8]!="XPAKPACK":
+	if myheader[0:8] != _unicode_encode('XPAKPACK'):
 		myfile.close()
 		return
 	indexsize=decodeint(myheader[8:12])
@@ -143,9 +162,10 @@
 def getboth(infile):
 	"""(infile) -- grabs the index and data segments from the infile.
 	Returns an array [indexSegment,dataSegment]"""
-	myfile = open(infile, 'rb')
+	myfile = open(_unicode_encode(infile,
+		encoding=_fs_encoding, errors='strict'), 'rb')
 	myheader=myfile.read(16)
-	if myheader[0:8]!="XPAKPACK":
+	if myheader[0:8] != _unicode_encode('XPAKPACK'):
 		myfile.close()
 		return
 	indexsize=decodeint(myheader[8:12])
@@ -217,7 +237,8 @@
 		if dirname:
 			if not os.path.exists(dirname):
 				os.makedirs(dirname)
-		mydat = open(myname, 'wb')
+		mydat = open(_unicode_encode(myname,
+			encoding=_fs_encoding, errors='strict'), 'wb')
 		mydat.write(mydata[datapos:datapos+datalen])
 		mydat.close()
 		startpos=startpos+namelen+12
@@ -227,7 +248,7 @@
 	def __init__(self,myfile):
 		self.file=myfile
 		self.filestat=None
-		self.index=""
+		self.index = _unicode_encode('')
 		self.infosize=0
 		self.xpaksize=0
 		self.indexsize=None
@@ -262,12 +283,13 @@
 
 	def recompose_mem(self, xpdata):
 		self.scan() # Don't care about condition... We'll rewrite the data anyway.
-		myfile = open(self.file, 'ab+')
+		myfile = open(_unicode_encode(self.file,
+			encoding=_fs_encoding, errors='strict'), 'ab+')
 		if not myfile:
 			raise IOError
 		myfile.seek(-self.xpaksize,2) # 0,2 or -0,2 just mean EOF.
 		myfile.truncate()
-		myfile.write(xpdata+encodeint(len(xpdata))+"STOP")
+		myfile.write(xpdata+encodeint(len(xpdata)) + _unicode_encode('STOP'))
 		myfile.flush()
 		myfile.close()
 		return 1
@@ -292,28 +314,30 @@
 			mystat=os.stat(self.file)
 			if self.filestat:
 				changed=0
-				for x in [ST_SIZE, ST_MTIME, ST_CTIME]:
-					if mystat[x] != self.filestat[x]:
-						changed=1
+				if mystat.st_size != self.filestat.st_size \
+					or mystat.st_mtime != self.filestat.st_mtime \
+					or mystat.st_ctime != self.filestat.st_ctime:
+					changed = True
 				if not changed:
 					return 1
 			self.filestat=mystat
-			a = open(self.file, 'rb')
+			a = open(_unicode_encode(self.file,
+				encoding=_fs_encoding, errors='strict'), 'rb')
 			a.seek(-16,2)
 			trailer=a.read()
 			self.infosize=0
 			self.xpaksize=0
-			if trailer[-4:]!="STOP":
+			if trailer[-4:] != _unicode_encode('STOP'):
 				a.close()
 				return 0
-			if trailer[0:8]!="XPAKSTOP":
+			if trailer[0:8] != _unicode_encode('XPAKSTOP'):
 				a.close()
 				return 0
 			self.infosize=decodeint(trailer[8:12])
 			self.xpaksize=self.infosize+8
 			a.seek(-(self.xpaksize),2)
 			header=a.read(16)
-			if header[0:8]!="XPAKPACK":
+			if header[0:8] != _unicode_encode('XPAKPACK'):
 				a.close()
 				return 0
 			self.indexsize=decodeint(header[8:12])
@@ -341,7 +365,8 @@
 		myresult=searchindex(self.index,myfile)
 		if not myresult:
 			return mydefault
-		a = open(self.file, 'rb')
+		a = open(_unicode_encode(self.file,
+			encoding=_fs_encoding, errors='strict'), 'rb')
 		a.seek(self.datapos+myresult[0],0)
 		myreturn=a.read(myresult[1])
 		a.close()
@@ -365,7 +390,8 @@
 		except:
 			os.chdir("/")
 			origdir="/"
-		a = open(self.file, 'rb')
+		a = open(_unicode_encode(self.file,
+			encoding=_fs_encoding, errors='strict'), 'rb')
 		if not os.path.exists(mydest):
 			os.makedirs(mydest)
 		os.chdir(mydest)
@@ -379,7 +405,8 @@
 			if dirname:
 				if not os.path.exists(dirname):
 					os.makedirs(dirname)
-			mydat = open(myname, 'wb')
+			mydat = open(_unicode_encode(myname,
+				encoding=_fs_encoding, errors='strict'), 'wb')
 			a.seek(self.datapos+datapos)
 			mydat.write(a.read(datalen))
 			mydat.close()
@@ -392,7 +419,8 @@
 		"""Returns all the files from the dataSegment as a map object."""
 		if not self.scan():
 			return 0
-		a = open(self.file, 'rb')
+		a = open(_unicode_encode(self.file,
+			encoding=_fs_encoding, errors='strict'), 'rb')
 		mydata = {}
 		startpos=0
 		while ((startpos+8)<self.indexsize):
@@ -411,7 +439,8 @@
 		if not self.scan():
 			return None
 
-		a = open(self.file, 'rb')
+		a = open(_unicode_encode(self.file,
+			encoding=_fs_encoding, errors='strict'), 'rb')
 		a.seek(self.datapos)
 		mydata =a.read(self.datasize)
 		a.close()