[gentoo-commits] portage r14136 - in main/branches/prefix/pym: _emerge portage portage/dbapi portage/elog portage/sets

[Fabian Groffen (grobian)]

Author: grobian
Date: 2009-08-24 09:20:57 +0000 (Mon, 24 Aug 2009)
New Revision: 14136

Modified:
   main/branches/prefix/pym/_emerge/PackageUninstall.py
   main/branches/prefix/pym/portage/__init__.py
   main/branches/prefix/pym/portage/dbapi/vartree.py
   main/branches/prefix/pym/portage/elog/__init__.py
   main/branches/prefix/pym/portage/elog/messages.py
   main/branches/prefix/pym/portage/elog/mod_mail_summary.py
   main/branches/prefix/pym/portage/elog/mod_save.py
   main/branches/prefix/pym/portage/elog/mod_save_summary.py
   main/branches/prefix/pym/portage/manifest.py
   main/branches/prefix/pym/portage/sets/__init__.py
   main/branches/prefix/pym/portage/sets/dbapi.py
   main/branches/prefix/pym/portage/sets/files.py
   main/branches/prefix/pym/portage/sets/profiles.py
   main/branches/prefix/pym/portage/sets/security.py
   main/branches/prefix/pym/portage/sets/shell.py
Log:
   Merged from trunk -r14047:14057

   | 14048   | Revert r14042 since we're not going to use it.               |
   | zmedico |                                                              |
   
   | 14049   | In dblink._unmerge_pkgfiles(), if the package appears to     |
   | zmedico | have been merged with a different value of                   |
   |         | sys.getfilesystemencoding(), fall back to utf_8 if           |
   |         | appropriate.                                                 |
   
   | 14050   | Inside dblink.treewalk(), handle filenames with incorrect    |
   | zmedico | encoding like we do after src_install. The check needs to be |
   |         | repeated here for binary packages (it's inexpensive since we |
   |         | call os.walk() here anyway).                                 |
   
   | 14051   | Update imports to import portage.os (with unicode wrappers). |
   | zmedico |                                                              |
   
   | 14052   | Update imports to import portage.os (with unicode wrappers), |
   | zmedico | and use _unicode_encode() and _unicode_decode() where        |
   |         | appropriate.                                                 |
   
   | 14053   | Use portage._fs_encoding where applicable.                   |
   | zmedico |                                                              |
   
   | 14054   | Handle UnicodeDecodeError for os.walk() inside               |
   | zmedico | digestcheck().                                               |
   
   | 14055   | Use portage._fs_encoding where appropriage, and use strict   |
   | zmedico | handling for errors.                                         |
   
   | 14056   | Add a portage._content_encoding constant, set to utf_8.      |
   | zmedico |                                                              |
   
   | 14057   | Fix broken _selinux_merge definition.                        |
   | zmedico |                                                              |


Modified: main/branches/prefix/pym/_emerge/PackageUninstall.py
===================================================================
--- main/branches/prefix/pym/_emerge/PackageUninstall.py	2009-08-24 06:25:30 UTC (rev 14135)
+++ main/branches/prefix/pym/_emerge/PackageUninstall.py	2009-08-24 09:20:57 UTC (rev 14136)
@@ -12,14 +12,12 @@
 
 class PackageUninstall(AsynchronousTask):
 
-	__slots__ = ("clean_world", "ldpath_mtimes", "opts",
-		"pkg", "scheduler", "settings")
+	__slots__ = ("ldpath_mtimes", "opts", "pkg", "scheduler", "settings")
 
 	def _start(self):
 		try:
 			unmerge(self.pkg.root_config, self.opts, "unmerge",
-				[self.pkg.cpv], self.ldpath_mtimes,
-				clean_world=self.clean_world,
+				[self.pkg.cpv], self.ldpath_mtimes, clean_world=0,
 				clean_delay=0, raise_on_error=1, scheduler=self.scheduler,
 				writemsg_level=self._writemsg_level)
 		except UninstallFailure, e:

Modified: main/branches/prefix/pym/portage/__init__.py
===================================================================
--- main/branches/prefix/pym/portage/__init__.py	2009-08-24 06:25:30 UTC (rev 14135)
+++ main/branches/prefix/pym/portage/__init__.py	2009-08-24 09:20:57 UTC (rev 14136)
@@ -73,6 +73,7 @@
 		'portage.getbinpkg',
 		'portage.locks',
 		'portage.locks:lockdir,lockfile,unlockdir,unlockfile',
+		'portage.mail',
 		'portage.output',
 		'portage.output:bold,colorize',
 		'portage.process',
@@ -118,12 +119,20 @@
 	sys.stderr.write("    "+str(e)+"\n\n")
 	raise
 
-def _unicode_encode(s, encoding='utf_8', errors='replace'):
+# Assume utf_8 encoding for content of all files.
+_content_encoding = 'utf_8'
+
+# Assume utf_8 fs encoding everywhere except in merge code.
+_fs_encoding = 'utf_8'
+
+_merge_encoding = sys.getfilesystemencoding()
+
+def _unicode_encode(s, encoding=_content_encoding, errors='replace'):
 	if isinstance(s, unicode):
 		s = s.encode(encoding, errors)
 	return s
 
-def _unicode_decode(s, encoding='utf_8', errors='replace'):
+def _unicode_decode(s, encoding=_content_encoding, errors='replace'):
 	if not isinstance(s, unicode):
 		if sys.hexversion < 0x3000000:
 			if isinstance(s, basestring):
@@ -197,8 +206,6 @@
 			result = _unicode_func_wrapper(result, encoding=encoding)
 		return result
 
-_merge_encoding = sys.getfilesystemencoding()
-
 import os as _os
 _os_overrides = {
 	id(_os.fdopen)        : _os.fdopen,
@@ -206,20 +213,22 @@
 	id(_os.system)        : _os.system,
 }
 
-os = _unicode_module_wrapper(_os, overrides=_os_overrides)
+os = _unicode_module_wrapper(_os, overrides=_os_overrides,
+	encoding=_fs_encoding)
 _os_merge = _unicode_module_wrapper(_os,
 	encoding=_merge_encoding, overrides=_os_overrides)
 
 import shutil as _shutil
-shutil = _unicode_module_wrapper(_shutil)
+shutil = _unicode_module_wrapper(_shutil, encoding=_fs_encoding)
 
 # Imports below this point rely on the above unicode wrapper definitions.
 _selinux = None
 selinux = None
-_selinux_merge = _unicode_module_wrapper(_selinux, encoding=_merge_encoding)
+_selinux_merge = None
 try:
 	import portage._selinux
-	selinux = _unicode_module_wrapper(_selinux)
+	selinux = _unicode_module_wrapper(_selinux, encoding=_fs_encoding)
+	_selinux_merge = _unicode_module_wrapper(_selinux, encoding=_merge_encoding)
 except OSError, e:
 	sys.stderr.write("!!! SELinux not loaded: %s\n" % str(e))
 	del e
@@ -5310,12 +5319,37 @@
 	filesdir = os.path.join(pkgdir, "files")
 
 	for parent, dirs, files in os.walk(filesdir):
-		parent = _unicode_decode(parent)
+		try:
+			parent = _unicode_decode(parent,
+				encoding=_fs_encoding, errors='strict')
+		except UnicodeDecodeError:
+			parent = _unicode_decode(parent,
+				encoding=_fs_encoding, errors='replace')
+			writemsg("!!! Path contains invalid " + \
+				"character(s) for encoding '%s': '%s'" \
+				% (_fs_encoding, parent), noiselevel=-1)
+			if strict:
+				return 0
+			continue
 		for d in dirs:
 			if d.startswith(".") or d == "CVS":
 				dirs.remove(d)
 		for f in files:
-			f = _unicode_decode(f)
+			try:
+				f = _unicode_decode(f,
+					encoding=_fs_encoding, errors='strict')
+			except UnicodeDecodeError:
+				f = _unicode_decode(f,
+					encoding=_fs_encoding, errors='replace')
+				if f.startswith("."):
+					continue
+				f = os.path.join(parent, f)[len(filesdir) + 1:]
+				writemsg("!!! File name contains invalid " + \
+					"character(s) for encoding '%s': '%s'" \
+					% (_fs_encoding, f), noiselevel=-1)
+				if strict:
+					return 0
+				continue
 			if f.startswith("."):
 				continue
 			f = os.path.join(parent, f)[len(filesdir) + 1:]
@@ -5645,34 +5679,10 @@
 			break
 
 	if unicode_errors:
-		from textwrap import wrap
 		from portage.elog.messages import eerror
-		def _eerror(l):
+		for l in _merge_unicode_error(unicode_errors):
 			eerror(l, phase='install', key=mysettings.mycpv, out=out)
 
-		msg = "This package installs one or more file names containing " + \
-			"characters that do not match your current locale " + \
-			"settings. The current setting for filesystem encoding is '%s'." \
-			% _merge_encoding
-		for l in wrap(msg, 72):
-			_eerror(l)
-
-		_eerror("")
-		for x in unicode_errors:
-			_eerror("\t" + x)
-		_eerror("")
-
-		if _merge_encoding.lower().replace('_', '').replace('-', '') != 'utf8':
-			msg = "For best results, UTF-8 encoding is recommended. See " + \
-				"the Gentoo Linux Localization Guide for instructions " + \
-				"about how to configure your locale for UTF-8 encoding:"
-			for l in wrap(msg, 72):
-				_eerror(l)
-			_eerror("")
-			_eerror("\t" + \
-				"http://www.gentoo.org/doc/en/guide-localization.xml")
-			_eerror("")
-
 	open(_unicode_encode(os.path.join(mysettings['PORTAGE_BUILDDIR'],
 		'build-info', 'SIZE')), 'w').write(str(size) + '\n')
 
@@ -5682,6 +5692,33 @@
 			(_shell_quote(mysettings["D"]),
 			_shell_quote(os.path.join(mysettings["T"], "bsdflags.mtree"))))
 
+def _merge_unicode_error(errors):
+	from textwrap import wrap
+	lines = []
+
+	msg = "This package installs one or more file names containing " + \
+		"characters that do not match your current locale " + \
+		"settings. The current setting for filesystem encoding is '%s'." \
+		% _merge_encoding
+	lines.extend(wrap(msg, 72))
+
+	lines.append("")
+	errors.sort()
+	lines.extend("\t" + x for x in errors)
+	lines.append("")
+
+	if _merge_encoding.lower().replace('_', '').replace('-', '') != 'utf8':
+		msg = "For best results, UTF-8 encoding is recommended. See " + \
+			"the Gentoo Linux Localization Guide for instructions " + \
+			"about how to configure your locale for UTF-8 encoding:"
+		lines.extend(wrap(msg, 72))
+		lines.append("")
+		lines.append("\t" + \
+			"http://www.gentoo.org/doc/en/guide-localization.xml")
+		lines.append("")
+
+	return lines
+
 def _post_pkg_preinst_cmd(mysettings):
 	"""
 	Post phase logic and tasks that have been factored out of
@@ -7129,7 +7166,7 @@
 			"mv '%s' '%s'" % (src, dest))
 
 def movefile(src, dest, newmtime=None, sstat=None, mysettings=None,
-		hardlink_candidates=None, encoding='utf_8'):
+		hardlink_candidates=None, encoding=_fs_encoding):
 	"""moves a file from src to dest, preserving all permissions and attributes; mtime will
 	be preserved even when moving across filesystems.  Returns true on success and false on
 	failure.  Move is atomic."""

Modified: main/branches/prefix/pym/portage/dbapi/vartree.py
===================================================================
--- main/branches/prefix/pym/portage/dbapi/vartree.py	2009-08-24 06:25:30 UTC (rev 14135)
+++ main/branches/prefix/pym/portage/dbapi/vartree.py	2009-08-24 09:20:57 UTC (rev 14136)
@@ -38,9 +38,11 @@
 
 # This is a special version of the os module, wrapped for unicode support.
 from portage import os
+from portage import _fs_encoding
 from portage import _merge_encoding
 from portage import _os_merge
 from portage import _selinux_merge
+from portage import _unicode_decode
 from portage import _unicode_encode
 
 from portage.cache.mappings import slot_dict_class
@@ -3074,6 +3076,20 @@
 					scheduler.scheduleYield()
 
 				obj = normalize_path(objkey)
+				if os is _os_merge:
+					try:
+						_unicode_encode(obj, encoding=_merge_encoding, errors='strict')
+					except UnicodeEncodeError:
+						# The package appears to have been merged with a 
+						# different value of sys.getfilesystemencoding(),
+						# so fall back to utf_8 if appropriate.
+						try:
+							_unicode_encode(obj, encoding=_fs_encoding, errors='strict')
+						except UnicodeEncodeError:
+							pass
+						else:
+							os = portage.os
+
 				file_data = pkgfiles[objkey]
 				file_type = file_data[0]
 				statobj = None
@@ -3991,31 +4007,82 @@
 					max_dblnk = dblnk
 			self._installed_instance = max_dblnk
 
-		myfilelist = []
-		mylinklist = []
-		paths_with_newlines = []
-		srcroot_len = len(srcroot)
-		def onerror(e):
-			raise
-		for parent, dirs, files in os.walk(srcroot, onerror=onerror):
-			parent = portage._unicode_decode(parent, encoding=_merge_encoding)
-			for f in files:
-				f = portage._unicode_decode(f, encoding=_merge_encoding)
-				file_path = os.path.join(parent, f)
-				relative_path = file_path[srcroot_len:]
+		# We check for unicode encoding issues after src_install. However,
+		# the check must be repeated here for binary packages (it's
+		# inexpensive since we call os.walk() here anyway).
+		unicode_errors = []
 
-				if "\n" in relative_path:
-					paths_with_newlines.append(relative_path)
+		while True:
 
-				file_mode = os.lstat(file_path).st_mode
-				if stat.S_ISREG(file_mode):
-					myfilelist.append(relative_path)
-				elif stat.S_ISLNK(file_mode):
-					# Note: os.walk puts symlinks to directories in the "dirs"
-					# list and it does not traverse them since that could lead
-					# to an infinite recursion loop.
-					mylinklist.append(relative_path)
+			unicode_error = False
 
+			myfilelist = []
+			mylinklist = []
+			paths_with_newlines = []
+			srcroot_len = len(srcroot)
+			def onerror(e):
+				raise
+			for parent, dirs, files in os.walk(srcroot, onerror=onerror):
+				try:
+					parent = _unicode_decode(parent,
+						encoding=_merge_encoding, errors='strict')
+				except UnicodeDecodeError:
+					new_parent = _unicode_decode(parent,
+						encoding=_merge_encoding, errors='replace')
+					new_parent = _unicode_encode(new_parent,
+						encoding=_merge_encoding, errors='backslashreplace')
+					new_parent = _unicode_decode(new_parent,
+						encoding=_merge_encoding, errors='replace')
+					os.rename(parent, new_parent)
+					unicode_error = True
+					unicode_errors.append(new_parent[srcroot_len:])
+					break
+
+				for fname in files:
+					try:
+						fname = _unicode_decode(fname,
+							encoding=_merge_encoding, errors='strict')
+					except UnicodeDecodeError:
+						fpath = portage._os.path.join(
+							parent.encode(_merge_encoding), fname)
+						new_fname = _unicode_decode(fname,
+							encoding=_merge_encoding, errors='replace')
+						new_fname = _unicode_encode(new_fname,
+							encoding=_merge_encoding, errors='backslashreplace')
+						new_fname = _unicode_decode(new_fname,
+							encoding=_merge_encoding, errors='replace')
+						new_fpath = os.path.join(parent, new_fname)
+						os.rename(fpath, new_fpath)
+						unicode_error = True
+						unicode_errors.append(new_fpath[srcroot_len:])
+						fname = new_fname
+						fpath = new_fpath
+					else:
+						fpath = os.path.join(parent, fname)
+
+					relative_path = fpath[srcroot_len:]
+
+					if "\n" in relative_path:
+						paths_with_newlines.append(relative_path)
+
+					file_mode = os.lstat(fpath).st_mode
+					if stat.S_ISREG(file_mode):
+						myfilelist.append(relative_path)
+					elif stat.S_ISLNK(file_mode):
+						# Note: os.walk puts symlinks to directories in the "dirs"
+						# list and it does not traverse them since that could lead
+						# to an infinite recursion loop.
+						mylinklist.append(relative_path)
+
+				if unicode_error:
+					break
+
+			if not unicode_error:
+				break
+
+		if unicode_errors:
+			eerror(portage._merge_unicode_error(unicode_errors))
+
 		if paths_with_newlines:
 			msg = []
 			msg.append(_("This package installs one or more files containing a newline (\\n) character:"))

Modified: main/branches/prefix/pym/portage/elog/__init__.py
===================================================================
--- main/branches/prefix/pym/portage/elog/__init__.py	2009-08-24 06:25:30 UTC (rev 14135)
+++ main/branches/prefix/pym/portage/elog/__init__.py	2009-08-24 09:20:57 UTC (rev 14136)
@@ -14,9 +14,8 @@
 from portage.elog.messages import collect_ebuild_messages, collect_messages
 from portage.elog.filtering import filter_loglevels
 from portage.localization import _
+from portage import os
 
-import os
-
 def _merge_logentries(a, b):
 	rValue = {}
 	phases = set(a)

Modified: main/branches/prefix/pym/portage/elog/messages.py
===================================================================
--- main/branches/prefix/pym/portage/elog/messages.py	2009-08-24 06:25:30 UTC (rev 14135)
+++ main/branches/prefix/pym/portage/elog/messages.py	2009-08-24 09:20:57 UTC (rev 14136)
@@ -11,9 +11,9 @@
 
 from portage.const import EBUILD_PHASES
 from portage.localization import _
+from portage import os
 
 import codecs
-import os
 import sys
 
 def collect_ebuild_messages(path):

Modified: main/branches/prefix/pym/portage/elog/mod_mail_summary.py
===================================================================
--- main/branches/prefix/pym/portage/elog/mod_mail_summary.py	2009-08-24 06:25:30 UTC (rev 14135)
+++ main/branches/prefix/pym/portage/elog/mod_mail_summary.py	2009-08-24 09:20:57 UTC (rev 14136)
@@ -3,11 +3,15 @@
 # Distributed under the terms of the GNU General Public License v2
 # $Id$
 
-import portage.mail, socket, os, time
+import portage
 from portage.exception import PortageException
 from portage.localization import _
 from portage.util import writemsg
+from portage import os
 
+import socket
+import time
+
 _items = {}
 def process(mysettings, key, logentries, fulltext):
 	global _items

Modified: main/branches/prefix/pym/portage/elog/mod_save.py
===================================================================
--- main/branches/prefix/pym/portage/elog/mod_save.py	2009-08-24 06:25:30 UTC (rev 14135)
+++ main/branches/prefix/pym/portage/elog/mod_save.py	2009-08-24 09:20:57 UTC (rev 14136)
@@ -4,7 +4,8 @@
 # $Id$
 
 import codecs
-import os, time
+import time
+from portage import os
 from portage.data import portage_uid, portage_gid
 from portage.util import ensure_dirs
 from portage.const import EPREFIX

Modified: main/branches/prefix/pym/portage/elog/mod_save_summary.py
===================================================================
--- main/branches/prefix/pym/portage/elog/mod_save_summary.py	2009-08-24 06:25:30 UTC (rev 14135)
+++ main/branches/prefix/pym/portage/elog/mod_save_summary.py	2009-08-24 09:20:57 UTC (rev 14136)
@@ -4,7 +4,8 @@
 # $Id$
 
 import codecs
-import os, time
+import time
+from portage import os
 from portage.data import portage_uid, portage_gid
 from portage.localization import _
 from portage.util import ensure_dirs, apply_permissions

Modified: main/branches/prefix/pym/portage/manifest.py
===================================================================
--- main/branches/prefix/pym/portage/manifest.py	2009-08-24 06:25:30 UTC (rev 14135)
+++ main/branches/prefix/pym/portage/manifest.py	2009-08-24 09:20:57 UTC (rev 14136)
@@ -12,6 +12,8 @@
 )
 
 from portage import os
+from portage import _content_encoding
+from portage import _fs_encoding
 from portage import _unicode_decode
 from portage import _unicode_encode
 from portage.exception import DigestException, FileNotFound, \
@@ -141,8 +143,9 @@
 		"""Parse a manifest.  If myhashdict is given then data will be added too it.
 		   Otherwise, a new dict will be created and returned."""
 		try:
-			fd = codecs.open(_unicode_encode(file_path), mode='r',
-				encoding='utf_8', errors='replace')
+			fd = codecs.open(_unicode_encode(file_path,
+				encoding=_fs_encoding, errors='strict'), mode='r',
+				encoding=_content_encoding, errors='replace')
 			if myhashdict is None:
 				myhashdict = {}
 			self._parseDigests(fd, myhashdict=myhashdict, **kwargs)
@@ -228,8 +231,9 @@
 			update_manifest = True
 			if not force:
 				try:
-					f = codecs.open(_unicode_encode(self.getFullname()),
-						mode='r', encoding='utf_8', errors='replace')
+					f = codecs.open(_unicode_encode(self.getFullname(),
+						encoding=_fs_encoding, errors='strict'),
+						mode='r', encoding=_content_encoding, errors='replace')
 					oldentries = list(self._parseManifestLines(f))
 					f.close()
 					if len(oldentries) == len(myentries):
@@ -320,7 +324,11 @@
 		for pkgdir, pkgdir_dirs, pkgdir_files in os.walk(pkgdir):
 			break
 		for f in pkgdir_files:
-			f = _unicode_decode(f)
+			try:
+				f = _unicode_decode(f,
+					encoding=_fs_encoding, errors='strict')
+			except UnicodeDecodeError:
+				continue
 			if f[:1] == ".":
 				continue
 			pf = None
@@ -351,6 +359,11 @@
 		cut_len = len(os.path.join(pkgdir, "files") + os.sep)
 		for parentdir, dirs, files in os.walk(os.path.join(pkgdir, "files")):
 			for f in files:
+				try:
+					f = _unicode_decode(f,
+						encoding=_fs_encoding, errors='strict')
+				except UnicodeDecodeError:
+					continue
 				full_path = os.path.join(parentdir, f)
 				recursive_files.append(full_path[cut_len:])
 		for f in recursive_files:
@@ -508,8 +521,9 @@
 		mfname = self.getFullname()
 		if not os.path.exists(mfname):
 			return rVal
-		myfile = codecs.open(_unicode_encode(mfname),
-			mode='r', encoding='utf_8', errors='replace')
+		myfile = codecs.open(_unicode_encode(mfname,
+			encoding=_fs_encoding, errors='strict'),
+			mode='r', encoding=_content_encoding, errors='replace')
 		lines = myfile.readlines()
 		myfile.close()
 		for l in lines:

Modified: main/branches/prefix/pym/portage/sets/__init__.py
===================================================================
--- main/branches/prefix/pym/portage/sets/__init__.py	2009-08-24 06:25:30 UTC (rev 14135)
+++ main/branches/prefix/pym/portage/sets/__init__.py	2009-08-24 09:20:57 UTC (rev 14136)
@@ -5,7 +5,7 @@
 __all__ = ["SETPREFIX", "get_boolean", "SetConfigError",
 	"SetConfig", "load_default_config"]
 
-import os
+from portage import os
 from ConfigParser import SafeConfigParser, NoOptionError
 from portage import load_mod
 from portage.const import USER_CONFIG_PATH, GLOBAL_CONFIG_PATH

Modified: main/branches/prefix/pym/portage/sets/dbapi.py
===================================================================
--- main/branches/prefix/pym/portage/sets/dbapi.py	2009-08-24 06:25:30 UTC (rev 14135)
+++ main/branches/prefix/pym/portage/sets/dbapi.py	2009-08-24 09:20:57 UTC (rev 14136)
@@ -2,6 +2,9 @@
 # Distributed under the terms of the GNU General Public License v2
 # $Id$
 
+import time
+
+from portage import os
 from portage.versions import catpkgsplit, catsplit, pkgcmp, best
 from portage.dep import Atom
 from portage.localization import _
@@ -280,7 +283,6 @@
 		self._age = age
 
 	def _filter(self, atom):
-		import time, os
 	
 		cpv = self._db.match(atom)[0]
 		path = self._db.getpath(cpv, filename="COUNTER")

Modified: main/branches/prefix/pym/portage/sets/files.py
===================================================================
--- main/branches/prefix/pym/portage/sets/files.py	2009-08-24 06:25:30 UTC (rev 14135)
+++ main/branches/prefix/pym/portage/sets/files.py	2009-08-24 09:20:57 UTC (rev 14136)
@@ -2,10 +2,13 @@
 # Distributed under the terms of the GNU General Public License v2
 # $Id$
 
-import os
 import re
 from itertools import chain
 
+from portage import os
+from portage import _fs_encoding
+from portage import _unicode_decode
+from portage import _unicode_encode
 from portage.util import grabfile, write_atomic, ensure_dirs, normalize_path
 from portage.const import PRIVATE_PATH, USER_CONFIG_PATH, EPREFIX_LSTRIP
 from portage.localization import _
@@ -125,35 +128,43 @@
 			except KeyError:
 				raise SetConfigError(_("Could not find repository '%s'") % match.groupdict()["reponame"])
 
+		try:
+			directory = _unicode_decode(directory,
+				encoding=_fs_encoding, errors='strict')
+			# Now verify that we can also encode it.
+			_unicode_encode(directory,
+				encoding=_fs_encoding, errors='strict')
+		except UnicodeError:
+			directory = _unicode_decode(directory,
+				encoding=_fs_encoding, errors='replace')
+			raise SetConfigError(
+				_("Directory path contains invalid character(s) for encoding '%s': '%s'") \
+				% (_fs_encoding, directory))
+
 		if os.path.isdir(directory):
 			directory = normalize_path(directory)
 
-			if isinstance(directory, unicode):
-				# Avoid UnicodeDecodeError raised from
-				# os.path.join when called by os.walk.
-				directory_unicode = directory
-				directory = directory.encode('utf_8', 'replace')
-			else:
-				directory_unicode = unicode(directory,
-					encoding='utf_8', errors='replace')
-
 			for parent, dirs, files in os.walk(directory):
-				if not isinstance(parent, unicode):
-					parent = unicode(parent,
-						encoding='utf_8', errors='replace')
+				try:
+					parent = _unicode_decode(parent,
+						encoding=_fs_encoding, errors='strict')
+				except UnicodeDecodeError:
+					continue
 				for d in dirs[:]:
 					if d[:1] == '.':
 						dirs.remove(d)
 				for filename in files:
-					if not isinstance(filename, unicode):
-						filename = unicode(filename,
-							encoding='utf_8', errors='replace')
+					try:
+						filename = _unicode_decode(filename,
+							encoding=_fs_encoding, errors='strict')
+					except UnicodeDecodeError:
+						continue
 					if filename[:1] == '.':
 						continue
 					if filename.endswith(".metadata"):
 						continue
 					filename = os.path.join(parent,
-						filename)[1 + len(directory_unicode):]
+						filename)[1 + len(directory):]
 					myname = name_pattern.replace("$name", filename)
 					myname = myname.replace("${name}", filename)
 					rValue[myname] = StaticFileSet(

Modified: main/branches/prefix/pym/portage/sets/profiles.py
===================================================================
--- main/branches/prefix/pym/portage/sets/profiles.py	2009-08-24 06:25:30 UTC (rev 14135)
+++ main/branches/prefix/pym/portage/sets/profiles.py	2009-08-24 09:20:57 UTC (rev 14136)
@@ -3,7 +3,8 @@
 # $Id$
 
 import logging
-import os
+
+from portage import os
 from portage.util import grabfile_package, stack_lists
 from portage.sets.base import PackageSet
 from portage.sets import get_boolean

Modified: main/branches/prefix/pym/portage/sets/security.py
===================================================================
--- main/branches/prefix/pym/portage/sets/security.py	2009-08-24 06:25:30 UTC (rev 14135)
+++ main/branches/prefix/pym/portage/sets/security.py	2009-08-24 09:20:57 UTC (rev 14136)
@@ -2,9 +2,7 @@
 # Distributed under the terms of the GNU General Public License v2
 # $Id$
 
-import os
 import portage.glsa as glsa
-from portage.util import grabfile, write_atomic
 from portage.sets.base import PackageSet
 from portage.versions import catpkgsplit, pkgcmp
 from portage.sets import get_boolean

Modified: main/branches/prefix/pym/portage/sets/shell.py
===================================================================
--- main/branches/prefix/pym/portage/sets/shell.py	2009-08-24 06:25:30 UTC (rev 14135)
+++ main/branches/prefix/pym/portage/sets/shell.py	2009-08-24 09:20:57 UTC (rev 14136)
@@ -3,8 +3,9 @@
 # $Id$
 
 import subprocess
-import os
 
+from portage import os
+from portage import _unicode_decode
 from portage.sets.base import PackageSet
 from portage.sets import SetConfigError
 
@@ -35,8 +36,7 @@
 		pipe = subprocess.Popen(self._command, stdout=subprocess.PIPE, shell=True)
 		stdout, stderr = pipe.communicate()
 		if pipe.wait() == os.EX_OK:
-			self._setAtoms(unicode(stdout,
-				encoding='utf_8', errors='replace').splitlines())
+			self._setAtoms(_unicode_decode(stdout).splitlines())
 
 	def singleBuilder(self, options, settings, trees):
 		if not "command" in options: