[gentoo-commits] portage r14148 - in main/branches/prefix: bin man pym/portage pym/portage/dbapi

[Fabian Groffen (grobian)]

Author: grobian
Date: 2009-08-24 09:50:34 +0000 (Mon, 24 Aug 2009)
New Revision: 14148

Modified:
   main/branches/prefix/bin/repoman
   main/branches/prefix/man/repoman.1
   main/branches/prefix/pym/portage/dbapi/bintree.py
   main/branches/prefix/pym/portage/dbapi/vartree.py
   main/branches/prefix/pym/portage/output.py
   main/branches/prefix/pym/portage/update.py
   main/branches/prefix/pym/portage/xpak.py
Log:
   Merged from trunk -r14131:14140

   | 14132   | Add a LIVEVCS.unmasked category for ebuilds that have        |
   | zmedico | non-empty KEYWORDS and are not masked in the global          |
   |         | package.mask. Thanks to Diego E. Petten?\195?\178 for the            |
   |         | suggestion.                                                  |
   
   | 14133   | Bug #282306 - Inside dblink._match_contents(), fall back to  |
   | zmedico | utf_8 encoding if a path cannot be encoded under the user's  |
   |         | chosen encoding. This should fix the traceback shown in bug  |
   |         | #281199, comment #26.                                        |
   
   | 14134   | Bug #282505 - Fix unicode handling for package moves in      |
   | zmedico | binary packages.                                             |
   
   | 14140   | remove FIXME, since zmedico fixed it                         |
   | grobian |                                                              |


Modified: main/branches/prefix/bin/repoman
===================================================================
--- main/branches/prefix/bin/repoman	2009-08-24 09:49:02 UTC (rev 14147)
+++ main/branches/prefix/bin/repoman	2009-08-24 09:50:34 UTC (rev 14148)
@@ -300,6 +300,7 @@
 	"variable.invalidchar":"A variable contains an invalid character that is not part of the ASCII character set",
 	"variable.readonly":"Assigning a readonly variable",
 	"LIVEVCS.stable":"This ebuild is a live checkout from a VCS but has stable keywords.",
+	"LIVEVCS.unmasked":"This ebuild is a live checkout from a VCS but has keywords and is not masked in the global package.mask.",
 	"IUSE.invalid":"This ebuild has a variable in IUSE that is not in the use.desc or use.local.desc file",
 	"IUSE.undefined":"This ebuild does not define IUSE (style guideline says to define IUSE even when empty)",
 	"LICENSE.invalid":"This ebuild is listing a license that doesnt exist in portages license/ dir.",
@@ -362,7 +363,8 @@
 "virtual.unavailable",
 "usage.obsolete",
 "upstream.workaround",
-"LIVEVCS.stable"
+"LIVEVCS.stable",
+"LIVEVCS.unmasked",
 ))
 
 non_ascii_re = re.compile(r'[^\x00-\x7f]')
@@ -570,6 +572,7 @@
 kwlist = set()
 liclist = set()
 uselist = set()
+global_pmasklines = []
 
 for path in portdb.porttrees:
 	try:
@@ -600,6 +603,9 @@
 				if x:
 					uselist.add(use_prefix + x[0])
 
+	global_pmasklines.append(portage.util.grabfile_package(
+		os.path.join(path, 'profiles', 'package.mask'), recursive=1))
+
 	desc_path = os.path.join(path, 'profiles', 'profiles.desc')
 	try:
 		desc_file = codecs.open(_unicode_encode(desc_path,
@@ -635,6 +641,21 @@
 repoman_settings['PORTAGE_ARCHLIST'] = ' '.join(sorted(kwlist))
 repoman_settings.backup_changes('PORTAGE_ARCHLIST')
 
+global_pmasklines = portage.util.stack_lists(global_pmasklines, incremental=1)
+global_pmaskdict = {}
+for x in global_pmasklines:
+	global_pmaskdict.setdefault(portage.dep_getkey(x), []).append(x)
+del global_pmasklines
+
+def has_global_mask(pkg):
+	mask_atoms = global_pmaskdict.get(pkg.cp)
+	if mask_atoms:
+		pkg_list = [pkg]
+		for x in mask_atoms:
+			if portage.dep.match_from_list(x, pkg_list):
+				return x
+	return None
+
 # Ensure that profile sub_path attributes are unique. Process in reverse order
 # so that profiles with duplicate sub_path from overlays will override
 # profiles with the same sub_path from parent repos.
@@ -1347,6 +1368,10 @@
 						bad_stable_keywords)
 			del bad_stable_keywords
 
+			if keywords and not has_global_mask(pkg):
+				stats["LIVEVCS.unmasked"] += 1
+				fails["LIVEVCS.unmasked"].append(relative_path)
+
 		if options.ignore_arches:
 			arches = [[repoman_settings["ARCH"], repoman_settings["ARCH"],
 				repoman_settings["ACCEPT_KEYWORDS"].split()]]

Modified: main/branches/prefix/man/repoman.1
===================================================================
--- main/branches/prefix/man/repoman.1	2009-08-24 09:49:02 UTC (rev 14147)
+++ main/branches/prefix/man/repoman.1	2009-08-24 09:50:34 UTC (rev 14148)
@@ -149,6 +149,10 @@
 .B LIVEVCS.stable
 Ebuild is a live ebuild (cvs, git, darcs, svn, etc) checkout with stable keywords.
 .TP
+.B LIVEVCS.unmasked
+Ebuild is a live ebuild (cvs, git, darcs, svn, etc) checkout but has keywords
+and is not masked in the global package.mask.
+.TP
 .B PDEPEND.bad
 User-visible ebuilds with bad PDEPEND settings (matched against *visible* ebuilds)
 .TP

Modified: main/branches/prefix/pym/portage/dbapi/bintree.py
===================================================================
--- main/branches/prefix/pym/portage/dbapi/bintree.py	2009-08-24 09:49:02 UTC (rev 14147)
+++ main/branches/prefix/pym/portage/dbapi/bintree.py	2009-08-24 09:50:34 UTC (rev 14148)
@@ -266,13 +266,21 @@
 			mydata = mytbz2.get_data()
 			updated_items = update_dbentries([mylist], mydata)
 			mydata.update(updated_items)
-			mydata["PF"] = mynewpkg + "\n"
-			mydata["CATEGORY"] = mynewcat+"\n"
+			mydata[_unicode_encode('PF',
+				encoding=_encodings['repo.content'])] = \
+				_unicode_encode(mynewpkg + "\n",
+				encoding=_encodings['repo.content'])
+			mydata[_unicode_encode('CATEGORY',
+				encoding=_encodings['repo.content'])] = \
+				_unicode_encode(mynewcat + "\n",
+				encoding=_encodings['repo.content'])
 			if mynewpkg != myoldpkg:
-				ebuild_data = mydata.get(myoldpkg+".ebuild")
+				ebuild_data = mydata.pop(_unicode_encode(myoldpkg + '.ebuild',
+					encoding=_encodings['repo.content']), None)
 				if ebuild_data is not None:
-					mydata[mynewpkg+".ebuild"] = ebuild_data
-					del mydata[myoldpkg+".ebuild"]
+					mydata[_unicode_encode(mynewpkg + '.ebuild',
+						encoding=_encodings['repo.content'])] = ebuild_data
+
 			mytbz2.recompose_mem(portage.xpak.xpak_mem(mydata))
 
 			self.dbapi.cpv_remove(mycpv)

Modified: main/branches/prefix/pym/portage/dbapi/vartree.py
===================================================================
--- main/branches/prefix/pym/portage/dbapi/vartree.py	2009-08-24 09:49:02 UTC (rev 14147)
+++ main/branches/prefix/pym/portage/dbapi/vartree.py	2009-08-24 09:50:34 UTC (rev 14148)
@@ -3339,23 +3339,65 @@
 			if the file is not owned by this package.
 		"""
 
-		os = _os_merge
-
 		filename = _unicode_decode(filename,
 			encoding=_encodings['content'], errors='strict')
 
 		destroot = _unicode_decode(destroot,
 			encoding=_encodings['content'], errors='strict')
 
+		# The given filename argument might have a different encoding than the
+		# the filenames contained in the contents, so use separate wrapped os
+		# modules for each. The basename is more likely to contain non-ascii
+		# characters than the directory path, so use os_filename_arg for all
+		# operations involving the basename of the filename arg.
+		os_filename_arg = _os_merge
+		os = _os_merge
+
+		try:
+			_unicode_encode(filename,
+				encoding=_encodings['merge'], errors='strict')
+		except UnicodeEncodeError:
+			# The package appears to have been merged with a
+			# different value of sys.getfilesystemencoding(),
+			# so fall back to utf_8 if appropriate.
+			try:
+				_unicode_encode(filename,
+					encoding=_encodings['fs'], errors='strict')
+			except UnicodeEncodeError:
+				pass
+			else:
+				os_filename_arg = portage.os
+
 		destfile = normalize_path(
-			os.path.join(destroot, filename.lstrip(os.path.sep)))
+			os_filename_arg.path.join(destroot,
+			filename.lstrip(os_filename_arg.path.sep)))
 
 		pkgfiles = self.getcontents()
 		if pkgfiles and destfile in pkgfiles:
 			return destfile
 		if pkgfiles:
-			basename = os.path.basename(destfile)
+			basename = os_filename_arg.path.basename(destfile)
 			if self._contents_basenames is None:
+
+				try:
+					for x in pkgfiles:
+						_unicode_encode(x,
+							encoding=_encodings['merge'],
+							errors='strict')
+				except UnicodeEncodeError:
+					# The package appears to have been merged with a
+					# different value of sys.getfilesystemencoding(),
+					# so fall back to utf_8 if appropriate.
+					try:
+						for x in pkgfiles:
+							_unicode_encode(x,
+								encoding=_encodings['fs'],
+								errors='strict')
+					except UnicodeEncodeError:
+						pass
+					else:
+						os = portage.os
+
 				self._contents_basenames = set(
 					os.path.basename(x) for x in pkgfiles)
 			if basename not in self._contents_basenames:
@@ -3366,15 +3408,36 @@
 
 			# Use stat rather than lstat since we want to follow
 			# any symlinks to the real parent directory.
-			parent_path = os.path.dirname(destfile)
+			parent_path = os_filename_arg.path.dirname(destfile)
 			try:
-				parent_stat = os.stat(parent_path)
+				parent_stat = os_filename_arg.stat(parent_path)
 			except EnvironmentError, e:
 				if e.errno != errno.ENOENT:
 					raise
 				del e
 				return False
 			if self._contents_inodes is None:
+
+				if os is _os_merge:
+					try:
+						for x in pkgfiles:
+							_unicode_encode(x,
+								encoding=_encodings['merge'],
+								errors='strict')
+					except UnicodeEncodeError:
+						# The package appears to have been merged with a 
+						# different value of sys.getfilesystemencoding(),
+						# so fall back to utf_8 if appropriate.
+						try:
+							for x in pkgfiles:
+								_unicode_encode(x,
+									encoding=_encodings['fs'],
+									errors='strict')
+						except UnicodeEncodeError:
+							pass
+						else:
+							os = portage.os
+
 				self._contents_inodes = {}
 				parent_paths = set()
 				for x in pkgfiles:
@@ -3396,11 +3459,12 @@
 							self._contents_inodes[inode_key] = p_path_list
 						if p_path not in p_path_list:
 							p_path_list.append(p_path)
+
 			p_path_list = self._contents_inodes.get(
 				(parent_stat.st_dev, parent_stat.st_ino))
 			if p_path_list:
 				for p_path in p_path_list:
-					x = os.path.join(p_path, basename)
+					x = os_filename_arg.path.join(p_path, basename)
 					if x in pkgfiles:
 						return x
 

Modified: main/branches/prefix/pym/portage/output.py
===================================================================
--- main/branches/prefix/pym/portage/output.py	2009-08-24 09:49:02 UTC (rev 14147)
+++ main/branches/prefix/pym/portage/output.py	2009-08-24 09:50:34 UTC (rev 14148)
@@ -152,11 +152,6 @@
 	@rtype: dict
 	@return: a dictionary mapping color classes to color codes
 	"""
-	# FIXME: use config_root iso / (breaks Prefix)
-	# We can use ObjectProxy to implement lazy initialization of
-	# codes and _styles, and add an init(config_root="/") function
-	# that can be called in order adjust the location that color.map
-	# is read from.
 	global codes, _styles
 	myfile = os.path.join(config_root, COLOR_MAP_FILE)
 	ansi_code_pattern = re.compile("^[0-9;]*m$") 

Modified: main/branches/prefix/pym/portage/update.py
===================================================================
--- main/branches/prefix/pym/portage/update.py	2009-08-24 09:49:02 UTC (rev 14147)
+++ main/branches/prefix/pym/portage/update.py	2009-08-24 09:50:34 UTC (rev 14148)
@@ -55,12 +55,18 @@
 	dict containing only the updated items."""
 	updated_items = {}
 	for k, mycontent in mydata.iteritems():
-		if k not in ignored_dbentries:
+		k_unicode = _unicode_decode(k,
+			encoding=_encodings['repo.content'], errors='replace')
+		if k_unicode not in ignored_dbentries:
+			mycontent = _unicode_decode(mycontent,
+				encoding=_encodings['repo.content'], errors='replace')
 			orig_content = mycontent
 			for update_cmd in update_iter:
 				mycontent = update_dbentry(update_cmd, mycontent)
 			if mycontent != orig_content:
-				updated_items[k] = mycontent
+				updated_items[k] = _unicode_encode(mycontent,
+					encoding=_encodings['repo.content'],
+					errors='backslashreplace')
 	return updated_items
 
 def fixdbentries(update_iter, dbdir):
@@ -77,7 +83,7 @@
 	updated_items = update_dbentries(update_iter, mydata)
 	for myfile, mycontent in updated_items.iteritems():
 		file_path = os.path.join(dbdir, myfile)
-		write_atomic(file_path, mycontent)
+		write_atomic(file_path, mycontent, encoding=_encodings['repo.content'])
 	return len(updated_items) > 0
 
 def grab_updates(updpath, prev_mtimes=None):

Modified: main/branches/prefix/pym/portage/xpak.py
===================================================================
--- main/branches/prefix/pym/portage/xpak.py	2009-08-24 09:49:02 UTC (rev 14147)
+++ main/branches/prefix/pym/portage/xpak.py	2009-08-24 09:50:34 UTC (rev 14148)
@@ -97,6 +97,17 @@
 
 def xpak_mem(mydata):
 	"""Create an xpack segement from a map object."""
+
+	mydata_encoded = {}
+	for k, v in mydata.iteritems():
+		k = _unicode_encode(k,
+			encoding=_encodings['repo.content'], errors='backslashreplace')
+		v = _unicode_encode(v,
+			encoding=_encodings['repo.content'], errors='backslashreplace')
+		mydata_encoded[k] = v
+	mydata = mydata_encoded
+	del mydata_encoded
+
 	indexglob = _unicode_encode('')
 	indexpos=0
 	dataglob = _unicode_encode('')