[gentoo-portage-dev] [PATCH v3 2/2] FEATURES=case-insensitive-fs for bug #524236

[Zac Medico]

When case-insensitive-fs is enabled in FEATURES, the dblink.isowner
method, _owners_db class, and ConfigProtect class will be
case-insensitive. This causes the collision-protect and unmerge code
to behave correctly for a case-insensitive file system. If the file
system is case-insensitive but case-preserving, then case is preserved
in the CONTENTS data of installed packages.

X-Gentoo-Bug: 524236
X-Gentoo-Url: https://bugs.gentoo.org/show_bug.cgi?id=524236
---
 bin/dispatch-conf              |  8 ++++-
 bin/etc-update                 | 10 ++++--
 bin/portageq                   |  7 +++--
 bin/quickpkg                   |  4 ++-
 man/make.conf.5                |  6 ++++
 pym/_emerge/depgraph.py        |  4 ++-
 pym/portage/_global_updates.py |  4 ++-
 pym/portage/const.py           |  1 +
 pym/portage/dbapi/vartree.py   | 71 +++++++++++++++++++++++++++---------------
 pym/portage/update.py          |  6 ++--
 pym/portage/util/__init__.py   |  8 ++++-
 11 files changed, 92 insertions(+), 37 deletions(-)

diff --git a/bin/dispatch-conf b/bin/dispatch-conf
index 8058d6f..b679910 100755
--- a/bin/dispatch-conf
+++ b/bin/dispatch-conf
@@ -35,6 +35,10 @@ from portage.process import find_binary, spawn
 FIND_EXTANT_CONFIGS  = "find '%s' %s -name '._cfg????_%s' ! -name '.*~' ! -iname '.*.bak' -print"
 DIFF_CONTENTS        = "diff -Nu '%s' '%s'"
 
+if "case-insensitive-fs" in portage.settings.features:
+    FIND_EXTANT_CONFIGS = FIND_EXTANT_CONFIGS.replace(
+        "-name '._cfg", "-iname '._cfg")
+
 # We need a secure scratch dir and python does silly verbose errors on the use of tempnam
 oldmask = os.umask(0o077)
 SCRATCH_DIR = None
@@ -152,7 +156,9 @@ class dispatch:
         protect_obj = portage.util.ConfigProtect(
             config_root, config_paths,
             portage.util.shlex_split(
-            portage.settings.get('CONFIG_PROTECT_MASK', '')))
+            portage.settings.get('CONFIG_PROTECT_MASK', '')),
+            case_insensitive=("case-insensitive-fs"
+            in portage.settings.features))
 
         #
         # Remove new configs identical to current
diff --git a/bin/etc-update b/bin/etc-update
index 0307688..e0f7224 100755
--- a/bin/etc-update
+++ b/bin/etc-update
@@ -113,12 +113,15 @@ scan() {
 			[[ -d ${path%/*} ]] || continue
 			local name_opt=$(get_basename_find_opt "${path##*/}")
 			path="${path%/*}"
-			find_opts=( -maxdepth 1 -name "$name_opt" )
+			find_opts=( -maxdepth 1 )
 		else
 			# Do not traverse hidden directories such as .svn or .git.
 			local name_opt=$(get_basename_find_opt '*')
-			find_opts=( -name '.*' -type d -prune -o -name "$name_opt" )
+			find_opts=( -name '.*' -type d -prune -o )
 		fi
+		${case_insensitive} && \
+			find_opts+=( -iname ) || find_opts+=( -name )
+		find_opts+=( "$name_opt" )
 		find_opts+=( ! -name '.*~' ! -iname '.*.bak' -print )
 
 		if [ ! -w "${path}" ] ; then
@@ -743,6 +746,7 @@ fi
 
 portage_vars=(
 	CONFIG_PROTECT{,_MASK}
+	FEATURES
 	PORTAGE_CONFIGROOT
 	PORTAGE_INST_{G,U}ID
 	PORTAGE_TMPDIR
@@ -759,6 +763,8 @@ fi
 
 export PORTAGE_TMPDIR
 SCAN_PATHS=${*:-${CONFIG_PROTECT}}
+[[ " ${FEATURES} " == *" case-insensitive-fs "* ]] && \
+	case_insensitive=true || case_insensitive=false
 
 TMP="${PORTAGE_TMPDIR}/etc-update-$$"
 trap "die terminated" SIGTERM
diff --git a/bin/portageq b/bin/portageq
index 6a42bfd..2c4f548 100755
--- a/bin/portageq
+++ b/bin/portageq
@@ -379,8 +379,8 @@ def is_protected(argv):
 	protect = portage.util.shlex_split(settings.get("CONFIG_PROTECT", ""))
 	protect_mask = portage.util.shlex_split(
 		settings.get("CONFIG_PROTECT_MASK", ""))
-	protect_obj = ConfigProtect(root, protect, protect_mask)
-
+	protect_obj = ConfigProtect(root, protect, protect_mask,
+		case_insensitive=("case-insensitive-fs" in settings.features))
 	if protect_obj.isprotected(f):
 		return 0
 	return 1
@@ -414,7 +414,8 @@ def filter_protected(argv):
 	protect = portage.util.shlex_split(settings.get("CONFIG_PROTECT", ""))
 	protect_mask = portage.util.shlex_split(
 		settings.get("CONFIG_PROTECT_MASK", ""))
-	protect_obj = ConfigProtect(root, protect, protect_mask)
+	protect_obj = ConfigProtect(root, protect, protect_mask,
+		case_insensitive=("case-insensitive-fs" in settings.features))
 
 	errors = 0
 
diff --git a/bin/quickpkg b/bin/quickpkg
index cf75791..2c69a69 100755
--- a/bin/quickpkg
+++ b/bin/quickpkg
@@ -102,7 +102,9 @@ def quickpkg_atom(options, infos, arg, eout):
 			if not include_config:
 				confprot = ConfigProtect(eroot,
 					shlex_split(settings.get("CONFIG_PROTECT", "")),
-					shlex_split(settings.get("CONFIG_PROTECT_MASK", "")))
+					shlex_split(settings.get("CONFIG_PROTECT_MASK", "")),
+					case_insensitive=("case-insensitive-fs"
+					in settings.features))
 				def protect(filename):
 					if not confprot.isprotected(filename):
 						return False
diff --git a/man/make.conf.5 b/man/make.conf.5
index 84e894b..69d95fc 100644
--- a/man/make.conf.5
+++ b/man/make.conf.5
@@ -265,6 +265,12 @@ Build binary packages for just packages in the system set.
 Enable a special progress indicator when \fBemerge\fR(1) is calculating
 dependencies.
 .TP
+.B case\-insensitive\-fs
+Use case\-insensitive file name comparisions when merging and unmerging
+files. Most users should not enable this feature, since most filesystems
+are case\-sensitive. You should only enable this feature if you are
+using portage to install files to a case\-insensitive filesystem.
+.TP
 .B ccache
 Enable portage support for the ccache package.  If the ccache dir is not
 present in the user's environment, then portage will default to
diff --git a/pym/_emerge/depgraph.py b/pym/_emerge/depgraph.py
index 2a839d0..6f1910d 100644
--- a/pym/_emerge/depgraph.py
+++ b/pym/_emerge/depgraph.py
@@ -7813,7 +7813,9 @@ class depgraph(object):
 				settings = self._frozen_config.roots[root].settings
 				protect_obj[root] = ConfigProtect(settings["EROOT"], \
 					shlex_split(settings.get("CONFIG_PROTECT", "")),
-					shlex_split(settings.get("CONFIG_PROTECT_MASK", "")))
+					shlex_split(settings.get("CONFIG_PROTECT_MASK", "")),
+					case_insensitive=("case-insensitive-fs"
+					in settings.features))
 
 		def write_changes(root, changes, file_to_write_to):
 			file_contents = None
diff --git a/pym/portage/_global_updates.py b/pym/portage/_global_updates.py
index 17dc080..81ee484 100644
--- a/pym/portage/_global_updates.py
+++ b/pym/portage/_global_updates.py
@@ -208,7 +208,9 @@ def _do_global_updates(trees, prev_mtimes, quiet=False, if_mtime_changed=True):
 		update_config_files(root,
 			shlex_split(mysettings.get("CONFIG_PROTECT", "")),
 			shlex_split(mysettings.get("CONFIG_PROTECT_MASK", "")),
-			repo_map, match_callback=_config_repo_match)
+			repo_map, match_callback=_config_repo_match,
+			case_insensitive="case-insensitive-fs"
+			in mysettings.features)
 
 		# The above global updates proceed quickly, so they
 		# are considered a single mtimedb transaction.
diff --git a/pym/portage/const.py b/pym/portage/const.py
index d472075..febdb4a 100644
--- a/pym/portage/const.py
+++ b/pym/portage/const.py
@@ -125,6 +125,7 @@ SUPPORTED_FEATURES       = frozenset([
 	"buildpkg",
 	"buildsyspkg",
 	"candy",
+	"case-insensitive-fs",
 	"ccache",
 	"cgroup",
 	"chflags",
diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index 81059b1..0fd1bd9 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -1052,13 +1052,13 @@ class vardbapi(dbapi):
 
 		def add(self, cpv):
 			eroot_len = len(self._vardb._eroot)
-			contents = self._vardb._dblink(cpv).getcontents()
 			pkg_hash = self._hash_pkg(cpv)
-			if not contents:
+			db = self._vardb._dblink(cpv)
+			if not db.getcontents():
 				# Empty path is a code used to represent empty contents.
 				self._add_path("", pkg_hash)
 
-			for x in contents:
+			for x in db._contents.keys():
 				self._add_path(x[eroot_len:], pkg_hash)
 
 			self._vardb._aux_cache["modified"].add(cpv)
@@ -1190,6 +1190,8 @@ class vardbapi(dbapi):
 			hash_pkg = owners_cache._hash_pkg
 			hash_str = owners_cache._hash_str
 			base_names = self._vardb._aux_cache["owners"]["base_names"]
+			case_insensitive = "case-insensitive-fs" \
+				in vardb.settings.features
 
 			dblink_cache = {}
 
@@ -1206,6 +1208,8 @@ class vardbapi(dbapi):
 			while path_iter:
 
 				path = path_iter.pop()
+				if case_insensitive:
+					path = path.lower()
 				is_basename = os.sep != path[:1]
 				if is_basename:
 					name = path
@@ -1236,12 +1240,16 @@ class vardbapi(dbapi):
 								continue
 
 							if is_basename:
-								for p in dblink(cpv).getcontents():
+								for p in dblink(cpv)._contents.keys():
 									if os.path.basename(p) == name:
-										owners.append((cpv, p[len(root):]))
+										owners.append((cpv, dblink(cpv).
+										_contents.unmap_key(
+										p)[len(root):]))
 							else:
-								if dblink(cpv).isowner(path):
-									owners.append((cpv, path))
+								key = dblink(cpv)._match_contents(path)
+								if key is not False:
+									owners.append(
+										(cpv, key[len(root):]))
 
 					except StopIteration:
 						path_iter.append(path)
@@ -1266,8 +1274,12 @@ class vardbapi(dbapi):
 			if not path_list:
 				return
 
+			case_insensitive = "case-insensitive-fs" \
+				in self._vardb.settings.features
 			path_info_list = []
 			for path in path_list:
+				if case_insensitive:
+					path = path.lower()
 				is_basename = os.sep != path[:1]
 				if is_basename:
 					name = path
@@ -1285,12 +1297,16 @@ class vardbapi(dbapi):
 				dblnk = self._vardb._dblink(cpv)
 				for path, name, is_basename in path_info_list:
 					if is_basename:
-						for p in dblnk.getcontents():
+						for p in dblnk._contents.keys():
 							if os.path.basename(p) == name:
-								search_pkg.results.append((dblnk, p[len(root):]))
+								search_pkg.results.append((dblnk,
+									dblnk._contents.unmap_key(
+										p)[len(root):]))
 					else:
-						if dblnk.isowner(path):
-							search_pkg.results.append((dblnk, path))
+						key = dblnk._match_contents(path)
+						if key is not False:
+							search_pkg.results.append(
+								(dblnk, key[len(root):]))
 				search_pkg.complete = True
 				return False
 
@@ -1542,7 +1558,9 @@ class dblink(object):
 			portage.util.shlex_split(
 				self.settings.get("CONFIG_PROTECT", "")),
 			portage.util.shlex_split(
-				self.settings.get("CONFIG_PROTECT_MASK", "")))
+				self.settings.get("CONFIG_PROTECT_MASK", "")),
+			case_insensitive=("case-insensitive-fs"
+					in self.settings.features))
 
 		return self._protect_obj
 
@@ -1620,9 +1638,9 @@ class dblink(object):
 		"""
 		Get the installed files of a given package (aka what that package installed)
 		"""
-		contents_file = os.path.join(self.dbdir, "CONTENTS")
 		if self.contentscache is not None:
 			return self.contentscache
+		contents_file = os.path.join(self.dbdir, "CONTENTS")
 		pkgfiles = {}
 		try:
 			with io.open(_unicode_encode(contents_file,
@@ -2764,15 +2782,18 @@ class dblink(object):
 			os_filename_arg.path.join(destroot,
 			filename.lstrip(os_filename_arg.path.sep)))
 
-		pkgfiles = self.getcontents()
-		if pkgfiles and destfile in pkgfiles:
-			return destfile
-		if pkgfiles:
+		if "case-insensitive-fs" in self.settings.features:
+			destfile = destfile.lower()
+
+		if self._contents.contains(destfile):
+			return self._contents.unmap_key(destfile)
+
+		if self.getcontents():
 			basename = os_filename_arg.path.basename(destfile)
 			if self._contents_basenames is None:
 
 				try:
-					for x in pkgfiles:
+					for x in self._contents.keys():
 						_unicode_encode(x,
 							encoding=_encodings['merge'],
 							errors='strict')
@@ -2781,7 +2802,7 @@ class dblink(object):
 					# different value of sys.getfilesystemencoding(),
 					# so fall back to utf_8 if appropriate.
 					try:
-						for x in pkgfiles:
+						for x in self._contents.keys():
 							_unicode_encode(x,
 								encoding=_encodings['fs'],
 								errors='strict')
@@ -2791,7 +2812,7 @@ class dblink(object):
 						os = portage.os
 
 				self._contents_basenames = set(
-					os.path.basename(x) for x in pkgfiles)
+					os.path.basename(x) for x in self._contents.keys())
 			if basename not in self._contents_basenames:
 				# This is a shortcut that, in most cases, allows us to
 				# eliminate this package as an owner without the need
@@ -2812,7 +2833,7 @@ class dblink(object):
 
 				if os is _os_merge:
 					try:
-						for x in pkgfiles:
+						for x in self._contents.keys():
 							_unicode_encode(x,
 								encoding=_encodings['merge'],
 								errors='strict')
@@ -2821,7 +2842,7 @@ class dblink(object):
 						# different value of sys.getfilesystemencoding(),
 						# so fall back to utf_8 if appropriate.
 						try:
-							for x in pkgfiles:
+							for x in self._contents.keys():
 								_unicode_encode(x,
 									encoding=_encodings['fs'],
 									errors='strict')
@@ -2832,7 +2853,7 @@ class dblink(object):
 
 				self._contents_inodes = {}
 				parent_paths = set()
-				for x in pkgfiles:
+				for x in self._contents.keys():
 					p_path = os.path.dirname(x)
 					if p_path in parent_paths:
 						continue
@@ -2857,8 +2878,8 @@ class dblink(object):
 			if p_path_list:
 				for p_path in p_path_list:
 					x = os_filename_arg.path.join(p_path, basename)
-					if x in pkgfiles:
-						return x
+					if self._contents.contains(x):
+						return self._contents.unmap_key(x)
 
 		return False
 
diff --git a/pym/portage/update.py b/pym/portage/update.py
index df4e11b..83fc3d2 100644
--- a/pym/portage/update.py
+++ b/pym/portage/update.py
@@ -282,7 +282,8 @@ def parse_updates(mycontent):
 		myupd.append(mysplit)
 	return myupd, errors
 
-def update_config_files(config_root, protect, protect_mask, update_iter, match_callback = None):
+def update_config_files(config_root, protect, protect_mask, update_iter,
+	match_callback=None, case_insensitive=False):
 	"""Perform global updates on /etc/portage/package.*, /etc/portage/profile/package.*,
 	/etc/portage/profile/packages and /etc/portage/sets.
 	config_root - location of files to update
@@ -406,7 +407,8 @@ def update_config_files(config_root, protect, protect_mask, update_iter, match_c
 							sys.stdout.flush()
 
 	protect_obj = ConfigProtect(
-		config_root, protect, protect_mask)
+		config_root, protect, protect_mask,
+		case_insensitive=case_insensitive)
 	for x in update_files:
 		updating_file = os.path.join(abs_user_config, x)
 		if protect_obj.isprotected(updating_file):
diff --git a/pym/portage/util/__init__.py b/pym/portage/util/__init__.py
index ad3a351..d0cca5b 100644
--- a/pym/portage/util/__init__.py
+++ b/pym/portage/util/__init__.py
@@ -1555,10 +1555,12 @@ class LazyItemsDict(UserDict):
 			return result
 
 class ConfigProtect(object):
-	def __init__(self, myroot, protect_list, mask_list):
+	def __init__(self, myroot, protect_list, mask_list,
+		case_insensitive=False):
 		self.myroot = myroot
 		self.protect_list = protect_list
 		self.mask_list = mask_list
+		self.case_insensitive = case_insensitive
 		self.updateprotect()
 
 	def updateprotect(self):
@@ -1586,6 +1588,8 @@ class ConfigProtect(object):
 		for x in self.mask_list:
 			ppath = normalize_path(
 				os.path.join(self.myroot, x.lstrip(os.path.sep)))
+			if self.case_insensitive:
+				ppath = ppath.lower()
 			try:
 				"""Use lstat so that anything, even a broken symlink can be
 				protected."""
@@ -1606,6 +1610,8 @@ class ConfigProtect(object):
 		masked = 0
 		protected = 0
 		sep = os.path.sep
+		if self.case_insensitive:
+			obj = obj.lower()
 		for ppath in self.protect:
 			if len(ppath) > masked and obj.startswith(ppath):
 				if ppath in self._dirs:
-- 
2.0.4