[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/, lib/portage/util/

[Zac Medico]

commit:     0b7eda500a0dcb98a67f33bf9ef25b202b358986
Author:     Arfrever Frehtes Taifersar Arahesis <Arfrever <AT> Apache <DOT> Org>
AuthorDate: Wed Aug  7 17:06:11 2019 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sun Aug 11 19:11:47 2019 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=0b7eda50

dblink._collision_protect: Detect internal collisions.

Implement detection of internal collisions (between files of the same package,
located in separate directories in the installation image (${D}) corresponding
to merged directories in the target filesystem (${ROOT})).

This provides protection against overwriting some files when performing merging
of files from ${D} to ${ROOT} in some filesystem layouts (such as /-merged layout
or /usr-merged layout).

Internal collisions between identical files are silently ignored.

Bug: https://bugs.gentoo.org/690484
Signed-off-by: Arfrever Frehtes Taifersar Arahesis <Arfrever <AT> Apache.Org>
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>

 lib/portage/dbapi/vartree.py       |  55 ++++++++++++++++++--
 lib/portage/util/_compare_files.py | 103 +++++++++++++++++++++++++++++++++++++
 2 files changed, 154 insertions(+), 4 deletions(-)

diff --git a/lib/portage/dbapi/vartree.py b/lib/portage/dbapi/vartree.py
index e2fce7736..4f069474b 100644
--- a/lib/portage/dbapi/vartree.py
+++ b/lib/portage/dbapi/vartree.py
@@ -30,6 +30,7 @@ portage.proxy.lazyimport.lazyimport(globals(),
 	'portage.util:apply_secpass_permissions,ConfigProtect,ensure_dirs,' + \
 		'writemsg,writemsg_level,write_atomic,atomic_ofstream,writedict,' + \
 		'grabdict,normalize_path,new_protect_filename',
+	'portage.util._compare_files:compare_files',
 	'portage.util.digraph:digraph',
 	'portage.util.env_update:env_update',
 	'portage.util.install_mask:install_mask_dir,InstallMask',
@@ -87,6 +88,7 @@ import io
 from itertools import chain
 import logging
 import os as _os
+import operator
 import platform
 import pwd
 import re
@@ -3418,6 +3420,8 @@ class dblink(object):
 
 			os = _os_merge
 
+			real_relative_paths = {}
+
 			collision_ignore = []
 			for x in portage.util.shlex_split(
 				self.settings.get("COLLISION_IGNORE", "")):
@@ -3469,8 +3473,13 @@ class dblink(object):
 					previous = current
 					progress_shown = True
 
-				dest_path = normalize_path(
-					os.path.join(destroot, f.lstrip(os.path.sep)))
+				dest_path = normalize_path(os.path.join(destroot, f.lstrip(os.path.sep)))
+
+				# Relative path with symbolic links resolved only in parent directories
+				real_relative_path = os.path.join(os.path.realpath(os.path.dirname(dest_path)),
+					os.path.basename(dest_path))[len(destroot):]
+
+				real_relative_paths.setdefault(real_relative_path, []).append(f.lstrip(os.path.sep))
 
 				parent = os.path.dirname(dest_path)
 				if parent not in dirs:
@@ -3556,9 +3565,24 @@ class dblink(object):
 							break
 					if stopmerge:
 						collisions.append(f)
+
+			internal_collisions = {}
+			for real_relative_path, files in real_relative_paths.items():
+				# Detect internal collisions between non-identical files.
+				if len(files) >= 2:
+					files.sort()
+					for i in range(len(files) - 1):
+						file1 = normalize_path(os.path.join(srcroot, files[i]))
+						file2 = normalize_path(os.path.join(srcroot, files[i+1]))
+						# Compare files, ignoring differences in times.
+						differences = compare_files(file1, file2, skipped_types=("atime", "mtime", "ctime"))
+						if differences:
+							internal_collisions.setdefault(real_relative_path, {})[(files[i], files[i+1])] = differences
+
 			if progress_shown:
 				showMessage(_("100% done\n"))
-			return collisions, dirs_ro, symlink_collisions, plib_collisions
+
+			return collisions, internal_collisions, dirs_ro, symlink_collisions, plib_collisions
 
 	def _lstat_inode_map(self, path_iter):
 		"""
@@ -4081,7 +4105,7 @@ class dblink(object):
 			if blocker.exists():
 				blockers.append(blocker)
 
-		collisions, dirs_ro, symlink_collisions, plib_collisions = \
+		collisions, internal_collisions, dirs_ro, symlink_collisions, plib_collisions = \
 			self._collision_protect(srcroot, destroot,
 			others_in_slot + blockers, filelist, linklist)
 
@@ -4109,6 +4133,29 @@ class dblink(object):
 			eerror(msg)
 			return 1
 
+		if internal_collisions:
+			msg = _("Package '%s' has internal collisions between non-identical files "
+				"(located in separate directories in the installation image (${D}) "
+				"corresponding to merged directories in the target "
+				"filesystem (${ROOT})):") % self.settings.mycpv
+			msg = textwrap.wrap(msg, 70)
+			msg.append("")
+			for k, v in sorted(internal_collisions.items(), key=operator.itemgetter(0)):
+				msg.append("\t%s" % os.path.join(destroot, k.lstrip(os.path.sep)))
+				for (file1, file2), differences in sorted(v.items()):
+					msg.append("\t\t%s" % os.path.join(destroot, file1.lstrip(os.path.sep)))
+					msg.append("\t\t%s" % os.path.join(destroot, file2.lstrip(os.path.sep)))
+					msg.append("\t\t\tDifferences: %s" % ", ".join(differences))
+					msg.append("")
+			self._elog("eerror", "preinst", msg)
+
+			msg = _("Package '%s' NOT merged due to internal collisions "
+				"between non-identical files.") % self.settings.mycpv
+			msg += _(" If necessary, refer to your elog messages for the whole "
+				"content of the above message.")
+			eerror(textwrap.wrap(msg, 70))
+			return 1
+
 		if symlink_collisions:
 			# Symlink collisions need to be distinguished from other types
 			# of collisions, in order to avoid confusion (see bug #409359).

diff --git a/lib/portage/util/_compare_files.py b/lib/portage/util/_compare_files.py
new file mode 100644
index 000000000..bd993e501
--- /dev/null
+++ b/lib/portage/util/_compare_files.py
@@ -0,0 +1,103 @@
+# Copyright 2019 Gentoo Authors
+# Distributed under the terms of the GNU General Public License v2
+
+__all__ = ["compare_files"]
+
+import io
+import os
+import stat
+import sys
+
+from portage import _encodings
+from portage import _unicode_encode
+from portage.util._xattr import xattr
+
+def compare_files(file1, file2, skipped_types=()):
+	"""
+	Compare metadata and contents of two files.
+
+	@param file1: File 1
+	@type file1: str
+	@param file2: File 2
+	@type file2: str
+	@param skipped_types: Tuple of strings specifying types of properties excluded from comparison.
+		Supported strings: type, mode, owner, group, device_number, xattr, atime, mtime, ctime, size, content
+	@type skipped_types: tuple of str
+	@rtype: tuple of str
+	@return: Tuple of strings specifying types of properties different between compared files
+	"""
+
+	file1_stat = os.lstat(_unicode_encode(file1, encoding=_encodings["fs"], errors="strict"))
+	file2_stat = os.lstat(_unicode_encode(file2, encoding=_encodings["fs"], errors="strict"))
+
+	differences = []
+
+	if (file1_stat.st_dev, file1_stat.st_ino) == (file2_stat.st_dev, file2_stat.st_ino):
+		return ()
+
+	if "type" not in skipped_types and stat.S_IFMT(file1_stat.st_mode) != stat.S_IFMT(file2_stat.st_mode):
+		differences.append("type")
+	if "mode" not in skipped_types and stat.S_IMODE(file1_stat.st_mode) != stat.S_IMODE(file2_stat.st_mode):
+		differences.append("mode")
+	if "owner" not in skipped_types and file1_stat.st_uid != file2_stat.st_uid:
+		differences.append("owner")
+	if "group" not in skipped_types and file1_stat.st_gid != file2_stat.st_gid:
+		differences.append("group")
+	if "device_number" not in skipped_types and file1_stat.st_rdev != file2_stat.st_rdev:
+		differences.append("device_number")
+
+	if "xattr" not in skipped_types and sorted(xattr.get_all(file1, nofollow=True)) != sorted(xattr.get_all(file2, nofollow=True)):
+		differences.append("xattr")
+
+	if sys.hexversion >= 0x3030000:
+		if "atime" not in skipped_types and file1_stat.st_atime_ns != file2_stat.st_atime_ns:
+			differences.append("atime")
+		if "mtime" not in skipped_types and file1_stat.st_mtime_ns != file2_stat.st_mtime_ns:
+			differences.append("mtime")
+		if "ctime" not in skipped_types and file1_stat.st_ctime_ns != file2_stat.st_ctime_ns:
+			differences.append("ctime")
+	else:
+		if "atime" not in skipped_types and file1_stat.st_atime != file2_stat.st_atime:
+			differences.append("atime")
+		if "mtime" not in skipped_types and file1_stat.st_mtime != file2_stat.st_mtime:
+			differences.append("mtime")
+		if "ctime" not in skipped_types and file1_stat.st_ctime != file2_stat.st_ctime:
+			differences.append("ctime")
+
+	if "type" in differences:
+		pass
+	elif file1_stat.st_size != file2_stat.st_size:
+		if "size" not in skipped_types:
+			differences.append("size")
+		if "content" not in skipped_types:
+			differences.append("content")
+	else:
+		if "content" not in skipped_types:
+			if stat.S_ISLNK(file1_stat.st_mode):
+				file1_stream = io.BytesIO(os.readlink(_unicode_encode(file1,
+									encoding=_encodings["fs"],
+									errors="strict")))
+			else:
+				file1_stream = open(_unicode_encode(file1,
+							encoding=_encodings["fs"],
+							errors="strict"), "rb")
+			if stat.S_ISLNK(file2_stat.st_mode):
+				file2_stream = io.BytesIO(os.readlink(_unicode_encode(file2,
+									encoding=_encodings["fs"],
+									errors="strict")))
+			else:
+				file2_stream = open(_unicode_encode(file2,
+							encoding=_encodings["fs"],
+							errors="strict"), "rb")
+			while True:
+				file1_content = file1_stream.read(4096)
+				file2_content = file2_stream.read(4096)
+				if file1_content != file2_content:
+					differences.append("content")
+					break
+				if not file1_content or not file2_content:
+					break
+			file1_stream.close()
+			file2_stream.close()
+
+	return tuple(differences)