[gentoo-commits] portage r11565 - in main/branches/prefix: bin pym/portage/dbapi

[Fabian Groffen (grobian)]

Author: grobian
Date: 2008-09-27 14:52:10 +0000 (Sat, 27 Sep 2008)
New Revision: 11565

Modified:
   main/branches/prefix/bin/misc-functions.sh
   main/branches/prefix/pym/portage/dbapi/vartree.py
Log:
Now we have scanmacho, we can eventually do some fast examination of libraries and get all info at once that we need.  Get rid of otool, and now also store the arch of objects (enabling theoretical support for FAT objects), hence bump to NEEDED.MACHO.3

Modified: main/branches/prefix/bin/misc-functions.sh
===================================================================
--- main/branches/prefix/bin/misc-functions.sh	2008-09-27 05:55:10 UTC (rev 11564)
+++ main/branches/prefix/bin/misc-functions.sh	2008-09-27 14:52:10 UTC (rev 11565)
@@ -407,53 +407,56 @@
 	done
 	[[ ${abort} == "yes" ]] && die "soiled libtool library files found"
 
-	# Check that we don't get kernel traps at runtime because of broken
-	# install_names on Darwin, at the same time generate the NEEDED
-	# entries.  As long as we don't have a "scanelf" tool for this, we
-	# use otool to do the magic.  Since this is expensive, we do it
-	# together with the scan for broken installs.
+	# While we generate the NEEDED files, check that we don't get kernel
+	# traps at runtime because of broken install_names on Darwin.
 	rm -f "${T}"/.install_name_check_failed
-	[[ ${CHOST} == *-darwin* ]] && find "${ED}" -type f | while IFS= read f ; do
-		rm -f "${T}"/.NEEDED.tmp
-		install_name=$(otool -DX "${f}")
-		otool -LX "${f}" \
-			| grep -v "Archive : " \
-			| sed -e 's/^\t//' -e 's/ (compa.*$//' \
-			| while read r ;
-		do
-			# skip the self reference in libraries
-			[[ -n ${install_name} && ${install_name} == ${r} ]] && continue
+	[[ ${CHOST} == *-darwin* ]] && scanmacho -qyRF '%a;%p;%S;%n' "${D}" | { while IFS= read l ; do
+		arch=${l%%;*}; l=${l#*;}
+		obj="/${l%%;*}"; l=${l#*;}
+		install_name=${l%%;*}; l=${l#*;}
+		needed=${l%%;*}; l=${l#*;}
 
-			if [[ ! -e ${r} && ! -e ${D}${r} && ${r} != *"@executable_path"* ]] ; then
+		# this is ugly, paths with spaces won't work
+		reevaluate=0
+		for lib in $(echo ${needed} | tr , ' '); do
+			if [[ ! -e ${lib} && ! -e ${D}${lib} && ${lib} != *"@executable_path"* ]] ; then
 				# try to "repair" this if possible, happens because of
 				# gen_usr_ldscript tactics
-				s=${r%usr/*}${r##*/usr/}
+				s=${lib%usr/*}${lib##*/usr/}
 				if [[ -e ${D}${s} ]] ; then
-					ewarn "correcting install_name from ${r} to ${s} in ${f#${D}}"
+					ewarn "correcting install_name from ${lib} to ${s} in ${obj}"
 					install_name_tool -change \
-						"${r}" "${s}" "${f}"
-					r=${s} # for the NEEDED entries
+						"${lib}" "${s}" "${D}${obj}"
+					reevaluate=1
 				else
-					eqawarn "QA Notice: invalid reference to ${r} in ${f}"
+					eqawarn "QA Notice: invalid reference to ${lib} in ${obj}"
 					# remember we are in an implicit subshell, that's
 					# why we touch a file here ... ideally we should be
 					# able to die correctly/nicely here
 					touch "${T}"/.install_name_check_failed
 				fi
 			fi
-			echo -n ",${r}" >> "${T}"/.NEEDED.tmp
 		done
-		if [[ -f "${T}"/.NEEDED.tmp ]] ; then
-			needed=$(< "${T}"/.NEEDED.tmp)
-			echo "/${f#${D}} ${needed#,}" >> "${PORTAGE_BUILDDIR}"/build-info/NEEDED
-			echo "/${f#${D}};${install_name};${needed#,}" >> "${PORTAGE_BUILDDIR}"/build-info/NEEDED.MACHO.2
+		if [[ reevaluate == 1 ]]; then
+			# install_name(s) have been changed, refresh data so we
+			# store the correct meta data
+			l=$(scanmacho -qyF '%a;%p;%S;%n' ${D}${obj})
+			arch=${l%%;*}; l=${l#*;}
+			obj="/${l%%;*}"; l=${l#*;}
+			install_name=${l%%;*}; l=${l#*;}
+			needed=${l%%;*}; l=${l#*;}
 		fi
-	done
+
+		# backwards compatability
+		echo "${obj} ${needed}" >> "${PORTAGE_BUILDDIR}"/build-info/NEEDED
+		# what we use
+		echo "${arch};${obj};${install_name};${needed}" >> "${PORTAGE_BUILDDIR}"/build-info/NEEDED.MACHO.3
+	done }
 	if [[ -f ${T}/.install_name_check_failed ]] ; then
 		# secret switch "allow_broken_install_names" to get
 		# around this and install broken crap (not a good idea)
 		hasq allow_broken_install_names ${FEATURES} || \
-			die "invalid install_name found, your application will crash at runtime"
+			die "invalid install_name found, your application or library will crash at runtime"
 	fi
 
 	# Evaluate misc gcc warnings

Modified: main/branches/prefix/pym/portage/dbapi/vartree.py
===================================================================
--- main/branches/prefix/pym/portage/dbapi/vartree.py	2008-09-27 05:55:10 UTC (rev 11564)
+++ main/branches/prefix/pym/portage/dbapi/vartree.py	2008-09-27 14:52:10 UTC (rev 11565)
@@ -622,10 +622,6 @@
 			return hash(self._key)
 
 		def __eq__(self, other):
-			if isinstance(other, self.__class__):
-				other_key = other._key
-			else:
-				other_key = other
 			return self._key == other_key
 
 		def _generate_object_key(self, object):
@@ -669,59 +665,39 @@
 		obj_properties = {}
 		lines = []
 		for cpv in self._dbapi.cpv_all():
-			lines += self._dbapi.aux_get(cpv, ["NEEDED.MACHO.2"])[0].split('\n')
+			lines += self._dbapi.aux_get(cpv, ["NEEDED.MACHO.3"])[0].split('\n')
 		# Cache NEEDED.* files avoid doing excessive IO for every rebuild.
 		self._dbapi.flush_cache()
 
 		if include_file:
 			lines += grabfile(include_file)
 
-		# have to call otool for preserved libs here as they aren't 
-		# registered in NEEDED.MACHO.2 files
+		# have to call scanmacho for preserved libs here as they aren't 
+		# registered in NEEDED.MACHO.3 files
 		if self._dbapi.plib_registry and self._dbapi.plib_registry.getPreservedLibs():
-			otool = EPREFIX+"/usr/bin/otool"
+			args = [EPREFIX+"/usr/bin/scanmacho", "-qF", "%a;%F;%S;%n"]
 			for items in self._dbapi.plib_registry.getPreservedLibs().values():
-				for x in items:
-					try:
-						proc = subprocess.Popen([otool, "-DX", x.lstrip(".")],
-								stdout=subprocess.PIPE)
-						install_name = proc.communicate()[0].split("\n")[0]
-						proc = subprocess.Popen([otool, "-LX", x.lstrip(".")],
-								stdout=subprocess.PIPE)
-						output = [l.lstrip() for l in proc.communicate()[0].split("\n")]
-					except OSError:
-						# if otool can't be found (like in an odcctools
-						# upgrade -> binutils-config needs to update
-						# symlinks) just ignore it, don't crash
-						lines += [x + ";" + x.lstrip(".") + ";\n"]
-						continue
+				args += [x.lstrip(".") for x in items]
+			proc = subprocess.Popen(args, stdout=subprocess.PIPE)
+			output = [proc.communicate()[0].split("\n")]
+			lines += output
 
-					n = ""
-					for l in output:
-						if l == '':
-							continue
-						p = l.rfind(" (compatibility")
-						if p != -1:
-							l = l[:p]
-						n += "," + l
-
-					lines += [x + ";" + install_name + ";" + n.lstrip(",") + "\n"]
-
 		for l in lines:
 			if l.strip() == "":
 				continue
 			fields = l.strip("\n").split(";")
-			if len(fields) < 3:
+			if len(fields) < 4:
 				print "Error", fields
 				# insufficient field length
 				continue
 
 			# Linking an object to a library is registered by recording
 			# the install_name of the library in the object.
-			obj = fields[0]
+			arch = fields[0]
+			obj = fields[1]
 			obj_key = self._ObjectKey(obj)
-			install_name = os.path.normpath(fields[1])
-			needed = filter(None, fields[2].split(","))
+			install_name = os.path.normpath(fields[2])
+			needed = filter(None, fields[3].split(","))
 
 			# build an internal structure that contains for each
 			# install_name, what libs have that install_name
@@ -730,16 +706,19 @@
 			# reference the install_name
 			if install_name:
 				libs.setdefault(install_name, \
+						{arch: {"providers": set(), "consumers": set()}})
+				libs[install_name].setdefault(arch, \
 						{"providers": set(), "consumers": set()})
-				libs[install_name]["providers"].add(obj_key)
+				libs[install_name][arch]["providers"].add(obj_key)
 			for x in needed:
 				libs.setdefault(x, \
-						{"providers": set(), "consumers": set()})
-				libs[x]["consumers"].add(obj_key)
+						{arch: {"providers": set(), "consumers": set()}})
+				libs[x].setdefault(arch, {"providers": set(), "consumers": set()})
+				libs[x][arch]["consumers"].add(obj_key)
 			obj_key_cache.setdefault(obj, obj_key)
 			# All object paths are added into the obj_properties tuple
 			obj_properties.setdefault(obj_key, \
-					(needed, install_name, set()))[2].add(obj)
+					(arch, needed, install_name, set()))[3].add(obj)
 		
 		self._libs = libs
 		self._obj_properties = obj_properties
@@ -798,13 +777,13 @@
 					if obj_key.file_exists():
 						# Get the install_name from LinkageMapMachO._obj_properties if
 						# it exists. Otherwise, None.
-						_, install_name, _ = \
-								self._obj_properties.get(obj_key, (None,)*3)
+						arch, _, install_name, _ = \
+								self._obj_properties.get(obj_key, (None,)*4)
 						return cache_self.cache.setdefault(obj, \
-								(install_name, obj_key, True))
+								(arch, install_name, obj_key, True))
 					else:
 						return cache_self.cache.setdefault(obj, \
-								(None, obj_key, False))
+								(None, None, obj_key, False))
 
 		rValue = {}
 		cache = _LibraryCache()
@@ -812,7 +791,7 @@
 
 		# Iterate over all obj_keys and their providers.
 		for obj_key, install_names in providers.items():
-			_, _, objs = self._obj_properties[obj_key]
+			arch, _, _, objs = self._obj_properties[obj_key]
 			# Iterate over each needed install_name and the set of
 			# library paths that fulfill the install_name to determine
 			# if the dependency is broken.
@@ -823,12 +802,12 @@
 				# If unsatisfied, objects associated with obj_key must
 				# be emerged.
 				validLibrary = None
-				cachedInstallname, cachedRealpath, cachedExists = \
+				cachedArch, cachedInstallname, cachedRealpath, cachedExists = \
 						cache.get(install_name)
 				# Check that the this library provides the needed soname.  Doing
 				# this, however, will cause consumers of libraries missing
 				# sonames to be unnecessarily emerged. (eg libmix.so)
-				if cachedInstallname == install_name:
+				if cachedInstallname == install_name and cachedArch == arch:
 					validLibrary = cachedRealpath
 					if debug and cachedRealpath not in libraries:
 						print "Found provider outside of findProviders:", \
@@ -894,7 +873,7 @@
 		obj_key = self._ObjectKey(obj)
 		if obj_key not in self._obj_properties:
 			raise KeyError("%s (%s) not in object list" % (obj_key, obj))
-		install_name = self._obj_properties[obj_key][1]
+		install_name = self._obj_properties[obj_key][2]
 		return (len(basename) < len(os.path.basename(install_name)))
 
 	def listLibraryObjects(self):
@@ -911,8 +890,9 @@
 		if not self._libs:
 			self.rebuild()
 		for install_name in self._libs:
-			for obj_key in self._libs[install_name]["providers"]:
-				rValue.extend(self._obj_properties[obj_key][2])
+			for arch in self._libs[install_name]:
+				for obj_key in self._libs[install_name][arch]["providers"]:
+					rValue.extend(self._obj_properties[obj_key][3])
 		return rValue
 	
 	def getSoname(self, obj):
@@ -929,7 +909,7 @@
 			self.rebuild()
 		if obj not in self._obj_key_cache:
 			raise KeyError("%s not in object list" % obj)
-		return self._obj_properties[self._obj_key_cache[obj]][1]
+		return self._obj_properties[self._obj_key_cache[obj]][2]
 
 	def findProviders(self, obj):
 		"""
@@ -968,15 +948,15 @@
 				if obj_key not in self._obj_properties:
 					raise KeyError("%s (%s) not in object list" % (obj_key, obj))
 
-		needed, install_name, _ = self._obj_properties[obj_key]
+		arch, needed, install_name, _ = self._obj_properties[obj_key]
 		for install_name in needed:
 			rValue[install_name] = set()
-			if install_name not in self._libs:
+			if install_name not in self._libs or arch not in self._libs[install_name]:
 				continue
 			# For each potential provider of the install_name, add it to
 			# rValue if it exists.  (Should be one)
-			for provider_key in self._libs[install_name]["providers"]:
-				providers = self._obj_properties[provider_key][2]
+			for provider_key in self._libs[install_name][arch]["providers"]:
+				providers = self._obj_properties[provider_key][3]
 				for provider in providers:
 					if os.path.exists(provider):
 						rValue[install_name].add(provider)
@@ -1012,7 +992,7 @@
 			obj_key = obj
 			if obj_key not in self._obj_properties:
 				raise KeyError("%s not in object list" % obj_key)
-			objs = self._obj_properties[obj_key][2]
+			objs = self._obj_properties[obj_key][3]
 		else:
 			objs = set([obj])
 			obj_key = self._obj_key_cache.get(obj)
@@ -1026,7 +1006,7 @@
 		# other version, this lib will be shadowed and won't
 		# have any consumers.
 		if not isinstance(obj, self._ObjectKey):
-			master_link = self._obj_properties[obj_key][1]
+			master_link = self._obj_properties[obj_key][2]
 			try:
 				master_st = os.stat(master_link)
 				obj_st = os.stat(obj)
@@ -1037,12 +1017,12 @@
 					(master_st.st_dev, master_st.st_ino):
 					return set()
 
-		_, install_name, _ = self._obj_properties[obj_key]
-		if install_name in self._libs:
+		arch, _, install_name, _ = self._obj_properties[obj_key]
+		if install_name in self._libs and arch in self._libs[install_name]:
 			# For each potential consumer, add it to rValue if an object from the
 			# arguments resides in the consumer's runpath.
-			for consumer_key in self._libs[install_name]["consumers"]:
-				_, _, consumer_objs = \
+			for consumer_key in self._libs[install_name][arch]["consumers"]:
+				_, _, _, consumer_objs = \
 						self._obj_properties[consumer_key]
 				rValue.update(consumer_objs)
 		return rValue