[gentoo-commits] portage r12132 - in main/branches/prefix: bin man pym/_emerge pym/portage pym/portage/dbapi

[Fabian Groffen (grobian)]

Author: grobian
Date: 2008-12-01 21:10:53 +0000 (Mon, 01 Dec 2008)
New Revision: 12132

Modified:
   main/branches/prefix/bin/ebuild.sh
   main/branches/prefix/bin/isolated-functions.sh
   main/branches/prefix/bin/repoman
   main/branches/prefix/man/portage.5
   main/branches/prefix/pym/_emerge/__init__.py
   main/branches/prefix/pym/portage/__init__.py
   main/branches/prefix/pym/portage/dbapi/vartree.py
Log:
   Merged from trunk -r12086:12117

   | 12090   | Note that file names in /etc/portage/package.* directories   |
   | zmedico | are sorted ascending alphabetical order before being summed  |
   |         | together.                                                    |
   
   | 12092   | Bug #248782 - Handle permission error in                     |
   | zmedico | EbuildFetchonly.execute() if PORTAGE_TMPDIR is not writable. |
   
   | 12094   | Bug #248464 - With git, there's never any keyword expansion, |
   | zmedico | so there's no need to regenerate manifests and all files     |
   |         | will be committed in one big commit at the end.              |
   
   | 12096   | Remove redundant reference to 'mynew' since 'myupdates'      |
   | zmedico | already contains those files.                                |
   
   | 12098   | Eliminate redundant Manifest separation code by combining    |
   | zmedico | mychanged + mynew sooner.                                    |
   
   | 12101   | Add some more clarification to the 'ebuild phase exited      |
   | zmedico | unexpectedly' message.                                       |
   
   | 12103   | Make Scheduler.merge() bail out early if PORTAGE_TMPDIR is   |
   | zmedico | not properly set.                                            |
   
   | 12105   | Bug #205044 - When creating $EBUILD_EXIT_STATUS_FILE, don't  |
   | zmedico | direct output to /dev/null since it should never fail and if |
   |         | it does then the error message might be useful.              |
   
   | 12107   | Don't direct to /dev/null when creating                      |
   | zmedico | $EBUILD_EXIT_STATUS_FILE inside die.                         |
   
   | 12109   | Bug #235642 - Create hardlinks when merging identical files. |
   | zmedico | This works by using a tuple of (md5, st_size) as a key to a  |
   |         | list of hardlink candidates. Multiple candidates are used in |
   |         | case some happen to be merged to separate devices.           |
   
   | 12111   | In movefile(), ignore the hardlink_candidates parameter when |
   | zmedico | it's an empty list.                                          |
   
   | 12113   | For bug #235642, include the stat mode, uid, and gid bits in |
   | zmedico | the hardlink key.                                            |
   
   | 12115   | Don't call prepare_build_dirs() inside doebuild() when       |
   | zmedico | called for parallel fetching.                                |
   
   | 12117   | Use stat st_dev attributes instead of the older approach.    |
   | zmedico |                                                              |


Modified: main/branches/prefix/bin/ebuild.sh
===================================================================
--- main/branches/prefix/bin/ebuild.sh	2008-12-01 20:59:48 UTC (rev 12131)
+++ main/branches/prefix/bin/ebuild.sh	2008-12-01 21:10:53 UTC (rev 12132)
@@ -275,9 +275,13 @@
 # Ensure that $PWD is sane whenever possible, to protect against
 # exploitation of insecure search path for python -c in ebuilds.
 # See bug #239560.
-if ! hasq "$EBUILD_PHASE" clean depend help ; then
+if ! hasq "$EBUILD_PHASE" clean cleanrm depend help ; then
 	cd "$PORTAGE_BUILDDIR" || \
 		die "PORTAGE_BUILDDIR does not exist: '$PORTAGE_BUILDDIR'"
+else
+	# Don't try to create this when it's parent
+	# directory doesn't necessarily exist.
+	unset EBUILD_EXIT_STATUS_FILE
 fi
 
 #if no perms are specified, dirs/files will have decent defaults
@@ -2087,8 +2091,10 @@
 		exit 1
 		;;
 	esac
-	[ -n "${EBUILD_EXIT_STATUS_FILE}" ] && \
-		touch "${EBUILD_EXIT_STATUS_FILE}" &>/dev/null
+	if [ -n "$EBUILD_EXIT_STATUS_FILE" ] ; then
+		> "$EBUILD_EXIT_STATUS_FILE" || \
+			die "failed to create '$EBUILD_EXIT_STATUS_FILE'"
+	fi
 }
 
 [[ -n $EBUILD_SH_ARGS ]] && ebuild_main

Modified: main/branches/prefix/bin/isolated-functions.sh
===================================================================
--- main/branches/prefix/bin/isolated-functions.sh	2008-12-01 20:59:48 UTC (rev 12131)
+++ main/branches/prefix/bin/isolated-functions.sh	2008-12-01 21:10:53 UTC (rev 12132)
@@ -128,22 +128,8 @@
 		done
 	fi
 
-	[[ -n ${PORTAGE_LOG_FILE} ]] \
-		&& eerror "build log: '${PORTAGE_LOG_FILE}'"
-	if [ -f "${T}/environment" ] ; then
-		eerror "ebuild environment: '${T}/environment'"
-	elif [ -d "${T}" ] ; then
-		{
-			set
-			export
-		} > "${T}/die.env"
-		eerror "ebuild environment: '${T}/die.env'"
-	fi
-	eerror "S: '${S}'"
+	[ -n "$EBUILD_EXIT_STATUS_FILE" ] && > "$EBUILD_EXIT_STATUS_FILE"
 
-	[ -n "${EBUILD_EXIT_STATUS_FILE}" ] && \
-		touch "${EBUILD_EXIT_STATUS_FILE}" &>/dev/null
-
 	# subshell die support
 	kill -s SIGTERM ${EBUILD_MASTER_PID}
 	exit 1

Modified: main/branches/prefix/bin/repoman
===================================================================
--- main/branches/prefix/bin/repoman	2008-12-01 20:59:48 UTC (rev 12131)
+++ main/branches/prefix/bin/repoman	2008-12-01 21:10:53 UTC (rev 12132)
@@ -1758,23 +1758,14 @@
 	# Manifests need to be regenerated after all other commits, so don't commit
 	# them now even if they have changed.
 	mymanifests = set()
-	changed_set = set()
-	new_set = set()
-	for f in mychanged:
+	myupdates = set()
+	for f in mychanged + mynew:
 		if "Manifest" == os.path.basename(f):
 			mymanifests.add(f)
 		else:
-			changed_set.add(f)
-	for f in mynew:
-		if "Manifest" == os.path.basename(f):
-			mymanifests.add(f)
-		else:
-			new_set.add(f)
-	mychanged = list(changed_set)
-	mynew =  list(new_set)
+			myupdates.add(f)
+	myupdates = list(myupdates)
 	mymanifests = list(mymanifests)
-	del changed_set, new_set
-	myupdates = mychanged + mynew
 	myheaders = []
 	mydirty = []
 	headerstring = "'\$(Header|Id)"
@@ -1795,8 +1786,16 @@
 		if myout[0] == 0:
 			myheaders.append(myfile)
 
-	print "*",green(str(len(myupdates))),"files being committed...",green(str(len(myheaders))),"have headers that will change."
-	print "*","Files with headers will cause the manifests to be made and recommited."
+	print "* %s files being committed..." % green(str(len(myupdates))),
+	if vcs == 'git':
+		# With git, there's never any keyword expansion, so there's
+		# no need to regenerate manifests and all files will be
+		# committed in one big commit at the end.
+		print
+	else:
+		print "%s have headers that will change." % green(str(len(myheaders)))
+		print "* Files with headers will cause the " + \
+			"manifests to be made and recommited."
 	logging.info("myupdates:", str(myupdates))
 	logging.info("myheaders:", str(myheaders))
 
@@ -1844,7 +1843,7 @@
 		commitmessage += ", RepoMan options: --force"
 	commitmessage += ")"
 
-	if myupdates or myremoved:
+	if vcs != 'git' and (myupdates or myremoved):
 		myfiles = myupdates + myremoved
 		if not myheaders and "sign" not in repoman_settings.features:
 			myfiles += mymanifests
@@ -1937,8 +1936,8 @@
 			write_atomic(x, "".join(mylines))
 
 	manifest_commit_required = True
-	if myupdates or myremoved or mynew:
-		myfiles=myupdates+myremoved+mynew
+	if vcs != 'git' and (myupdates or myremoved):
+		myfiles = myupdates + myremoved
 		for x in range(len(myfiles)-1, -1, -1):
 			if myfiles[x].count("/") < 4-repolevel:
 				del myfiles[x]
@@ -2059,8 +2058,14 @@
 			portage.writemsg("!!! Disabled FEATURES='sign'\n")
 			signed = False
 
-	if manifest_commit_required or signed:
+	if vcs == 'git' or manifest_commit_required or signed:
 
+		myfiles = mymanifests[:]
+		if vcs == 'git':
+			myfiles += myupdates
+			myfiles += myremoved
+		myfiles.sort()
+
 		fd, commitmessagefile = tempfile.mkstemp(".repoman.msg")
 		mymsg = os.fdopen(fd, "w")
 		mymsg.write(commitmessage)
@@ -2075,7 +2080,7 @@
 		commit_cmd.append("commit")
 		commit_cmd.extend(vcs_local_opts)
 		commit_cmd.extend(["-F", commitmessagefile])
-		commit_cmd.extend(f.lstrip("./") for f in mymanifests)
+		commit_cmd.extend(f.lstrip("./") for f in myfiles)
 
 		try:
 			if options.pretend:

Modified: main/branches/prefix/man/portage.5
===================================================================
--- main/branches/prefix/man/portage.5	2008-12-01 20:59:48 UTC (rev 12131)
+++ main/branches/prefix/man/portage.5	2008-12-01 21:10:53 UTC (rev 12132)
@@ -371,13 +371,14 @@
 .BR /etc/portage/
 Any file in this directory that begins with "package." can be more than just a 
 flat file.  If it is a directory, then all the files in that directory will be 
-summed together as if it were a single file.
+sorted in ascending alphabetical order by file name and summed together as if
+it were a single file.
 
 .I Example:
 .nf
-/etc/portage/package.keywords/kde
 /etc/portage/package.keywords/common
 /etc/portage/package.keywords/e17
+/etc/portage/package.keywords/kde
 .fi
 .RS
 .TP

Modified: main/branches/prefix/pym/_emerge/__init__.py
===================================================================
--- main/branches/prefix/pym/_emerge/__init__.py	2008-12-01 20:59:48 UTC (rev 12131)
+++ main/branches/prefix/pym/_emerge/__init__.py	2008-12-01 21:10:53 UTC (rev 12132)
@@ -1632,7 +1632,12 @@
 		settings = self.settings
 		global_tmpdir = settings["PORTAGE_TMPDIR"]
 		from tempfile import mkdtemp
-		private_tmpdir = mkdtemp("", "._portage_fetch_.", global_tmpdir)
+		try:
+			private_tmpdir = mkdtemp("", "._portage_fetch_.", global_tmpdir)
+		except OSError, e:
+			if e.errno != portage.exception.PermissionDenied.errno:
+				raise
+			raise portage.exception.PermissionDenied(global_tmpdir)
 		settings["PORTAGE_TMPDIR"] = private_tmpdir
 		settings.backup_changes("PORTAGE_TMPDIR")
 		try:
@@ -9734,6 +9739,22 @@
 
 		for root in self.trees:
 			root_config = self.trees[root]["root_config"]
+
+			# Even for --pretend --fetch mode, PORTAGE_TMPDIR is required
+			# since it might spawn pkg_nofetch which requires PORTAGE_BUILDDIR
+			# for ensuring sane $PWD (bug #239560) and storing elog messages.
+			tmpdir = root_config.settings.get("PORTAGE_TMPDIR", "")
+			if not tmpdir or not os.path.isdir(tmpdir):
+				msg = "The directory specified in your " + \
+					"PORTAGE_TMPDIR variable, '%s', " % tmpdir + \
+				"does not exist. Please create this " + \
+				"directory or correct your PORTAGE_TMPDIR setting."
+				msg = textwrap.wrap(msg, 70)
+				out = portage.output.EOutput()
+				for l in msg:
+					out.eerror(l)
+				return 1
+
 			if self._background:
 				root_config.settings.unlock()
 				root_config.settings["PORTAGE_BACKGROUND"] = "1"

Modified: main/branches/prefix/pym/portage/__init__.py
===================================================================
--- main/branches/prefix/pym/portage/__init__.py	2008-12-01 20:59:48 UTC (rev 12131)
+++ main/branches/prefix/pym/portage/__init__.py	2008-12-01 21:10:53 UTC (rev 12132)
@@ -5190,9 +5190,20 @@
 	"is known to be triggered " + \
 	"by things such as failed variable " + \
 	"assignments (bug #190128) or bad substitution " + \
-	"errors (bug #200313). This behavior may also be " + \
-	"triggered by a corrupt bash binary or a hardware " + \
-	"problem such as memory or cpu malfunction."
+	"errors (bug #200313). Normally, before exiting, bash should " + \
+	"have displayed an error message above. If bash did not " + \
+	"produce an error message above, it's possible " + \
+	"that the ebuild has called `exit` when it " + \
+	"should have called `die` instead. This behavior may also " + \
+	"be triggered by a corrupt bash binary or a hardware " + \
+	"problem such as memory or cpu malfunction. If the problem is not " + \
+	"reproducible or it appears to occur randomly, then it is likely " + \
+	"to be triggered by a hardware problem. " + \
+	"If you suspect a hardware problem then you should " + \
+	"try some basic hardware diagnostics such as memtest. " + \
+	"Please do not report this as a bug unless it is consistently " + \
+	"reproducible and you are sure that your bash binary and hardware " + \
+	"are functioning properly."
 	return msg
 
 def _doebuild_exit_status_check_and_log(settings, mydo, retval):
@@ -5331,6 +5342,9 @@
 		fetchall = 1
 		mydo = "fetch"
 
+	parallel_fetchonly = mydo in ("fetch", "fetchall") and \
+		"PORTAGE_PARALLEL_FETCHONLY" in mysettings
+
 	if mydo not in clean_phases and not os.path.exists(myebuild):
 		writemsg("!!! doebuild: %s not found for %s\n" % (myebuild, mydo),
 			noiselevel=-1)
@@ -5564,7 +5578,7 @@
 
 		# Build directory creation isn't required for any of these.
 		have_build_dirs = False
-		if not mydo in ("digest", "help", "manifest"):
+		if not parallel_fetchonly and mydo not in ("digest", "help", "manifest"):
 			mystatus = prepare_build_dirs(myroot, mysettings, cleanup)
 			if mystatus:
 				return mystatus
@@ -6005,7 +6019,8 @@
 		raise portage.exception.PortageException(
 			"mv '%s' '%s'" % (src, dest))
 
-def movefile(src,dest,newmtime=None,sstat=None,mysettings=None):
+def movefile(src, dest, newmtime=None, sstat=None, mysettings=None,
+		hardlink_candidates=None):
 	"""moves a file from src to dest, preserving all permissions and attributes; mtime will
 	be preserved even when moving across filesystems.  Returns true on success and false on
 	failure.  Move is atomic."""
@@ -6077,8 +6092,44 @@
 			print "!!!",e
 			return None
 
+	hardlinked = False
+	# Since identical files might be merged to multiple filesystems,
+	# so os.link() calls might fail for some paths, so try them all.
+	# For atomic replacement, first create the link as a temp file
+	# and them use os.rename() to replace the destination.
+	if hardlink_candidates:
+		head, tail = os.path.split(dest)
+		hardlink_tmp = os.path.join(head, ".%s._portage_merge_.%s" % \
+			(tail, os.getpid()))
+		try:
+			os.unlink(hardlink_tmp)
+		except OSError, e:
+			if e.errno != errno.ENOENT:
+				writemsg("!!! Failed to remove hardlink temp file: %s\n" % \
+					(hardlink_tmp,), noiselevel=-1)
+				writemsg("!!! %s\n" % (e,), noiselevel=-1)
+				return None
+			del e
+		for hardlink_src in hardlink_candidates:
+			try:
+				os.link(hardlink_src, hardlink_tmp)
+			except OSError:
+				continue
+			else:
+				try:
+					os.rename(hardlink_tmp, dest)
+				except OSError, e:
+					writemsg("!!! Failed to rename %s to %s\n" % \
+						(hardlink_tmp, dest), noiselevel=-1)
+					writemsg("!!! %s\n" % (e,), noiselevel=-1)
+					return None
+				hardlinked = True
+				break
+
 	renamefailed=1
-	if sstat[stat.ST_DEV]==dstat[stat.ST_DEV] or selinux_enabled:
+	if hardlinked:
+		renamefailed = False
+	if not hardlinked and (selinux_enabled or sstat.st_dev == dstat.st_dev):
 		try:
 			if selinux_enabled:
 				ret=selinux.secure_rename(src,dest)
@@ -6139,11 +6190,14 @@
 			return None
 
 	try:
-		if newmtime is not None:
-			os.utime(dest, (newmtime, newmtime))
+		if hardlinked:
+			newmtime = long(os.stat(dest).st_mtime)
 		else:
-			os.utime(dest, (sstat.st_atime, sstat.st_mtime))
-			newmtime = long(sstat.st_mtime)
+			if newmtime is not None:
+				os.utime(dest, (newmtime, newmtime))
+			else:
+				os.utime(dest, (sstat.st_atime, sstat.st_mtime))
+				newmtime = long(sstat.st_mtime)
 	except OSError:
 		# The utime can fail here with EPERM even though the move succeeded.
 		# Instead of failing, use stat to return the mtime if possible.

Modified: main/branches/prefix/pym/portage/dbapi/vartree.py
===================================================================
--- main/branches/prefix/pym/portage/dbapi/vartree.py	2008-12-01 20:59:48 UTC (rev 12131)
+++ main/branches/prefix/pym/portage/dbapi/vartree.py	2008-12-01 21:10:53 UTC (rev 12132)
@@ -2282,6 +2282,7 @@
 		self._contents_inodes = None
 		self._contents_basenames = None
 		self._linkmap_broken = False
+		self._md5_merge_map = {}
 
 	def lockdb(self):
 		if self._lock_vdb:
@@ -3895,6 +3896,7 @@
 			if self.mergeme(srcroot, destroot, outfile, None,
 				secondhand, cfgfiledict, mymtime):
 				return 1
+		self._md5_merge_map.clear()
 
 		#restore umask
 		os.umask(prevmask)
@@ -4296,9 +4298,18 @@
 				# whether config protection or not, we merge the new file the
 				# same way.  Unless moveme=0 (blocking directory)
 				if moveme:
-					mymtime = movefile(mysrc, mydest, newmtime=thismtime, sstat=mystat, mysettings=self.settings)
+					hardlink_key = (mymd5, mystat.st_size,
+						mystat.st_mode, mystat.st_uid, mystat.st_gid)
+					hardlink_candidates = self._md5_merge_map.get(hardlink_key)
+					if hardlink_candidates is None:
+						hardlink_candidates = []
+						self._md5_merge_map[hardlink_key] = hardlink_candidates
+					mymtime = movefile(mysrc, mydest, newmtime=thismtime,
+						sstat=mystat, mysettings=self.settings,
+						hardlink_candidates=hardlink_candidates)
 					if mymtime is None:
 						return 1
+					hardlink_candidates.append(mydest)
 					zing = ">>>"
 
 				if mymtime != None: