[gentoo-commits] portage r11734 - in main/branches/prefix: bin pym/_emerge pym/portage pym/portage/dbapi

[Fabian Groffen (grobian)]

Author: grobian
Date: 2008-10-28 18:47:54 +0000 (Tue, 28 Oct 2008)
New Revision: 11734

Modified:
   main/branches/prefix/bin/isolated-functions.sh
   main/branches/prefix/bin/misc-functions.sh
   main/branches/prefix/pym/_emerge/__init__.py
   main/branches/prefix/pym/portage/__init__.py
   main/branches/prefix/pym/portage/dbapi/vartree.py
Log:
   Merged from trunk -r11701:11713

   | 11702   | Remove the "found_available_arg" variable from               |
   | zmedico | depgraph._select_pkg() and use the "matched_packages"        |
   |         | variable instead. This solve a problem where the             |
   |         | "found_available_arg" flag is unreliable due to the slot     |
   |         | filtering behavior of depgraph._iter_atoms_for_pkg(). For    |
   |         | example, a higher version in a different slot than the       |
   |         | available ebuild might be masked or unavailable, and         |
   |         | depgraph._iter_atoms_for_pkg() will not identify the         |
   |         | available ebuild as a match due to this installed package    |
   |         | (which is not available for reinstall).                      |
   
   | 11703   | Simplify logic from the previous commit.                     |
   | zmedico |                                                              |
   
   | 11704   | Revert previous 2 commits since they broke "selective"       |
   | zmedico | behavior.                                                    |
   
   | 11705   | Handle the case that was intended to be fixed by r11702      |
   | zmedico | (which got reverted).                                        |
   
   | 11706   | Prefer thirdpartymirrors over normal mirrors in cases when   |
   | zmedico | the file does not yet exist on the normal mirrors. Thanks to |
   |         | Maurice van der Pot <griffon26@g.o> for reporting.           |
   
   | 11707   | Remove redundant duplicate uri elimination code. It's        |
   | zmedico | already handled in the fetch loop.                           |
   
   | 11708   | Add PORTAGE_QUIET to the environment whitelist and filter it |
   | zmedico | from the saved ebuild environment.                           |
   
   | 11709   | Bug #186842 - Suppress file merge display, as previously     |
   | zmedico | done in --quiet mode, whenever --verbose is not enabled.     |
   |         | Also, export PORTAGE_VERBOSE to the ebuild environment and   |
   |         | use it to control tar verbosity when creating binary         |
   |         | packages.                                                    |
   
   | 11710   | Use normal return statements instead of calling sys.exit()   |
   | zmedico | inside dblink.mergeme().                                     |
   
   | 11711   | Remove needlessly complex error handling code from           |
   | zmedico | vardbapi.cpv_counter().                                      |
   
   | 11712   | Remove unused imports found by pyflakes.                     |
   | zmedico |                                                              |
   
   | 11713   | Validate PROPERTIES inside doebuild().                       |
   | zmedico |                                                              |


Modified: main/branches/prefix/bin/isolated-functions.sh
===================================================================
--- main/branches/prefix/bin/isolated-functions.sh	2008-10-28 09:25:56 UTC (rev 11733)
+++ main/branches/prefix/bin/isolated-functions.sh	2008-10-28 18:47:54 UTC (rev 11734)
@@ -535,6 +535,7 @@
 			PORTAGE_COLORMAP PORTAGE_CONFIGROOT PORTAGE_DEBUG \
 			PORTAGE_DEPCACHEDIR PORTAGE_GID PORTAGE_INST_GID \
 			PORTAGE_INST_UID PORTAGE_LOG_FILE PORTAGE_MASTER_PID \
+			PORTAGE_QUIET \
 			PORTAGE_REPO_NAME PORTAGE_RESTRICT PORTAGE_UPDATE_ENV \
 			PORTAGE_WORKDIR_MODE PORTDIR \
 			PORTDIR_OVERLAY ${!PORTAGE_SANDBOX_*} PREROOTPATH \

Modified: main/branches/prefix/bin/misc-functions.sh
===================================================================
--- main/branches/prefix/bin/misc-functions.sh	2008-10-28 09:25:56 UTC (rev 11733)
+++ main/branches/prefix/bin/misc-functions.sh	2008-10-28 18:47:54 UTC (rev 11734)
@@ -739,7 +739,7 @@
 	cd "${T}"
 	install_mask "${PORTAGE_BUILDDIR}/image" "${PKG_INSTALL_MASK}"
 	local tar_options=""
-	[ "${PORTAGE_QUIET}" == "1" ] ||  tar_options="${tar_options} -v"
+	[[ $PORTAGE_VERBOSE = 1 ]] && tar_options+=" -v"
 	# Sandbox is disabled in case the user wants to use a symlink
 	# for $PKGDIR and/or $PKGDIR/All.
 	export SANDBOX_ON="0"

Modified: main/branches/prefix/pym/_emerge/__init__.py
===================================================================
--- main/branches/prefix/pym/_emerge/__init__.py	2008-10-28 09:25:56 UTC (rev 11733)
+++ main/branches/prefix/pym/_emerge/__init__.py	2008-10-28 18:47:54 UTC (rev 11734)
@@ -5708,6 +5708,22 @@
 					if pkg.cp == cp]
 				break
 
+		# If the installed version is in a different slot and it is higher than
+		# the highest available visible package, _iter_atoms_for_pkg() may fail
+		# to properly match the available package with a corresponding argument
+		# atom. Detect this case and correct it here.
+		if not selective and len(matched_packages) > 1 and \
+			matched_packages[-1].installed and \
+			matched_packages[-1].slot_atom != \
+			matched_packages[-2].slot_atom and \
+			matched_packages[-1] > matched_packages[-2]:
+			pkg = matched_packages[-2]
+			if pkg.root == self.target_root and \
+				self._set_atoms.findAtomForPackage(pkg):
+				# Select the available package instead
+				# of the installed package.
+				matched_packages.pop()
+
 		if len(matched_packages) > 1:
 			bestmatch = portage.best(
 				[pkg.cpv for pkg in matched_packages])
@@ -13513,6 +13529,10 @@
 		settings["PORTAGE_QUIET"]="1"
 		settings.backup_changes("PORTAGE_QUIET")
 
+	if "--verbose" in myopts:
+		settings["PORTAGE_VERBOSE"] = "1"
+		settings.backup_changes("PORTAGE_VERBOSE")
+
 	# Set so that configs will be merged regardless of remembered status
 	if ("--noconfmem" in myopts):
 		settings["NOCONFMEM"]="1"

Modified: main/branches/prefix/pym/portage/__init__.py
===================================================================
--- main/branches/prefix/pym/portage/__init__.py	2008-10-28 09:25:56 UTC (rev 11733)
+++ main/branches/prefix/pym/portage/__init__.py	2008-10-28 18:47:54 UTC (rev 11734)
@@ -938,8 +938,10 @@
 		"PORTAGE_GID", "PORTAGE_INST_GID", "PORTAGE_INST_UID",
 		"PORTAGE_IUSE",
 		"PORTAGE_LOG_FILE", "PORTAGE_MASTER_PID",
-		"PORTAGE_PYM_PATH", "PORTAGE_REPO_NAME", "PORTAGE_RESTRICT",
-		"PORTAGE_TMPDIR", "PORTAGE_UPDATE_ENV", "PORTAGE_WORKDIR_MODE",
+		"PORTAGE_PYM_PATH", "PORTAGE_QUIET",
+		"PORTAGE_REPO_NAME", "PORTAGE_RESTRICT",
+		"PORTAGE_TMPDIR", "PORTAGE_UPDATE_ENV",
+		"PORTAGE_VERBOSE", "PORTAGE_WORKDIR_MODE",
 		"PORTDIR", "PORTDIR_OVERLAY", "PREROOTPATH", "PROFILE_PATHS",
 		"ROOT", "ROOTPATH", "STARTDIR", "T", "TMP", "TMPDIR",
 		"USE_EXPAND", "USE_ORDER", "WORKDIR",
@@ -3514,6 +3516,7 @@
 	filedict={}
 	primaryuri_indexes={}
 	primaryuri_dict = {}
+	thirdpartymirror_uris = {}
 	for myfile, myuri in file_uri_tuples:
 		if myfile not in filedict:
 			filedict[myfile]=[]
@@ -3530,16 +3533,15 @@
 					for cmirr in custommirrors[mirrorname]:
 						filedict[myfile].append(
 							cmirr.rstrip("/") + "/" + path)
-						# remove the mirrors we tried from the list of official mirrors
-						if cmirr.strip() in thirdpartymirrors[mirrorname]:
-							thirdpartymirrors[mirrorname].remove(cmirr)
+
 				# now try the official mirrors
 				if mirrorname in thirdpartymirrors:
 					shuffle(thirdpartymirrors[mirrorname])
 
-					for locmirr in thirdpartymirrors[mirrorname]:
-						filedict[myfile].append(
-							locmirr.rstrip("/") + "/" + path)
+					uris = [locmirr.rstrip("/") + "/" + path \
+						for locmirr in thirdpartymirrors[mirrorname]]
+					filedict[myfile].extend(uris)
+					thirdpartymirror_uris.setdefault(myfile, []).extend(uris)
 
 				if not filedict[myfile]:
 					writemsg("No known mirror by the name: %s\n" % (mirrorname))
@@ -3565,6 +3567,11 @@
 				primaryuri_dict[myfile] = primaryuris
 			primaryuris.append(myuri)
 
+	# Prefer thirdpartymirrors over normal mirrors in cases when
+	# the file does not yet exist on the normal mirrors.
+	for myfile, uris in thirdpartymirror_uris.iteritems():
+		primaryuri_dict.setdefault(myfile, []).extend(uris)
+
 	can_fetch=True
 
 	if listonly:
@@ -5425,7 +5432,7 @@
 			set(["clean", "cleanrm", "help", "prerm", "postrm"])
 		mycpv = mysettings["CATEGORY"] + "/" + mysettings["PF"]
 		dep_keys = ["DEPEND", "RDEPEND", "PDEPEND"]
-		misc_keys = ["LICENSE", "PROVIDE", "RESTRICT", "SRC_URI"]
+		misc_keys = ["LICENSE", "PROPERTIES", "PROVIDE", "RESTRICT", "SRC_URI"]
 		other_keys = ["SLOT"]
 		all_keys = dep_keys + misc_keys + other_keys
 		metadata = dict(izip(all_keys, mydbapi.aux_get(mycpv, all_keys)))

Modified: main/branches/prefix/pym/portage/dbapi/vartree.py
===================================================================
--- main/branches/prefix/pym/portage/dbapi/vartree.py	2008-10-28 09:25:56 UTC (rev 11733)
+++ main/branches/prefix/pym/portage/dbapi/vartree.py	2008-10-28 18:47:54 UTC (rev 11734)
@@ -7,20 +7,19 @@
 	["write_contents", "tar_contents"]
 
 from portage.checksum import perform_md5
-from portage.const import CACHE_PATH, CONFIG_MEMORY_FILE, PORTAGE_BIN_PATH, \
+from portage.const import CACHE_PATH, CONFIG_MEMORY_FILE, \
 	PRIVATE_PATH, VDB_PATH, EPREFIX, EPREFIX_LSTRIP
 from portage.data import portage_gid, portage_uid, secpass, ostype
 from portage.dbapi import dbapi
-from portage.dep import dep_getslot, use_reduce, paren_reduce, isvalidatom, \
-	isjustname, dep_getkey, match_from_list
-from portage.exception import InvalidAtom, InvalidData, InvalidPackageName, \
+from portage.dep import use_reduce, paren_reduce, isvalidatom, \
+	isjustname, dep_getkey
+from portage.exception import InvalidData, InvalidPackageName, \
 	FileNotFound, PermissionDenied, UnsupportedAPIException
 from portage.locks import lockdir, unlockdir
 from portage.output import bold, red, green
 from portage.update import fixdbentries
 from portage.util import apply_secpass_permissions, ConfigProtect, ensure_dirs, \
-	writemsg, writemsg_stdout, writemsg_level, \
-	write_atomic, atomic_ofstream, writedict, \
+	writemsg, writemsg_level, write_atomic, atomic_ofstream, writedict, \
 	grabfile, grabdict, normalize_path, new_protect_filename, getlibpaths
 from portage.versions import pkgsplit, catpkgsplit, catsplit, best, pkgcmp
 
@@ -31,7 +30,7 @@
 from portage.elog import elog_process
 from portage.elog.filtering import filter_mergephases, filter_unmergephases
 
-import os, re, sys, stat, errno, commands, copy, time, subprocess
+import os, re, stat, errno, copy, subprocess
 import logging
 import shlex
 from itertools import izip
@@ -1110,54 +1109,11 @@
 			return long(self.aux_get(mycpv, ["COUNTER"])[0])
 		except (KeyError, ValueError):
 			pass
-		cdir = self.getpath(mycpv)
-		cpath = self.getpath(mycpv, filename="COUNTER")
+		writemsg_level(("portage: COUNTER for %s was corrupted; " + \
+			"resetting to value of 0\n") % (mycpv,),
+			level=logging.ERROR, noiselevel=-1)
+		return 0
 
-		# We write our new counter value to a new file that gets moved into
-		# place to avoid filesystem corruption on XFS (unexpected reboot.)
-		corrupted = 0
-		if os.path.exists(cpath):
-			cfile = open(cpath, "r")
-			try:
-				counter = long(cfile.readline())
-			except ValueError:
-				print "portage: COUNTER for", mycpv, "was corrupted; resetting to value of 0"
-				counter = long(0)
-				corrupted = 1
-			cfile.close()
-		elif os.path.exists(cdir):
-			mys = pkgsplit(mycpv)
-			myl = self.match(mys[0], use_cache=0)
-			print mys, myl
-			if len(myl) == 1:
-				try:
-					# Only one package... Counter doesn't matter.
-					write_atomic(cpath, "1")
-					counter = 1
-				except SystemExit, e:
-					raise
-				except Exception, e:
-					writemsg("!!! COUNTER file is missing for "+str(mycpv)+" in /var/db.\n",
-						noiselevel=-1)
-					writemsg("!!! Please run %s/fix-db.py or\n" % PORTAGE_BIN_PATH,
-						noiselevel=-1)
-					writemsg("!!! unmerge this exact version.\n", noiselevel=-1)
-					writemsg("!!! %s\n" % e, noiselevel=-1)
-					sys.exit(1)
-			else:
-				writemsg("!!! COUNTER file is missing for "+str(mycpv)+" in /var/db.\n",
-					noiselevel=-1)
-				writemsg("!!! Please run %s/fix-db.py or\n" % PORTAGE_BIN_PATH,
-					noiselevel=-1)
-				writemsg("!!! remerge the package.\n", noiselevel=-1)
-				sys.exit(1)
-		else:
-			counter = long(0)
-		if corrupted:
-			# update new global counter file
-			write_atomic(cpath, str(counter))
-		return counter
-
 	def cpv_inject(self, mycpv):
 		"injects a real package into our on-disk database; assumes mycpv is valid and doesn't already exist"
 		os.makedirs(self.getpath(mycpv))
@@ -2109,8 +2065,7 @@
 		self._lock_vdb = None
 
 		self.settings = mysettings
-		if self.settings == 1:
-			raise ValueError
+		self._verbose = self.settings.get("PORTAGE_VERBOSE") == "1"
 
 		self.myroot=myroot
 		protect_obj = ConfigProtect(myroot,
@@ -2544,6 +2499,8 @@
 		return os.EX_OK
 
 	def _display_merge(self, msg, level=0, noiselevel=0):
+		if not self._verbose and noiselevel >= 0 and level < logging.WARN:
+			return
 		if self._scheduler is not None:
 			self._scheduler.dblinkDisplayMerge(self, msg,
 				level=level, noiselevel=noiselevel)
@@ -3534,7 +3491,9 @@
 			# couldn't get merged will be added to thirdhand.
 
 			thirdhand = []
-			self.mergeme(srcroot, destroot, outfile, thirdhand, secondhand, cfgfiledict, mymtime)
+			if self.mergeme(srcroot, destroot, outfile, thirdhand,
+				secondhand, cfgfiledict, mymtime):
+				return 1
 
 			#swap hands
 			lastlen = len(secondhand)
@@ -3546,7 +3505,9 @@
 
 		if len(secondhand):
 			# force merge of remaining symlinks (broken or circular; oh well)
-			self.mergeme(srcroot, destroot, outfile, None, secondhand, cfgfiledict, mymtime)
+			if self.mergeme(srcroot, destroot, outfile, None,
+				secondhand, cfgfiledict, mymtime):
+				return 1
 
 		#restore umask
 		os.umask(prevmask)
@@ -3689,6 +3650,7 @@
 		"""
 
 		showMessage = self._display_merge
+		writemsg = self._display_merge
 		scheduler = self._scheduler
 
 		from os.path import sep, join
@@ -3725,7 +3687,7 @@
 				writemsg(red("!!!        and ensure your filesystem is in a sane state. ")+bold("'shutdown -Fr now'\n"))
 				writemsg(red("!!!        File:  ")+str(mysrc)+"\n", noiselevel=-1)
 				writemsg(red("!!!        Error: ")+str(e)+"\n", noiselevel=-1)
-				sys.exit(1)
+				return 1
 			except Exception, e:
 				writemsg("\n")
 				writemsg(red("!!! ERROR: An unknown error has occurred during the merge process.\n"))
@@ -3734,7 +3696,7 @@
 				writemsg(    "!!!        this as a portage bug at bugs.gentoo.org. Append 'emerge info'.\n")
 				writemsg(    "!!!        File:  "+str(mysrc)+"\n", noiselevel=-1)
 				writemsg(    "!!!        Error: "+str(e)+"\n", noiselevel=-1)
-				sys.exit(1)
+				return 1
 
 
 			mymode = mystat[stat.ST_MODE]
@@ -3800,9 +3762,11 @@
 					showMessage(">>> %s -> %s\n" % (mydest, myto))
 					outfile.write("sym "+myrealdest+" -> "+myto+" "+str(mymtime)+"\n")
 				else:
-					print "!!! Failed to move file."
-					print "!!!", mydest, "->", myto
-					sys.exit(1)
+					showMessage("!!! Failed to move file.\n",
+						level=logging.ERROR, noiselevel=-1)
+					showMessage("!!! %s -> %s\n" % (mydest, myto),
+						level=logging.ERROR, noiselevel=-1)
+					return 1
 			elif stat.S_ISDIR(mymode):
 				# we are merging a directory
 				if mydmode != None:
@@ -3831,8 +3795,9 @@
 					else:
 						# a non-directory and non-symlink-to-directory.  Won't work for us.  Move out of the way.
 						if movefile(mydest, mydest+".backup", mysettings=self.settings) is None:
-							sys.exit(1)
-						print "bak", mydest, mydest+".backup"
+							return 1
+						showMessage("bak %s %s.backup\n" % (mydest, mydest),
+							level=logging.ERROR, noiselevel=-1)
 						#now create our directory
 						if self.settings.selinux_enabled():
 							import selinux
@@ -3916,7 +3881,7 @@
 				if moveme:
 					mymtime = movefile(mysrc, mydest, newmtime=thismtime, sstat=mystat, mysettings=self.settings)
 					if mymtime is None:
-						sys.exit(1)
+						return 1
 					zing = ">>>"
 
 				if mymtime != None:
@@ -3930,7 +3895,7 @@
 					if movefile(mysrc, mydest, newmtime=thismtime, sstat=mystat, mysettings=self.settings) != None:
 						zing = ">>>"
 					else:
-						sys.exit(1)
+						return 1
 				if stat.S_ISFIFO(mymode):
 					outfile.write("fif %s\n" % myrealdest)
 				else: