[gentoo-commits] portage r12037 - in main/branches/prefix: bin cnf man pym/_emerge pym/portage pym/portage/cache

[Fabian Groffen (grobian)]

Author: grobian
Date: 2008-11-22 11:31:14 +0000 (Sat, 22 Nov 2008)
New Revision: 12037

Modified:
   main/branches/prefix/bin/emake
   main/branches/prefix/cnf/make.globals
   main/branches/prefix/man/ebuild.1
   main/branches/prefix/man/emerge.1
   main/branches/prefix/man/make.conf.5
   main/branches/prefix/pym/_emerge/__init__.py
   main/branches/prefix/pym/portage/__init__.py
   main/branches/prefix/pym/portage/cache/flat_hash.py
   main/branches/prefix/pym/portage/cache/fs_template.py
   main/branches/prefix/pym/portage/cache/metadata.py
   main/branches/prefix/pym/portage/cache/template.py
   main/branches/prefix/pym/portage/cache/util.py
   main/branches/prefix/pym/portage/eclass_cache.py
Log:
   Merged from trunk -r11982:12004

   | 11984    | Bug #139134 - Make the flat_hash and metadata modules write |
   | zmedico  | the _mtime_ field inside the file instead of mangling the   |
   |          | mtime of the cache entry file. Also, fix                    |
   |          | FsBased._ensure_access() to properly skip the utime() call  |
   |          | when no mtime is passed in. Theses cache changes are        |
   |          | compatible with current stable portage (2.1.4.x), which     |
   |          | uses the _mtime_ field contained in the file when           |
   |          | available.                                                  |
   
   | 11986    | Store the _eclasses_ field instead of INHERITED.            |
   | zmedico  |                                                             |
   
   | 11988    | include quotes when showing make output                     |
   | SpankMan |                                                             |
   
   | 11990    | For emerge --metadata runs, update _eclasses_ metadata to   |
   | zmedico  | insert local eclass paths.                                  |
   
   | 11992    | Make serialize_eclasses() sort eclass names.                |
   | zmedico  |                                                             |
   
   | 11994    | When warning about 'missing repo_name', also give the exact |
   | zmedico  | path where the entry should exist, and explain that it      |
   |          | should be a plain text file containing a unique name of the |
   |          | first line. This should give the users enough information   |
   |          | to correct the problem without needing to seek help.        |
   
   | 11995    | Move the repo_name check/warning into a function.           |
   | zmedico  |                                                             |
   
   | 11997    | Bug #245779 - Document the new category subdirectory        |
   | zmedico  | behavior for PKGDIR.                                        |
   
   | 11999    | Bug #245716 - When generating QA notices for 'command not   |
   | zmedico  | found' messages in the build log, exclude messages that are |
   |          | emitted from configure scripts since those are normal in    |
   |          | some cases.                                                 |
   
   | 12001    | Remove --fetchonly --resume time.sleep() call which is an   |
   | zmedico  | artifact from the previous parallel-fetch implementation.   |
   
   | 12003    | Bug #247625 - Update PORTAGE_IONICE_COMMAND docs to reflect |
   | zmedico  | that it is unset by default.                                |
   
   | 12004    | Remove PORTAGE_IONICE_COMMAND setting since it's commented  |
   | zmedico  | out.                                                        |


Modified: main/branches/prefix/bin/emake
===================================================================
--- main/branches/prefix/bin/emake	2008-11-22 11:29:10 UTC (rev 12036)
+++ main/branches/prefix/bin/emake	2008-11-22 11:31:14 UTC (rev 12037)
@@ -11,6 +11,15 @@
 #        /etc/make.globals here because emake is only called from an
 #        ebuild.
 
-[[ $PORTAGE_QUIET = 1 ]] || echo ${MAKE:-make} $MAKEOPTS $EXTRA_EMAKE "$@" >&2
+if [[ $PORTAGE_QUIET != 1 ]] ; then
+	(
+	for arg in ${MAKE:-make} $MAKEOPTS $EXTRA_EMAKE "$@" ; do
+		[[ ${arg} == *" "* ]] \
+			&& printf "'%s' " "${arg}" \
+			|| printf "%s " "${arg}"
+	done
+	printf "\n"
+	) >&2
+fi
 
 exec ${MAKE:-make} ${MAKEOPTS} ${EXTRA_EMAKE} "$@"

Modified: main/branches/prefix/cnf/make.globals
===================================================================
--- main/branches/prefix/cnf/make.globals	2008-11-22 11:29:10 UTC (rev 12036)
+++ main/branches/prefix/cnf/make.globals	2008-11-22 11:31:14 UTC (rev 12037)
@@ -71,10 +71,6 @@
 # Minimum size of existing file for RESUMECOMMAND to be called.
 PORTAGE_FETCH_RESUME_MIN_SIZE="350K"
 
-# Command called to adjust the io priority of portage and it's subprocesses.
-# Note: should be wrapped inside a uid check
-#PORTAGE_IONICE_COMMAND="ionice -c 3 -p \${PID}"
-
 # Number of times 'emerge --sync' will run before giving up.
 PORTAGE_RSYNC_RETRIES="3"
 

Modified: main/branches/prefix/man/ebuild.1
===================================================================
--- main/branches/prefix/man/ebuild.1	2008-11-22 11:29:10 UTC (rev 12036)
+++ main/branches/prefix/man/ebuild.1	2008-11-22 11:31:14 UTC (rev 12037)
@@ -161,9 +161,7 @@
 .BR package
 This command is a lot like the \fImerge\fR command, except that after
 fetching, unpacking, compiling and installing, a .tbz2 binary package
-tarball is created and stored in  ${PKGDIR}/All (${PKGDIR} defaults to
-/usr/portage/packages).  A symbolic link is created in
-${PKGDIR}/${CATEGORY} that points to the package in ${PKGDIR}/All.
+tarball is created and stored in \fBPKGDIR\fR (see \fBmake.conf\fR(5)).
 .TP
 .BR rpm
 Builds a RedHat RPM package from the files in the temporary

Modified: main/branches/prefix/man/emerge.1
===================================================================
--- main/branches/prefix/man/emerge.1	2008-11-22 11:29:10 UTC (rev 12036)
+++ main/branches/prefix/man/emerge.1	2008-11-22 11:31:14 UTC (rev 12037)
@@ -246,8 +246,8 @@
 addition to actually merging the packages.  Useful for maintainers
 or if you administrate multiple Gentoo Linux systems (build once,
 emerge tbz2s everywhere) as well as disaster recovery. The package
-will be created in the
-\fI${PKGDIR}/All\fR directory.  An alternative for already\-merged
+will be created in the \fBPKGDIR\fR directory (see \fBmake.conf\fR(5)).
+An alternative for already\-merged
 packages is to use \fBquickpkg\fR(1) which creates a tbz2 from the
 live filesystem.
 .TP

Modified: main/branches/prefix/man/make.conf.5
===================================================================
--- main/branches/prefix/man/make.conf.5	2008-11-22 11:29:10 UTC (rev 12036)
+++ main/branches/prefix/man/make.conf.5	2008-11-22 11:31:14 UTC (rev 12037)
@@ -401,7 +401,13 @@
 Defaults to false.
 .TP
 \fBPKGDIR\fR = \fI[path]\fR
-Defines the location where created .tbz2 binary packages will be stored.  Note
+Defines the location where created .tbz2 binary packages will be
+stored when the \fBemerge\fR(1) \fB\-\-buildpkg\fR option is enabled.
+By default, a given package is stored in a subdirectory corresponding
+to it's category. However, for backward compatibility with the layout
+used by older versions of portage, if the \fI${PKGDIR}/All\fR directory
+exists then all packages will be stored inside of it and symlinks to
+the packages will be created in the category subdirectories. Note
 that locations under /usr/portage are not necessarily safe for data storage.
 See the \fBPORTDIR\fR documentation for more information.
 .br
@@ -482,11 +488,9 @@
 This variable should contain a command for portage to call in order
 to adjust the io priority of portage and it's subprocesses. The command
 string should contain a \\${PID} place-holder that will be substituted
-with an integer pid. For more information about ionice, see \fBionice\fR(1).
-Set this variable to an empty string if you do not want portage to call
-ionice.
-.br
-Defaults to "ionice -c 3 -p \\${PID}".
+with an integer pid. For example, a value of "ionice \-c 3 \-p \\${PID}"
+will set idle io priority. For more information about ionice, see
+\fBionice\fR(1). This variable is unset by default.
 .TP
 \fBPORTAGE_NICENESS\fR = \fI[number]\fR
 The value of this variable will be added to the current nice level that

Modified: main/branches/prefix/pym/_emerge/__init__.py
===================================================================
--- main/branches/prefix/pym/_emerge/__init__.py	2008-11-22 11:29:10 UTC (rev 12036)
+++ main/branches/prefix/pym/_emerge/__init__.py	2008-11-22 11:31:14 UTC (rev 12037)
@@ -13338,11 +13338,6 @@
 
 		if ("--resume" in myopts):
 			favorites=mtimedb["resume"]["favorites"]
-			if "PORTAGE_PARALLEL_FETCHONLY" in settings:
-				""" parallel-fetch uses --resume --fetchonly and we don't want
-				it to write the mtimedb"""
-				mtimedb.filename = None
-				time.sleep(3) # allow the parent to have first fetch
 			mymergelist = mydepgraph.altlist()
 			mydepgraph.break_refs(mymergelist)
 			mergetask = Scheduler(settings, trees, mtimedb, myopts,
@@ -13934,6 +13929,32 @@
 			newargs.append(a)
 	return (newargs, retval)
 
+def repo_name_check(trees):
+	missing_repo_names = set()
+	for root, root_trees in trees.iteritems():
+		if "porttree" in root_trees:
+			portdb = root_trees["porttree"].dbapi
+			missing_repo_names.update(portdb.porttrees)
+			repos = portdb.getRepositories()
+			for r in repos:
+				missing_repo_names.discard(portdb.getRepositoryPath(r))
+
+	if missing_repo_names:
+		msg = []
+		msg.append("WARNING: One or more repositories " + \
+			"have missing repo_name entries:")
+		msg.append("")
+		for p in missing_repo_names:
+			msg.append("\t%s/profiles/repo_name" % (p,))
+		msg.append("")
+		msg.extend(textwrap.wrap("NOTE: Each repo_name entry " + \
+			"should be a plain text file containing a unique " + \
+			"name for the repository on the first line.", 70))
+		writemsg_level("".join("%s\n" % l for l in msg),
+			level=logging.WARNING, noiselevel=-1)
+
+	return bool(missing_repo_names)
+
 def emerge_main():
 	global portage	# NFC why this is necessary now - genone
 	portage._disable_legacy_globals()
@@ -13994,21 +14015,7 @@
 
 	if "--quiet" not in myopts:
 		portage.deprecated_profile_check()
-		for root in trees:
-			if "porttree" in trees[root]:
-				db = trees[root]["porttree"].dbapi
-				paths = (db.mysettings["PORTDIR"]+" "+db.mysettings["PORTDIR_OVERLAY"]).split()
-				paths = [os.path.realpath(p) for p in paths]
-				repos = db.getRepositories()
-				for r in repos:
-					p = db.getRepositoryPath(r)
-					try:
-						paths.remove(p)
-					except ValueError:
-						pass
-				for p in paths:
-					writemsg("WARNING: repository at %s is missing a repo_name entry\n" % p)
-					
+		repo_name_check(trees)
 
 	eclasses_overridden = {}
 	for mytrees in trees.itervalues():

Modified: main/branches/prefix/pym/portage/__init__.py
===================================================================
--- main/branches/prefix/pym/portage/__init__.py	2008-11-22 11:29:10 UTC (rev 12036)
+++ main/branches/prefix/pym/portage/__init__.py	2008-11-22 11:31:14 UTC (rev 12037)
@@ -4529,6 +4529,7 @@
 	bash_command_not_found = []
 	bash_command_not_found_re = re.compile(
 		r'(.*): line (\d*): (.*): command not found$')
+	command_not_found_exclude_re = re.compile(r'/configure: line ')
 	helper_missing_file = []
 	helper_missing_file_re = re.compile(
 		r'^!!! (do|new).*: .* does not exist$')
@@ -4550,7 +4551,8 @@
 				am_maintainer_mode_exclude_re.search(line) is None:
 				am_maintainer_mode.append(line.rstrip("\n"))
 
-			if bash_command_not_found_re.match(line) is not None:
+			if bash_command_not_found_re.match(line) is not None and \
+				command_not_found_exclude_re.search(line) is None:
 				bash_command_not_found.append(line.rstrip("\n"))
 
 			if helper_missing_file_re.match(line) is not None:

Modified: main/branches/prefix/pym/portage/cache/flat_hash.py
===================================================================
--- main/branches/prefix/pym/portage/cache/flat_hash.py	2008-11-22 11:29:10 UTC (rev 12036)
+++ main/branches/prefix/pym/portage/cache/flat_hash.py	2008-11-22 11:31:14 UTC (rev 12037)
@@ -18,6 +18,7 @@
 			self.label.lstrip(os.path.sep).rstrip(os.path.sep))
 		write_keys = set(self._known_keys)
 		write_keys.add("_eclasses_")
+		write_keys.add("_mtime_")
 		self._write_keys = sorted(write_keys)
 		if not self.readonly and not os.path.exists(self.location):
 			self._ensure_dirs()
@@ -83,7 +84,7 @@
 				myf.write("%s=%s\n" % (k, v))
 		finally:
 			myf.close()
-		self._ensure_access(fp, mtime=values["_mtime_"])
+		self._ensure_access(fp)
 
 		#update written.  now we move it.
 

Modified: main/branches/prefix/pym/portage/cache/fs_template.py
===================================================================
--- main/branches/prefix/pym/portage/cache/fs_template.py	2008-11-22 11:29:10 UTC (rev 12036)
+++ main/branches/prefix/pym/portage/cache/fs_template.py	2008-11-22 11:31:14 UTC (rev 12037)
@@ -36,7 +36,7 @@
 		try:
 			os.chown(path, -1, self._gid)
 			os.chmod(path, self._perms)
-			if mtime:
+			if mtime != -1:
 				mtime=long(mtime)
 				os.utime(path, (mtime, mtime))
 		except (OSError, IOError):

Modified: main/branches/prefix/pym/portage/cache/metadata.py
===================================================================
--- main/branches/prefix/pym/portage/cache/metadata.py	2008-11-22 11:29:10 UTC (rev 12036)
+++ main/branches/prefix/pym/portage/cache/metadata.py	2008-11-22 11:31:14 UTC (rev 12037)
@@ -78,12 +78,4 @@
 
 		
 	def _setitem(self, cpv, values):
-		values = ProtectedDict(values)
-		
-		# hack.  proper solution is to make this a __setitem__ override, since template.__setitem__ 
-		# serializes _eclasses_, then we reconstruct it.
-		if "_eclasses_" in values:
-			values["INHERITED"] = ' '.join(reconstruct_eclasses(cpv, values["_eclasses_"]).keys())
-			del values["_eclasses_"]
-
 		flat_hash.database._setitem(self, cpv, values)

Modified: main/branches/prefix/pym/portage/cache/template.py
===================================================================
--- main/branches/prefix/pym/portage/cache/template.py	2008-11-22 11:29:10 UTC (rev 12036)
+++ main/branches/prefix/pym/portage/cache/template.py	2008-11-22 11:31:14 UTC (rev 12037)
@@ -189,8 +189,8 @@
 	"""
 	if not eclass_dict:
 		return ""
-	return "\t".join(["%s\t%s\t%s" % (k, v[0], str(v[1])) \
-		for k, v in eclass_dict.iteritems()])
+	return "\t".join(k + "\t%s\t%s" % eclass_dict[k] \
+		for k in sorted(eclass_dict))
 
 def reconstruct_eclasses(cpv, eclass_string):
 	"""returns a dict when handed a string generated by serialize_eclasses"""

Modified: main/branches/prefix/pym/portage/cache/util.py
===================================================================
--- main/branches/prefix/pym/portage/cache/util.py	2008-11-22 11:29:10 UTC (rev 12036)
+++ main/branches/prefix/pym/portage/cache/util.py	2008-11-22 11:31:14 UTC (rev 12037)
@@ -3,6 +3,8 @@
 # License: GPL2
 # $Id$
 
+__all__ = ["mirror_cache", "non_quiet_mirroring", "quiet_mirroring"]
+
 from itertools import chain
 from portage.cache import cache_errors
 
@@ -65,26 +67,35 @@
 
 		if write_it:
 			try:
-				inherited = entry.get("INHERITED", None)
+				inherited = entry.get("INHERITED", "")
+				eclasses = entry.get("_eclasses_")
 			except cache_errors.CacheError, ce:
 				noise.exception(x, ce)
 				del ce
 				continue
+
+			if eclasses is not None:
+				if not eclass_cache.is_eclass_data_valid(entry["_eclasses_"]):
+					noise.eclass_stale(x)
+					continue
+				inherited = eclasses
+			else:
+				inherited = inherited.split()
+
 			if inherited:
-				if src_cache.complete_eclass_entries:
-					if not "_eclasses_" in entry:
-						noise.corruption(x,"missing _eclasses_ field")
-						continue
-					if not eclass_cache.is_eclass_data_valid(entry["_eclasses_"]):
-						noise.eclass_stale(x)
-						continue
-				else:
-					entry["_eclasses_"] = eclass_cache.get_eclass_data(entry["INHERITED"].split(), \
-						from_master_only=True)
-					if not entry["_eclasses_"]:
-						noise.eclass_stale(x)
-						continue
+				if src_cache.complete_eclass_entries and eclasses is None:
+					noise.corruption(x, "missing _eclasses_ field")
+					continue
 
+				# Even if _eclasses_ already exists, replace it with data from
+				# eclass_cache, in order to insert local eclass paths.
+				eclasses = eclass_cache.get_eclass_data(inherited,
+					from_master_only=True)
+				if eclasses is None:
+					noise.eclass_stale(x)
+					continue
+				entry["_eclasses_"] = eclasses
+
 			# by this time, if it reaches here, the eclass has been validated, and the entry has 
 			# been updated/translated (if needs be, for metadata/cache mainly)
 			try:	trg_cache[x] = entry

Modified: main/branches/prefix/pym/portage/eclass_cache.py
===================================================================
--- main/branches/prefix/pym/portage/eclass_cache.py	2008-11-22 11:29:10 UTC (rev 12036)
+++ main/branches/prefix/pym/portage/eclass_cache.py	2008-11-22 11:31:14 UTC (rev 12037)
@@ -3,6 +3,8 @@
 # License: GPL2
 # $Id$
 
+__all__ = ["cache"]
+
 from portage.util import normalize_path, writemsg
 import errno, os, sys
 from portage.data import portage_gid
@@ -63,14 +65,22 @@
 				except OSError:
 					continue
 				ys=y[:-eclass_len]
-				self.eclasses[ys] = (x, long(mtime))
-				self._eclass_locations[ys] = x
 				if x == self._master_eclass_root:
 					master_eclasses[ys] = mtime
-				else:
-					master_mtime = master_eclasses.get(ys)
-					if master_mtime and master_mtime != mtime:
+					self.eclasses[ys] = (x, mtime)
+					self._eclass_locations[ys] = x
+					continue
+
+				master_mtime = master_eclasses.get(ys)
+				if master_mtime is not None:
+					if master_mtime == mtime:
+						# It appears to be identical to the master,
+						# so prefer the master entry.
+						continue
+					else:
 						self._master_eclasses_overridden[ys] = x
+				self.eclasses[ys] = (x, mtime)
+				self._eclass_locations[ys] = x
 
 	def is_eclass_data_valid(self, ec_dict):
 		if not isinstance(ec_dict, dict):