[gentoo-commits] portage r14743 - in main/branches/prefix: bin man pym/_emerge pym/portage pym/portage/dbapi pym/portage/elog

[Fabian Groffen (grobian)]

Author: grobian
Date: 2009-10-28 18:36:02 +0000 (Wed, 28 Oct 2009)
New Revision: 14743

Modified:
   main/branches/prefix/bin/ebuild.sh
   main/branches/prefix/bin/repoman
   main/branches/prefix/man/emerge.1
   main/branches/prefix/pym/_emerge/JobStatusDisplay.py
   main/branches/prefix/pym/_emerge/help.py
   main/branches/prefix/pym/_emerge/main.py
   main/branches/prefix/pym/portage/__init__.py
   main/branches/prefix/pym/portage/dbapi/vartree.py
   main/branches/prefix/pym/portage/elog/messages.py
   main/branches/prefix/pym/portage/output.py
   main/branches/prefix/pym/portage/util.py
Log:
   Merged from trunk -r14724:14735

   | 14725   | Don't set mtime on downloaded metadata.dtd when using        |
   | zmedico | python3, since the rfc822.parsedate() function is not        |
   |         | available. Thanks to Arfrever for reporting.                 |
   
   | 14728   | Use calendar.timegm instead of time.mktime, for correct      |
   | zmedico | timezone handling.                                           |
   
   | 14729   | Add a parsedate() function which emulates                    |
   | zmedico | rfc822.parsedate(), since python3 doesn't have it.           |
   
   | 14730   | Add -R as a shortcut for --depclean. Thanks to Jonathan      |
   | zmedico | Callen <abcd@g.o> for the suggestion.                        |
   
   | 14731   | Take the -c option and make it mean --depclean instead of    |
   | zmedico | --clean, since --clean is pretty useless anyway.             |
   
   | 14732   | Add a reference to bug #141118 inside                        |
   | zmedico | _expand_new_virtuals().                                      |
   
   | 14733   | Revert the workaround for bug #288863. This will require a   |
   | zmedico | dependency on >=sys-apps/sandbox-2.2.                        |
   
   | 14734   | Bug #290625 - Manually encode output to stdout in python3,   |
   | zmedico | in order to avoid potential UnicodeEncodeError exceptions.   |
   
   | 14735   | Bug #290428 - Update mtime of /var/db/pkg and category       |
   | zmedico | subdirectories when stuff inside is modified, so that        |
   |         | consumers can use directory mtimes to validate caches.       |


Modified: main/branches/prefix/bin/ebuild.sh
===================================================================
--- main/branches/prefix/bin/ebuild.sh	2009-10-27 22:56:07 UTC (rev 14742)
+++ main/branches/prefix/bin/ebuild.sh	2009-10-28 18:36:02 UTC (rev 14743)
@@ -6,12 +6,6 @@
 PORTAGE_BIN_PATH="${PORTAGE_BIN_PATH:-@PORTAGE_BASE@/bin}"
 PORTAGE_PYM_PATH="${PORTAGE_PYM_PATH:-@PORTAGE_BASE@/pym}"
 
-# Ensure that /dev/std* streams have appropriate sandbox permission for
-# bug #288863. This can be removed after sandbox is fixed and portage
-# depends on the fixed version.
-export SANDBOX_WRITE="${SANDBOX_WRITE:+${SANDBOX_WRITE}:}/dev/stdout:/dev/stderr"
-export SANDBOX_READ="${SANDBOX_READ:+${SANDBOX_READ}:}/dev/stdin"
-
 # Don't use sandbox's BASH_ENV for new shells because it does
 # 'source /etc/profile' which can interfere with the build
 # environment by modifying our PATH.

Modified: main/branches/prefix/bin/repoman
===================================================================
--- main/branches/prefix/bin/repoman	2009-10-27 22:56:07 UTC (rev 14742)
+++ main/branches/prefix/bin/repoman	2009-10-28 18:36:02 UTC (rev 14743)
@@ -9,6 +9,7 @@
 
 from __future__ import print_function
 
+import calendar
 import codecs
 try:
 	from subprocess import getstatusoutput as subprocess_getstatusoutput
@@ -25,8 +26,12 @@
 import tempfile
 import time
 import platform
-import urllib
 
+try:
+	from urllib.request import urlopen as urllib_request_urlopen
+except ImportError:
+	from urllib import urlopen as urllib_request_urlopen
+
 from io import StringIO
 from itertools import chain
 from stat import S_ISDIR, ST_CTIME
@@ -774,6 +779,33 @@
 xmllint_capable = False
 metadata_dtd = os.path.join(repoman_settings["DISTDIR"], 'metadata.dtd')
 
+def parsedate(s):
+	"""Parse a RFC 822 date and time string.
+	This is required for python3 compatibility, since the
+	rfc822.parsedate() function is not available."""
+
+	s_split = []
+	for x in s.upper().split():
+		for y in x.split(','):
+			if y:
+				s_split.append(y)
+
+	if len(s_split) != 6:
+		return None
+
+	# %a, %d %b %Y %H:%M:%S %Z
+	a, d, b, Y, H_M_S, Z = s_split
+
+	# Convert month to integer, since strptime %w is locale-dependent.
+	month_map = {'JAN':1, 'FEB':2, 'MAR':3, 'APR':4, 'MAY':5, 'JUN':6,
+		'JUL':7, 'AUG':8, 'SEP':9, 'OCT':10, 'NOV':11, 'DEC':12}
+	m = month_map.get(b)
+	if m is None:
+		return None
+	m = str(m).rjust(2, '0')
+
+	return time.strptime(':'.join((Y, m, d, H_M_S)), '%Y:%m:%d:%H:%M:%S')
+
 def fetch_metadata_dtd():
 	"""
 	Fetch metadata.dtd if it doesn't exist or the ctime is older than
@@ -803,10 +835,13 @@
 			"needs to be refetched, doing that now")
 		print()
 		try:
-			url_f = urllib.urlopen(metadata_dtd_uri)
-			last_modified = url_f.info().getdate('last-modified')
+			url_f = urllib_request_urlopen(metadata_dtd_uri)
+			msg_info = url_f.info()
+			last_modified = msg_info.get('last-modified')
 			if last_modified is not None:
-				last_modified = time.mktime(last_modified)
+				last_modified = parsedate(last_modified)
+				if last_modified is not None:
+					last_modified = calendar.timegm(last_modified)
 
 			metadata_dtd_tmp = "%s.%s" % (metadata_dtd, os.getpid())
 			try:

Modified: main/branches/prefix/man/emerge.1
===================================================================
--- main/branches/prefix/man/emerge.1	2009-10-27 22:56:07 UTC (rev 14742)
+++ main/branches/prefix/man/emerge.1	2009-10-28 18:36:02 UTC (rev 14743)
@@ -93,7 +93,7 @@
 to the \fBworld\fR file at the end, so that they are considered for
 later updating.
 .TP
-.BR "\-\-clean " (\fB\-c\fR)
+.BR \-\-clean
 Cleans up the system by examining the installed packages and removing older
 packages.  This is accomplished by looking at each installed package and separating
 the installed versions by \fBslot\fR.  Clean will \fBremove all but the most recently
@@ -105,7 +105,7 @@
 has completed.  This usually entails configuration file setup or other similar 
 setups that the user may wish to run.
 .TP
-.BR \-\-depclean
+.BR "\-\-depclean (-c)"
 Cleans the system by removing packages that are not associated
 with explicitly merged packages. Depclean works by creating the
 full dependency tree from the @world set,

Modified: main/branches/prefix/pym/_emerge/JobStatusDisplay.py
===================================================================
--- main/branches/prefix/pym/_emerge/JobStatusDisplay.py	2009-10-27 22:56:07 UTC (rev 14742)
+++ main/branches/prefix/pym/_emerge/JobStatusDisplay.py	2009-10-28 18:36:02 UTC (rev 14743)
@@ -17,6 +17,7 @@
 from portage import os
 from portage import _encodings
 from portage import _unicode_decode
+from portage import _unicode_encode
 from portage.output import xtermTitle
 
 from _emerge.getloadavg import getloadavg
@@ -75,11 +76,14 @@
 		return sys.stdout
 
 	def _write(self, s):
-		if sys.hexversion < 0x3000000 and isinstance(s, unicode):
-			# avoid potential UnicodeEncodeError
-			s = s.encode(_encodings['stdio'], 'backslashreplace')
-		self.out.write(s)
-		self.out.flush()
+		# avoid potential UnicodeEncodeError
+		s = _unicode_encode(s,
+			encoding=_encodings['stdio'], errors='backslashreplace')
+		out = self.out
+		if sys.hexversion >= 0x3000000:
+			out = out.buffer
+		out.write(s)
+		out.flush()
 
 	def _init_term(self):
 		"""

Modified: main/branches/prefix/pym/_emerge/help.py
===================================================================
--- main/branches/prefix/pym/_emerge/help.py	2009-10-27 22:56:07 UTC (rev 14742)
+++ main/branches/prefix/pym/_emerge/help.py	2009-10-28 18:36:02 UTC (rev 14743)
@@ -42,7 +42,7 @@
 		print("              emerge to display detailed help.")
 		print()
 		print(turquoise("Actions:"))
-		print("       "+green("--clean")+" ("+green("-c")+" short option)")
+		print("       "+green("--clean"))
 		print("              Cleans the system by removing outdated packages which will not")
 		print("              remove functionalities or prevent your system from working.")
 		print("              The arguments can be in several different formats :")
@@ -66,7 +66,7 @@
 		print("              emerge process has completed.  This usually entails configuration")
 		print("              file setup or other similar setups that the user may wish to run.")
 		print()
-		print("       "+green("--depclean"))
+		print("       "+green("--depclean")+" ("+green("-c")+" short option)")
 
 		paragraph = "Cleans the system by removing packages that are " + \
 		"not associated with explicitly merged packages. Depclean works " + \

Modified: main/branches/prefix/pym/_emerge/main.py
===================================================================
--- main/branches/prefix/pym/_emerge/main.py	2009-10-27 22:56:07 UTC (rev 14742)
+++ main/branches/prefix/pym/_emerge/main.py	2009-10-28 18:36:02 UTC (rev 14743)
@@ -77,7 +77,8 @@
 "1":"--oneshot",
 "a":"--ask",
 "b":"--buildpkg",  "B":"--buildpkgonly",
-"c":"--clean",     "C":"--unmerge",
+"c":"--depclean",
+"C":"--unmerge",
 "d":"--debug",
 "e":"--emptytree",
 "f":"--fetchonly", "F":"--fetch-all-uri",

Modified: main/branches/prefix/pym/portage/__init__.py
===================================================================
--- main/branches/prefix/pym/portage/__init__.py	2009-10-27 22:56:07 UTC (rev 14742)
+++ main/branches/prefix/pym/portage/__init__.py	2009-10-28 18:36:02 UTC (rev 14743)
@@ -7671,8 +7671,10 @@
 
 def _expand_new_virtuals(mysplit, edebug, mydbapi, mysettings, myroot="/",
 	trees=None, use_mask=None, use_force=None, **kwargs):
-	"""Recursively expand new-style virtuals so as to collapse one or more
-	levels of indirection.  In dep_zapdeps, new-style virtuals will be assigned
+	"""
+	In order to solve bug #141118, recursively expand new-style virtuals so
+	as to collapse one or more levels of indirection, generating an expanded
+	search space. In dep_zapdeps, new-style virtuals will be assigned
 	zero cost regardless of whether or not they are currently installed. Virtual
 	blockers are supported but only when the virtual expands to a single
 	atom because it wouldn't necessarily make sense to block all the components

Modified: main/branches/prefix/pym/portage/dbapi/vartree.py
===================================================================
--- main/branches/prefix/pym/portage/dbapi/vartree.py	2009-10-27 22:56:07 UTC (rev 14742)
+++ main/branches/prefix/pym/portage/dbapi/vartree.py	2009-10-28 18:36:02 UTC (rev 14743)
@@ -53,6 +53,7 @@
 import logging
 import os as _os
 import sys
+import time
 import warnings
 
 try:
@@ -1565,6 +1566,19 @@
 			rValue = _os.path.join(rValue, filename)
 		return rValue
 
+	def _bump_mtime(self, cpv):
+		"""
+		This is called before an after any modifications, so that consumers
+		can use directory mtimes to validate caches. See bug #290428.
+		"""
+		base = self.root + _os.sep + VDB_PATH
+		cat = catsplit(cpv)[0]
+		catdir = base + _os.sep + cat
+		t = time.time()
+		t = (t, t)
+		for x in (catdir, base):
+			os.utime(x, t)
+
 	def cpv_exists(self, mykey):
 		"Tells us whether an actual ebuild exists on disk (no masking)"
 		return os.path.exists(self.getpath(mykey))
@@ -2017,6 +2031,7 @@
 		return results
 
 	def aux_update(self, cpv, values):
+		self._bump_mtime(cpv)
 		cat, pkg = catsplit(cpv)
 		mylink = dblink(cat, pkg, self.root, self.settings,
 		treetype="vartree", vartree=self.vartree)
@@ -2030,6 +2045,7 @@
 					os.unlink(os.path.join(self.getpath(cpv), k))
 				except EnvironmentError:
 					pass
+		self._bump_mtime(cpv)
 
 	def counter_tick(self, myroot, mycpv=None):
 		return self.counter_tick_core(myroot, incrementing=1, mycpv=mycpv)
@@ -2151,9 +2167,11 @@
 				removed += 1
 
 		if removed:
+			self._bump_mtime(pkg.mycpv)
 			f = atomic_ofstream(os.path.join(pkg.dbdir, "CONTENTS"))
 			write_contents(new_contents, root, f)
 			f.close()
+			self._bump_mtime(pkg.mycpv)
 			pkg._clear_contents_cache()
 
 	class _owners_cache(object):
@@ -2783,6 +2801,7 @@
 		The caller must ensure that lockdb() and unlockdb() are called
 		before and after this method.
 		"""
+		self.vartree.dbapi._bump_mtime(self.mycpv)
 		showMessage = self._display_merge
 		if self.vartree.dbapi._categories is not None:
 			self.vartree.dbapi._categories = None
@@ -2924,6 +2943,7 @@
 				self.vartree.dbapi.plib_registry.pruneNonExisting()
 
 		finally:
+			self.vartree.dbapi._bump_mtime(self.mycpv)
 			if builddir_lock:
 				try:
 					if myebuildpath:
@@ -5066,6 +5086,7 @@
 		mydbapi=None, prev_mtimes=None):
 		retval = -1
 		self.lockdb()
+		self.vartree.dbapi._bump_mtime(self.mycpv)
 		try:
 			self.vartree.dbapi.plib_registry.load()
 			self.vartree.dbapi.plib_registry.pruneNonExisting()
@@ -5091,6 +5112,7 @@
 		finally:
 			self.vartree.dbapi.linkmap._clear_cache()
 			self.unlockdb()
+			self.vartree.dbapi._bump_mtime(self.mycpv)
 		return retval
 
 	def getstring(self,name):

Modified: main/branches/prefix/pym/portage/elog/messages.py
===================================================================
--- main/branches/prefix/pym/portage/elog/messages.py	2009-10-27 22:56:07 UTC (rev 14742)
+++ main/branches/prefix/pym/portage/elog/messages.py	2009-10-28 18:36:02 UTC (rev 14743)
@@ -96,11 +96,12 @@
 
 	formatted_msg = colorize(color, " * ") + msg + "\n"
 
-	if sys.hexversion < 0x3000000 and \
-		out in (sys.stdout, sys.stderr) and isinstance(formatted_msg, unicode):
-		# avoid potential UnicodeEncodeError
-		formatted_msg = formatted_msg.encode(
-			_encodings['stdio'], 'backslashreplace')
+	# avoid potential UnicodeEncodeError
+	if out in (sys.stdout, sys.stderr):
+		formatted_msg = _unicode_encode(formatted_msg,
+			encoding=_encodings['stdio'], errors='backslashreplace')
+		if sys.hexversion >= 0x3000000:
+			out = out.buffer
 
 	out.write(formatted_msg)
 

Modified: main/branches/prefix/pym/portage/output.py
===================================================================
--- main/branches/prefix/pym/portage/output.py	2009-10-27 22:56:07 UTC (rev 14742)
+++ main/branches/prefix/pym/portage/output.py	2009-10-28 18:36:02 UTC (rev 14743)
@@ -251,12 +251,16 @@
 			mystr = mystr[:_max_xtermTitle_len]
 		if not raw:
 			mystr = '\x1b]0;%s\x07' % mystr
-		if sys.hexversion < 0x3000000 and isinstance(mystr, unicode):
-			# avoid potential UnicodeEncodeError
-			mystr = mystr.encode(_encodings['stdio'], 'backslashreplace')
-		sys.stderr.write(mystr)
-		sys.stderr.flush()
 
+		# avoid potential UnicodeEncodeError
+		mystr = _unicode_encode(mystr,
+			encoding=_encodings['stdio'], errors='backslashreplace')
+		f = sys.stderr
+		if sys.hexversion >= 0x3000000:
+			f = f.buffer
+		f.write(mystr)
+		f.flush()
+
 default_xterm_title = None
 
 def xtermTitleReset():
@@ -374,11 +378,12 @@
 			self._write(self.write_listener, s)
 
 	def _write(self, f, s):
-		if sys.hexversion < 0x3000000 and \
-			isinstance(s, unicode) and \
-			f in (sys.stdout, sys.stderr):
-			# avoid potential UnicodeEncodeError
-			s = s.encode(_encodings['stdio'], 'backslashreplace')
+		# avoid potential UnicodeEncodeError
+		if f in (sys.stdout, sys.stderr):
+			s = _unicode_encode(s,
+				encoding=_encodings['stdio'], errors='backslashreplace')
+			if sys.hexversion >= 0x3000000:
+				f = f.buffer
 		f.write(s)
 
 	def writelines(self, lines):
@@ -484,9 +489,12 @@
 		sys.stderr.flush()
 
 	def _write(self, f, s):
-		if sys.hexversion < 0x3000000 and isinstance(s, unicode):
-			# avoid potential UnicodeEncodeError
-			s = s.encode(_encodings['stdio'], 'backslashreplace')
+		# avoid potential UnicodeEncodeError
+		s = _unicode_encode(s,
+			encoding=_encodings['stdio'], errors='backslashreplace')
+		f = sys.stderr
+		if sys.hexversion >= 0x3000000:
+			f = f.buffer
 		f.write(s)
 		f.flush()
 

Modified: main/branches/prefix/pym/portage/util.py
===================================================================
--- main/branches/prefix/pym/portage/util.py	2009-10-27 22:56:07 UTC (rev 14742)
+++ main/branches/prefix/pym/portage/util.py	2009-10-28 18:36:02 UTC (rev 14743)
@@ -68,10 +68,11 @@
 	if fd is None:
 		fd = sys.stderr
 	if noiselevel <= noiselimit:
-		if sys.hexversion < 0x3000000:
-			# avoid potential UnicodeEncodeError
-			mystr = _unicode_encode(mystr,
-				encoding=_encodings['stdio'], errors='backslashreplace')
+		# avoid potential UnicodeEncodeError
+		mystr = _unicode_encode(mystr,
+			encoding=_encodings['stdio'], errors='backslashreplace')
+		if sys.hexversion >= 0x3000000:
+			fd = fd.buffer
 		fd.write(mystr)
 		fd.flush()