[gentoo-commits] portage r10547 - in main/trunk: man pym/portage

[Zac Medico (zmedico)]

Author: zmedico
Date: 2008-06-03 02:24:40 +0000 (Tue, 03 Jun 2008)
New Revision: 10547

Modified:
   main/trunk/man/make.conf.5
   main/trunk/pym/portage/__init__.py
Log:
Add support for a PORTAGE_RO_DISTDIRS variable. When a given file does not
exist in DISTDIR, search for the file in this list of directories. Search
order is from left to right. Note that the current implementation works by
creating a symlink inside DISTDIR, but that may change in the future.


Modified: main/trunk/man/make.conf.5
===================================================================
--- main/trunk/man/make.conf.5	2008-06-03 01:41:09 UTC (rev 10546)
+++ main/trunk/man/make.conf.5	2008-06-03 02:24:40 UTC (rev 10547)
@@ -451,6 +451,12 @@
 it will increment it.  For more information about nice levels and what
 are acceptable ranges, see \fBnice\fR(1).
 .TP
+\fBPORTAGE_RO_DISTDIRS\fR = \fI[space delimited list of directories]\fR
+When a given file does not exist in \fBDISTDIR\fR, search for the file
+in this list of directories. Search order is from left to right. Note
+that the current implementation works by creating a symlink inside
+\fBDISTDIR\fR, but that may change in the future.
+.TP
 \fBPORTAGE_RSYNC_INITIAL_TIMEOUT\fR = \fIinteger\fR
 Used by \fBemerge \-\-sync\fR as a timeout for the initial connection to an
 rsync server.

Modified: main/trunk/pym/portage/__init__.py
===================================================================
--- main/trunk/pym/portage/__init__.py	2008-06-03 01:41:09 UTC (rev 10546)
+++ main/trunk/pym/portage/__init__.py	2008-06-03 02:24:40 UTC (rev 10547)
@@ -988,6 +988,7 @@
 		"PORTAGE_FETCH_CHECKSUM_TRY_MIRRORS", "PORTAGE_FETCH_RESUME_MIN_SIZE",
 		"PORTAGE_GPG_DIR",
 		"PORTAGE_GPG_KEY", "PORTAGE_PACKAGE_EMPTY_ABORT",
+		"PORTAGE_RO_DISTDIRS",
 		"PORTAGE_RSYNC_EXTRA_OPTS", "PORTAGE_RSYNC_OPTS",
 		"PORTAGE_RSYNC_RETRIES", "PORTAGE_USE", "PORT_LOGDIR",
 		"QUICKPKG_DEFAULT_OPTS",
@@ -3096,6 +3097,54 @@
 	os.rename(filename, temp_filename)
 	return temp_filename
 
+def _check_digests(filename, digests):
+	"""
+	Check digests and displey a message if an error occurs.
+	@return True if all digests match, False otherwise.
+	"""
+	verified_ok, reason = portage.checksum.verify_all(filename, digests)
+	if not verified_ok:
+		writemsg("!!! Previously fetched" + \
+			" file: '%s'\n" % filename, noiselevel=-1)
+		writemsg("!!! Reason: %s\n" % reason[0],
+			noiselevel=-1)
+		writemsg(("!!! Got:      %s\n" + \
+			"!!! Expected: %s\n") % \
+			(reason[1], reason[2]), noiselevel=-1)
+		return False
+	return True
+
+def _check_distfile(filename, digests, eout):
+	"""
+	@return a tuple of (match, stat_obj) where match is True if filename
+	matches all given digests (if any) and stat_obj is a stat result, or
+	None if the file does not exist.
+	"""
+	if digests is None:
+		digests = {}
+	size = digests.get("size")
+	if size is not None and len(digests) == 1:
+		digests = None
+
+	try:
+		st = os.stat(filename)
+	except OSError:
+		return (False, None)
+	if size is not None and size != st.st_size:
+		return (False, st)
+	if not digests:
+		if size is not None:
+			eout.ebegin("%s %s ;-)" % (os.path.basename(filename), "size"))
+			eout.eend(0)
+	else:
+		if _check_digests(filename, digests):
+			eout.ebegin("%s %s ;-)" % (os.path.basename(filename),
+				" ".join(sorted(digests))))
+			eout.eend(0)
+		else:
+			return (False, st)
+	return (True, st)
+
 _fetch_resume_size_re = re.compile('(^[\d]+)([KMGTPEZY]?$)')
 
 _size_suffix_map = {
@@ -3229,6 +3278,11 @@
 		# no digests because fetch was not called for a specific package
 		mydigests = {}
 
+	import shlex
+	ro_distdirs = [x for x in \
+		shlex.split(mysettings.get("PORTAGE_RO_DISTDIRS", "")) \
+		if os.path.isdir(x)]
+
 	fsmirrors = []
 	for x in range(len(mymirrors)-1,-1,-1):
 		if mymirrors[x] and mymirrors[x][0]=='/':
@@ -3356,8 +3410,16 @@
 		1        partially downloaded
 		2        completely downloaded
 		"""
+		fetched = 0
+
+		digests = mydigests.get(myfile, {})
+		size = digests.get("size")
+		if parallel_fetchonly:
+			digests.clear()
+			if size is not None:
+				digests["size"] = size
+
 		myfile_path = os.path.join(mysettings["DISTDIR"], myfile)
-		fetched=0
 		has_space = True
 		file_lock = None
 		if listonly:
@@ -3403,6 +3465,75 @@
 						waiting_msg=waiting_msg)
 		try:
 			if not listonly:
+
+				eout = portage.output.EOutput()
+				eout.quiet = mysettings.get("PORTAGE_QUIET") == "1"
+				match, mystat = _check_distfile(myfile_path, digests, eout)
+				if match:
+					if can_fetch and not fetch_to_ro:
+						try:
+							apply_secpass_permissions(myfile_path,
+								gid=portage_gid, mode=0664, mask=02,
+								stat_cached=mystat)
+						except portage.exception.PortageException, e:
+							if not os.access(myfile_path, os.R_OK):
+								writemsg("!!! Failed to adjust permissions:" + \
+									" %s\n" % str(e), noiselevel=-1)
+							del e
+					continue
+
+				if can_fetch and mystat is None:
+					# Remove broken symlinks if necessary.
+					try:
+						os.unlink(myfile_path)
+					except OSError:
+						pass
+
+				if mystat is not None:
+					if mystat.st_size == 0:
+						if can_fetch:
+							try:
+								os.unlink(myfile_path)
+							except OSError:
+								pass
+					elif can_fetch:
+						if mystat.st_size < fetch_resume_size and \
+							mystat.st_size < size:
+							writemsg((">>> Deleting distfile with size " + \
+								"%d (smaller than " "PORTAGE_FETCH_RESU" + \
+								"ME_MIN_SIZE)\n") % mystat.st_size)
+							try:
+								os.unlink(myfile_path)
+							except OSError, e:
+								if e.errno != errno.ENOENT:
+									raise
+								del e
+						elif mystat.st_size >= size:
+							temp_filename = \
+								_checksum_failure_temp_file(
+								mysettings["DISTDIR"], myfile)
+							writemsg_stdout("Refetching... " + \
+								"File renamed to '%s'\n\n" % \
+								temp_filename, noiselevel=-1)
+
+				if can_fetch and ro_distdirs:
+					readonly_file = None
+					for x in ro_distdirs:
+						filename = os.path.join(x, myfile)
+						match, mystat = _check_distfile(filename, digests, eout)
+						if match:
+							readonly_file = filename
+							break
+					if readonly_file is not None:
+						try:
+							os.unlink(myfile_path)
+						except OSError, e:
+							if e.errno != errno.ENOENT:
+								raise
+							del e
+						os.symlink(readonly_file, myfile_path)
+						continue
+
 				if fsmirrors and not os.path.exists(myfile_path) and has_space:
 					for mydir in fsmirrors:
 						mirror_file = os.path.join(mydir, myfile)

-- 
gentoo-commits@l.g.o mailing list