[gentoo-portage-dev] [PATCH 2/2] binrepos.conf: support fetchcommand customization (bug 661332)

[Zac Medico]

Support customization of fetchcommand and resumecommand in
binrepos.conf, allowing customized authentication mechanisms for
each repository.

Bug: https://bugs.gentoo.org/661332
Signed-off-by: Zac Medico <zmedico@g.o>
---
 lib/_emerge/BinpkgFetcher.py  | 29 +++++++++++++++++++----------
 lib/portage/binrepo/config.py |  2 ++
 lib/portage/dbapi/bintree.py  | 34 +++++++++++++++++++++++++---------
 man/portage.5                 | 14 ++++++++++++++
 4 files changed, 60 insertions(+), 19 deletions(-)

diff --git a/lib/_emerge/BinpkgFetcher.py b/lib/_emerge/BinpkgFetcher.py
index 218d4d2ab..9a96bde28 100644
--- a/lib/_emerge/BinpkgFetcher.py
+++ b/lib/_emerge/BinpkgFetcher.py
@@ -96,14 +96,17 @@ class _BinpkgFetcherProcess(SpawnProcess):
 
 		# urljoin doesn't work correctly with
 		# unrecognized protocols like sftp
+		fetchcommand = None
+		resumecommand = None
 		if bintree._remote_has_index:
-			instance_key = bintree.dbapi._instance_key(pkg.cpv)
-			rel_uri = bintree._remotepkgs[instance_key].get("PATH")
+			remote_metadata = bintree._remotepkgs[bintree.dbapi._instance_key(pkg.cpv)]
+			rel_uri = remote_metadata.get("PATH")
 			if not rel_uri:
 				rel_uri = pkg.cpv + ".tbz2"
-			remote_base_uri = bintree._remotepkgs[
-				instance_key]["BASE_URI"]
+			remote_base_uri = remote_metadata["BASE_URI"]
 			uri = remote_base_uri.rstrip("/") + "/" + rel_uri.lstrip("/")
+			fetchcommand = remote_metadata.get('FETCHCOMMAND')
+			resumecommand = remote_metadata.get('RESUMECOMMAND')
 		else:
 			uri = settings["PORTAGE_BINHOST"].rstrip("/") + \
 				"/" + pkg.pf + ".tbz2"
@@ -114,13 +117,19 @@ class _BinpkgFetcherProcess(SpawnProcess):
 			self._async_wait()
 			return
 
-		protocol = urllib_parse_urlparse(uri)[0]
-		fcmd_prefix = "FETCHCOMMAND"
+		fcmd = None
 		if resume:
-			fcmd_prefix = "RESUMECOMMAND"
-		fcmd = settings.get(fcmd_prefix + "_" + protocol.upper())
-		if not fcmd:
-			fcmd = settings.get(fcmd_prefix)
+			fcmd = resumecommand
+		else:
+			fcmd = fetchcommand
+		if fcmd is None:
+			protocol = urllib_parse_urlparse(uri)[0]
+			fcmd_prefix = "FETCHCOMMAND"
+			if resume:
+				fcmd_prefix = "RESUMECOMMAND"
+			fcmd = settings.get(fcmd_prefix + "_" + protocol.upper())
+			if not fcmd:
+				fcmd = settings.get(fcmd_prefix)
 
 		fcmd_vars = {
 			"DISTDIR" : os.path.dirname(pkg_path),
diff --git a/lib/portage/binrepo/config.py b/lib/portage/binrepo/config.py
index aa3ff7a77..0c01f6fae 100644
--- a/lib/portage/binrepo/config.py
+++ b/lib/portage/binrepo/config.py
@@ -15,7 +15,9 @@ class BinRepoConfig:
 	__slots__ = (
 		'name',
 		'name_fallback',
+		'fetchcommand',
 		'priority',
+		'resumecommand',
 		'sync_uri',
 	)
 	def __init__(self, opts):
diff --git a/lib/portage/dbapi/bintree.py b/lib/portage/dbapi/bintree.py
index a96183561..ebc6765b6 100644
--- a/lib/portage/dbapi/bintree.py
+++ b/lib/portage/dbapi/bintree.py
@@ -382,10 +382,10 @@ class binarytree:
 			self._pkgindex_keys.update(["CPV", "SIZE"])
 			self._pkgindex_aux_keys = \
 				["BASE_URI", "BDEPEND", "BUILD_ID", "BUILD_TIME", "CHOST",
-				"DEFINED_PHASES", "DEPEND", "DESCRIPTION", "EAPI",
+				"DEFINED_PHASES", "DEPEND", "DESCRIPTION", "EAPI", "FETCHCOMMAND",
 				"IUSE", "KEYWORDS", "LICENSE", "PDEPEND",
 				"PKGINDEX_URI", "PROPERTIES", "PROVIDES",
-				"RDEPEND", "repository", "REQUIRES", "RESTRICT",
+				"RDEPEND", "repository", "REQUIRES", "RESTRICT", "RESUMECOMMAND",
 				"SIZE", "SLOT", "USE"]
 			self._pkgindex_aux_keys = list(self._pkgindex_aux_keys)
 			self._pkgindex_use_evaluated_keys = \
@@ -972,7 +972,7 @@ class binarytree:
 
 				# Don't use urlopen for https, unless
 				# PEP 476 is supported (bug #469888).
-				if parsed_url.scheme not in ('https',) or _have_pep_476():
+				if repo.fetchcommand is None and (parsed_url.scheme not in ('https',) or _have_pep_476()):
 					try:
 						f = _urlopen(url, if_modified_since=local_timestamp)
 						if hasattr(f, 'headers') and f.headers.get('timestamp', ''):
@@ -997,7 +997,7 @@ class binarytree:
 
 					path = parsed_url.path.rstrip("/") + "/Packages"
 
-					if parsed_url.scheme == 'ssh':
+					if repo.fetchcommand is None and parsed_url.scheme == 'ssh':
 						# Use a pipe so that we can terminate the download
 						# early if we detect that the TIMESTAMP header
 						# matches that of the cached Packages file.
@@ -1016,12 +1016,15 @@ class binarytree:
 							stdout=subprocess.PIPE)
 						f = proc.stdout
 					else:
-						setting = 'FETCHCOMMAND_' + parsed_url.scheme.upper()
-						fcmd = self.settings.get(setting)
-						if not fcmd:
-							fcmd = self.settings.get('FETCHCOMMAND')
+						if repo.fetchcommand is None:
+							setting = 'FETCHCOMMAND_' + parsed_url.scheme.upper()
+							fcmd = self.settings.get(setting)
 							if not fcmd:
-								raise EnvironmentError("FETCHCOMMAND is unset")
+								fcmd = self.settings.get('FETCHCOMMAND')
+								if not fcmd:
+									raise EnvironmentError("FETCHCOMMAND is unset")
+						else:
+							fcmd = repo.fetchcommand
 
 						fd, tmp_filename = tempfile.mkstemp()
 						tmp_dirname, tmp_basename = os.path.split(tmp_filename)
@@ -1135,6 +1138,19 @@ class binarytree:
 					d["CPV"] = cpv
 					d["BASE_URI"] = remote_base_uri
 					d["PKGINDEX_URI"] = url
+					# FETCHCOMMAND and RESUMECOMMAND may be specified
+					# by binrepos.conf, and otherwise ensure that they
+					# do not propagate from the Packages index since
+					# it may be unsafe to execute remotely specified
+					# commands.
+					if repo.fetchcommand is None:
+						d.pop('FETCHCOMMAND', None)
+					else:
+						d['FETCHCOMMAND'] = repo.fetchcommand
+					if repo.resumecommand is None:
+						d.pop('RESUMECOMMAND', None)
+					else:
+						d['RESUMECOMMAND'] = repo.resumecommand
 					self._remotepkgs[self.dbapi._instance_key(cpv)] = d
 					self.dbapi.cpv_inject(cpv)
 
diff --git a/man/portage.5 b/man/portage.5
index 890a22adb..9c88bc3dc 100644
--- a/man/portage.5
+++ b/man/portage.5
@@ -634,6 +634,20 @@ is intended to be used as a replacement for the \fBmake.conf\fR(5)
 \- attributes are specified in "${attribute} = ${value}" format
 .fi
 
+.RS
+.I Attributes supported in DEFAULT section:
+.RS
+.TP
+.B fetchcommand
+Specifies a \fBFETCHCOMMAND\fR used to fetch files from a repository,
+overriding the value from \fBmake.conf\fR(5).
+.TP
+.B resumecommand
+Specifies a \fBRESUMECOMMAND\fR used to fetch files from a repository,
+overriding the value from \fBmake.conf\fR(5).
+.RE
+.RE
+
 .RS
 .I Attributes supported in sections of repositories:
 .RS
-- 
2.25.3