[gentoo-commits] proj/portage:master commit in: lib/portage/binrepo/, lib/_emerge/, lib/portage/dbapi/, man/

[Zac Medico]

commit:     5ebc8a249b08318da5a2ca89cee2eed604f7e639
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Mon Sep  7 00:13:13 2020 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Tue Sep  8 01:38:08 2020 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=5ebc8a24

binrepos.conf: support fetchcommand customization (bug 668302)

Support customization of fetchcommand and resumecommand in
binrepos.conf, allowing customized authentication mechanisms for
each repository.

Bug: https://bugs.gentoo.org/668302
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>

 lib/_emerge/BinpkgFetcher.py  | 29 +++++++++++++++++++----------
 lib/portage/binrepo/config.py |  2 ++
 lib/portage/dbapi/bintree.py  | 34 +++++++++++++++++++++++++---------
 man/portage.5                 | 14 ++++++++++++++
 4 files changed, 60 insertions(+), 19 deletions(-)

diff --git a/lib/_emerge/BinpkgFetcher.py b/lib/_emerge/BinpkgFetcher.py
index 218d4d2ab..9a96bde28 100644
--- a/lib/_emerge/BinpkgFetcher.py
+++ b/lib/_emerge/BinpkgFetcher.py
@@ -96,14 +96,17 @@ class _BinpkgFetcherProcess(SpawnProcess):
 
 		# urljoin doesn't work correctly with
 		# unrecognized protocols like sftp
+		fetchcommand = None
+		resumecommand = None
 		if bintree._remote_has_index:
-			instance_key = bintree.dbapi._instance_key(pkg.cpv)
-			rel_uri = bintree._remotepkgs[instance_key].get("PATH")
+			remote_metadata = bintree._remotepkgs[bintree.dbapi._instance_key(pkg.cpv)]
+			rel_uri = remote_metadata.get("PATH")
 			if not rel_uri:
 				rel_uri = pkg.cpv + ".tbz2"
-			remote_base_uri = bintree._remotepkgs[
-				instance_key]["BASE_URI"]
+			remote_base_uri = remote_metadata["BASE_URI"]
 			uri = remote_base_uri.rstrip("/") + "/" + rel_uri.lstrip("/")
+			fetchcommand = remote_metadata.get('FETCHCOMMAND')
+			resumecommand = remote_metadata.get('RESUMECOMMAND')
 		else:
 			uri = settings["PORTAGE_BINHOST"].rstrip("/") + \
 				"/" + pkg.pf + ".tbz2"
@@ -114,13 +117,19 @@ class _BinpkgFetcherProcess(SpawnProcess):
 			self._async_wait()
 			return
 
-		protocol = urllib_parse_urlparse(uri)[0]
-		fcmd_prefix = "FETCHCOMMAND"
+		fcmd = None
 		if resume:
-			fcmd_prefix = "RESUMECOMMAND"
-		fcmd = settings.get(fcmd_prefix + "_" + protocol.upper())
-		if not fcmd:
-			fcmd = settings.get(fcmd_prefix)
+			fcmd = resumecommand
+		else:
+			fcmd = fetchcommand
+		if fcmd is None:
+			protocol = urllib_parse_urlparse(uri)[0]
+			fcmd_prefix = "FETCHCOMMAND"
+			if resume:
+				fcmd_prefix = "RESUMECOMMAND"
+			fcmd = settings.get(fcmd_prefix + "_" + protocol.upper())
+			if not fcmd:
+				fcmd = settings.get(fcmd_prefix)
 
 		fcmd_vars = {
 			"DISTDIR" : os.path.dirname(pkg_path),

diff --git a/lib/portage/binrepo/config.py b/lib/portage/binrepo/config.py
index a4bce9073..6ba1a3e9f 100644
--- a/lib/portage/binrepo/config.py
+++ b/lib/portage/binrepo/config.py
@@ -15,7 +15,9 @@ class BinRepoConfig:
 	__slots__ = (
 		'name',
 		'name_fallback',
+		'fetchcommand',
 		'priority',
+		'resumecommand',
 		'sync_uri',
 	)
 	def __init__(self, opts):

diff --git a/lib/portage/dbapi/bintree.py b/lib/portage/dbapi/bintree.py
index 97018db6e..e4393e06d 100644
--- a/lib/portage/dbapi/bintree.py
+++ b/lib/portage/dbapi/bintree.py
@@ -382,10 +382,10 @@ class binarytree:
 			self._pkgindex_keys.update(["CPV", "SIZE"])
 			self._pkgindex_aux_keys = \
 				["BASE_URI", "BDEPEND", "BUILD_ID", "BUILD_TIME", "CHOST",
-				"DEFINED_PHASES", "DEPEND", "DESCRIPTION", "EAPI",
+				"DEFINED_PHASES", "DEPEND", "DESCRIPTION", "EAPI", "FETCHCOMMAND",
 				"IUSE", "KEYWORDS", "LICENSE", "PDEPEND",
 				"PKGINDEX_URI", "PROPERTIES", "PROVIDES",
-				"RDEPEND", "repository", "REQUIRES", "RESTRICT",
+				"RDEPEND", "repository", "REQUIRES", "RESTRICT", "RESUMECOMMAND",
 				"SIZE", "SLOT", "USE"]
 			self._pkgindex_aux_keys = list(self._pkgindex_aux_keys)
 			self._pkgindex_use_evaluated_keys = \
@@ -979,7 +979,7 @@ class binarytree:
 
 				# Don't use urlopen for https, unless
 				# PEP 476 is supported (bug #469888).
-				if parsed_url.scheme not in ('https',) or _have_pep_476():
+				if repo.fetchcommand is None and (parsed_url.scheme not in ('https',) or _have_pep_476()):
 					try:
 						f = _urlopen(url, if_modified_since=local_timestamp, proxies=proxies)
 						if hasattr(f, 'headers') and f.headers.get('timestamp', ''):
@@ -1004,7 +1004,7 @@ class binarytree:
 
 					path = parsed_url.path.rstrip("/") + "/Packages"
 
-					if parsed_url.scheme == 'ssh':
+					if repo.fetchcommand is None and parsed_url.scheme == 'ssh':
 						# Use a pipe so that we can terminate the download
 						# early if we detect that the TIMESTAMP header
 						# matches that of the cached Packages file.
@@ -1023,12 +1023,15 @@ class binarytree:
 							stdout=subprocess.PIPE)
 						f = proc.stdout
 					else:
-						setting = 'FETCHCOMMAND_' + parsed_url.scheme.upper()
-						fcmd = self.settings.get(setting)
-						if not fcmd:
-							fcmd = self.settings.get('FETCHCOMMAND')
+						if repo.fetchcommand is None:
+							setting = 'FETCHCOMMAND_' + parsed_url.scheme.upper()
+							fcmd = self.settings.get(setting)
 							if not fcmd:
-								raise EnvironmentError("FETCHCOMMAND is unset")
+								fcmd = self.settings.get('FETCHCOMMAND')
+								if not fcmd:
+									raise EnvironmentError("FETCHCOMMAND is unset")
+						else:
+							fcmd = repo.fetchcommand
 
 						fd, tmp_filename = tempfile.mkstemp()
 						tmp_dirname, tmp_basename = os.path.split(tmp_filename)
@@ -1142,6 +1145,19 @@ class binarytree:
 					d["CPV"] = cpv
 					d["BASE_URI"] = remote_base_uri
 					d["PKGINDEX_URI"] = url
+					# FETCHCOMMAND and RESUMECOMMAND may be specified
+					# by binrepos.conf, and otherwise ensure that they
+					# do not propagate from the Packages index since
+					# it may be unsafe to execute remotely specified
+					# commands.
+					if repo.fetchcommand is None:
+						d.pop('FETCHCOMMAND', None)
+					else:
+						d['FETCHCOMMAND'] = repo.fetchcommand
+					if repo.resumecommand is None:
+						d.pop('RESUMECOMMAND', None)
+					else:
+						d['RESUMECOMMAND'] = repo.resumecommand
 					self._remotepkgs[self.dbapi._instance_key(cpv)] = d
 					self.dbapi.cpv_inject(cpv)
 

diff --git a/man/portage.5 b/man/portage.5
index 4f183654c..82dd8a509 100644
--- a/man/portage.5
+++ b/man/portage.5
@@ -634,6 +634,20 @@ is intended to be used as a replacement for the \fBmake.conf\fR(5)
 \- attributes are specified in "${attribute} = ${value}" format
 .fi
 
+.RS
+.I Attributes supported in DEFAULT section:
+.RS
+.TP
+.B fetchcommand
+Specifies a \fBFETCHCOMMAND\fR used to fetch files from a repository,
+overriding the value from \fBmake.conf\fR(5).
+.TP
+.B resumecommand
+Specifies a \fBRESUMECOMMAND\fR used to fetch files from a repository,
+overriding the value from \fBmake.conf\fR(5).
+.RE
+.RE
+
 .RS
 .I Attributes supported in sections of repositories:
 .RS