[gentoo-portage-dev] [PATCH 1/2] Add binrepos.conf to replace PORTAGE_BINHOST (bug 668334)

[Zac Medico]

Support /etc/portage/binrepos.conf as a replacement for the
PORTAGE_BINHOST variable. Behavior is similar to repos.conf,
initially supporting just the sync-uri attribute. Both binrepos.conf
and PORTAGE_BINHOST can be used simultaneously, in the same way that
repos.conf and PORTDIR_OVERLAY can be used simultaneously.

The emerge --info output for binrepos.conf looks like this:

Binary Repositories:

example-binhost
    sync-uri: https://example.com/packages

Bug: https://bugs.gentoo.org/668334
Signed-off-by: Zac Medico <zmedico@g.o>
---
 lib/_emerge/actions.py                  |  13 ++-
 lib/portage/binrepo/__init__.py         |   0
 lib/portage/binrepo/config.py           | 131 ++++++++++++++++++++++++
 lib/portage/const.py                    |   1 +
 lib/portage/dbapi/bintree.py            |  14 ++-
 lib/portage/tests/emerge/test_simple.py |  14 ++-
 man/make.conf.5                         |   3 +-
 man/portage.5                           |  38 +++++++
 8 files changed, 206 insertions(+), 8 deletions(-)
 create mode 100644 lib/portage/binrepo/__init__.py
 create mode 100644 lib/portage/binrepo/config.py

diff --git a/lib/_emerge/actions.py b/lib/_emerge/actions.py
index a4ecfe43d..34344f046 100644
--- a/lib/_emerge/actions.py
+++ b/lib/_emerge/actions.py
@@ -32,7 +32,8 @@ portage.proxy.lazyimport.lazyimport(globals(),
 from portage import os
 from portage import shutil
 from portage import _encodings, _unicode_decode
-from portage.const import _DEPCLEAN_LIB_CHECK_DEFAULT
+from portage.binrepo.config import BinRepoConfigLoader
+from portage.const import BINREPOS_CONF_FILE, _DEPCLEAN_LIB_CHECK_DEFAULT
 from portage.dbapi.dep_expand import dep_expand
 from portage.dbapi._expand_new_virt import expand_new_virt
 from portage.dbapi.IndexedPortdb import IndexedPortdb
@@ -1836,6 +1837,16 @@ def action_info(settings, trees, myopts, myfiles):
 	for repo in repos:
 		append(repo.info_string())
 
+	binrepos_conf_path = os.path.join(settings['PORTAGE_CONFIGROOT'], BINREPOS_CONF_FILE)
+	binrepos_conf = BinRepoConfigLoader((binrepos_conf_path,), settings)
+	if binrepos_conf and any(repo.name for repo in binrepos_conf.values()):
+		append("Binary Repositories:\n")
+		for repo in reversed(list(binrepos_conf.values())):
+			# Omit repos from the PORTAGE_BINHOST variable, since they
+			# do not have a name to label them with.
+			if repo.name:
+				append(repo.info_string())
+
 	installed_sets = sorted(s for s in
 		root_config.sets['selected'].getNonAtoms() if s.startswith(SETPREFIX))
 	if installed_sets:
diff --git a/lib/portage/binrepo/__init__.py b/lib/portage/binrepo/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/lib/portage/binrepo/config.py b/lib/portage/binrepo/config.py
new file mode 100644
index 000000000..aa3ff7a77
--- /dev/null
+++ b/lib/portage/binrepo/config.py
@@ -0,0 +1,131 @@
+# Copyright 2020 Gentoo Authors
+# Distributed under the terms of the GNU General Public License v2
+
+from collections import OrderedDict
+from collections.abc import Mapping
+from hashlib import md5
+
+from portage.localization import _
+from portage.util import _recursive_file_list, writemsg
+from portage.util.configparser import (SafeConfigParser, ConfigParserError,
+	read_configs)
+
+
+class BinRepoConfig:
+	__slots__ = (
+		'name',
+		'name_fallback',
+		'priority',
+		'sync_uri',
+	)
+	def __init__(self, opts):
+		"""
+		Create a BinRepoConfig with options in opts.
+		"""
+		for k in self.__slots__:
+			setattr(self, k, opts.get(k.replace('_', '-')))
+
+	def info_string(self):
+		"""
+		Returns a formatted string containing informations about the repository.
+		Used by emerge --info.
+		"""
+		indent = " " * 4
+		repo_msg = []
+		repo_msg.append(self.name or self.name_fallback)
+		if self.priority is not None:
+			repo_msg.append(indent + "priority: " + str(self.priority))
+		repo_msg.append(indent + "sync-uri: " + self.sync_uri)
+		repo_msg.append("")
+		return "\n".join(repo_msg)
+
+
+class BinRepoConfigLoader(Mapping):
+	def __init__(self, paths, settings):
+		"""Load config from files in paths"""
+
+		# Defaults for value interpolation.
+		parser_defaults = {
+			"EPREFIX" : settings["EPREFIX"],
+			"EROOT" : settings["EROOT"],
+			"PORTAGE_CONFIGROOT" : settings["PORTAGE_CONFIGROOT"],
+			"ROOT" : settings["ROOT"],
+		}
+
+		try:
+			parser = self._parse(paths, parser_defaults)
+		except ConfigParserError as e:
+			writemsg(
+				_("!!! Error while reading binrepo config file: %s\n") % e,
+				noiselevel=-1)
+			parser = SafeConfigParser(defaults=parser_defaults)
+
+		repos = []
+		sync_uris = []
+		for section_name in parser.sections():
+			repo_data = dict(parser[section_name].items())
+			repo_data['name'] = section_name
+			repo = BinRepoConfig(repo_data)
+			if repo.sync_uri is None:
+				writemsg(_("!!! Missing sync-uri setting for binrepo %s\n") % (repo.name,), noiselevel=-1)
+				continue
+
+			sync_uri = self._normalize_uri(repo.sync_uri)
+			sync_uris.append(sync_uri)
+			repo.sync_uri = sync_uri
+			if repo.priority is not None:
+				try:
+					repo.priority = int(repo.priority)
+				except ValueError:
+					repo.priority = None
+			repos.append(repo)
+
+		sync_uris = set(sync_uris)
+		current_priority = 0
+		for sync_uri in reversed(settings.get("PORTAGE_BINHOST", "").split()):
+			sync_uri = self._normalize_uri(sync_uri)
+			if sync_uri not in sync_uris:
+				current_priority += 1
+				sync_uris.add(sync_uri)
+				repos.append(BinRepoConfig({
+					'name-fallback': self._digest_uri(sync_uri),
+					'name': None,
+					'priority': current_priority,
+					'sync-uri': sync_uri,
+				}))
+
+		self._data = OrderedDict((repo.name, repo) for repo in
+			sorted(repos, key=lambda repo: (repo.priority or 0, repo.name or repo.name_fallback)))
+
+	@staticmethod
+	def _digest_uri(uri):
+		return md5(uri.encode('utf_8')).hexdigest()
+
+	@staticmethod
+	def _normalize_uri(uri):
+		return uri.rstrip('/')
+
+	@staticmethod
+	def _parse(paths, defaults):
+		parser = SafeConfigParser(defaults=defaults)
+		recursive_paths = []
+		for p in paths:
+			if isinstance(p, str):
+				recursive_paths.extend(_recursive_file_list(p))
+			else:
+				recursive_paths.append(p)
+
+		read_configs(parser, recursive_paths)
+		return parser
+
+	def __iter__(self):
+		return iter(self._data)
+
+	def __contains__(self, key):
+		return key in self._data
+
+	def __getitem__(self, key):
+		return self._data[key]
+
+	def __len__(self):
+		return len(self._data)
diff --git a/lib/portage/const.py b/lib/portage/const.py
index 9a7ea23bd..b895f0fa9 100644
--- a/lib/portage/const.py
+++ b/lib/portage/const.py
@@ -28,6 +28,7 @@ import os
 
 # variables used with config_root (these need to be relative)
 USER_CONFIG_PATH         = "etc/portage"
+BINREPOS_CONF_FILE       = USER_CONFIG_PATH + "/binrepos.conf"
 MAKE_CONF_FILE           = USER_CONFIG_PATH + "/make.conf"
 MODULES_FILE_PATH        = USER_CONFIG_PATH + "/modules"
 CUSTOM_PROFILE_PATH      = USER_CONFIG_PATH + "/profile"
diff --git a/lib/portage/dbapi/bintree.py b/lib/portage/dbapi/bintree.py
index 620865a79..a96183561 100644
--- a/lib/portage/dbapi/bintree.py
+++ b/lib/portage/dbapi/bintree.py
@@ -22,8 +22,9 @@ portage.proxy.lazyimport.lazyimport(globals(),
 	'portage.versions:best,catpkgsplit,catsplit,_pkg_str',
 )
 
+from portage.binrepo.config import BinRepoConfigLoader
 from portage.cache.mappings import slot_dict_class
-from portage.const import CACHE_PATH, SUPPORTED_XPAK_EXTENSIONS
+from portage.const import BINREPOS_CONF_FILE, CACHE_PATH, SUPPORTED_XPAK_EXTENSIONS
 from portage.dbapi.virtual import fakedbapi
 from portage.dep import Atom, use_reduce, paren_enclose
 from portage.exception import AlarmSignal, InvalidData, InvalidPackageName, \
@@ -364,6 +365,7 @@ class binarytree:
 			self.move_slot_ent = self.dbapi.move_slot_ent
 			self.populated = 0
 			self.tree = {}
+			self._binrepos_conf = None
 			self._remote_has_index = False
 			self._remotepkgs = None # remote metadata indexed by cpv
 			self._additional_pkgs = {}
@@ -628,8 +630,10 @@ class binarytree:
 				self._populate_additional(add_repos)
 
 			if getbinpkgs:
-				if not self.settings.get("PORTAGE_BINHOST"):
-					writemsg(_("!!! PORTAGE_BINHOST unset, but use is requested.\n"),
+				config_path = os.path.join(self.settings['PORTAGE_CONFIGROOT'], BINREPOS_CONF_FILE)
+				self._binrepos_conf = BinRepoConfigLoader((config_path,), self.settings)
+				if not self._binrepos_conf:
+					writemsg(_("!!! %s is missing (or PORTAGE_BINHOST is unset), but use is requested.\n") % (config_path,),
 						noiselevel=-1)
 				else:
 					self._populate_remote(getbinpkg_refresh=getbinpkg_refresh)
@@ -903,7 +907,9 @@ class binarytree:
 
 		self._remote_has_index = False
 		self._remotepkgs = {}
-		for base_url in self.settings["PORTAGE_BINHOST"].split():
+		# Order by descending priority.
+		for repo in reversed(list(self._binrepos_conf.values())):
+			base_url = repo.sync_uri
 			parsed_url = urlparse(base_url)
 			host = parsed_url.netloc
 			port = parsed_url.port
diff --git a/lib/portage/tests/emerge/test_simple.py b/lib/portage/tests/emerge/test_simple.py
index c24f5c603..8635b70e4 100644
--- a/lib/portage/tests/emerge/test_simple.py
+++ b/lib/portage/tests/emerge/test_simple.py
@@ -7,7 +7,7 @@ import sys
 import portage
 from portage import shutil, os
 from portage import _unicode_decode
-from portage.const import (BASH_BINARY, PORTAGE_PYM_PATH, USER_CONFIG_PATH)
+from portage.const import (BASH_BINARY, BINREPOS_CONF_FILE, PORTAGE_PYM_PATH, USER_CONFIG_PATH)
 from portage.cache.mappings import Mapping
 from portage.process import find_binary
 from portage.tests import TestCase
@@ -419,13 +419,23 @@ call_has_and_best_version() {
 		)
 
 		# Test binhost support if FETCHCOMMAND is available.
+		binrepos_conf_file = os.path.join(os.sep, eprefix, BINREPOS_CONF_FILE)
+		with open(binrepos_conf_file, 'wt') as f:
+			f.write('[test-binhost]\n')
+			f.write('sync-uri = {}\n'.format(binhost_uri))
 		fetchcommand = portage.util.shlex_split(playground.settings['FETCHCOMMAND'])
 		fetch_bin = portage.process.find_binary(fetchcommand[0])
 		if fetch_bin is not None:
 			test_commands = test_commands + (
+				lambda: os.rename(pkgdir, binhost_dir),
+				emerge_cmd + ("-e", "--getbinpkgonly", "dev-libs/A"),
+				lambda: shutil.rmtree(pkgdir),
+				lambda: os.rename(binhost_dir, pkgdir),
+				# Remove binrepos.conf and test PORTAGE_BINHOST.
+				lambda: os.unlink(binrepos_conf_file),
 				lambda: os.rename(pkgdir, binhost_dir),
 				({"PORTAGE_BINHOST": binhost_uri},) + \
-					emerge_cmd + ("-e", "--getbinpkgonly", "dev-libs/A"),
+					emerge_cmd + ("-fe", "--getbinpkgonly", "dev-libs/A"),
 				lambda: shutil.rmtree(pkgdir),
 				lambda: os.rename(binhost_dir, pkgdir),
 			)
diff --git a/man/make.conf.5 b/man/make.conf.5
index eb812150f..f7b2460e5 100644
--- a/man/make.conf.5
+++ b/man/make.conf.5
@@ -853,7 +853,8 @@ Each entry in the list must specify the full address of a directory
 serving tbz2's for your system (this directory must contain a 'Packages' index
 file). This is only used when running with
 the get binary pkg options are given to \fBemerge\fR.  Review \fBemerge\fR(1)
-for more information.
+for more information. The \fBPORTAGE_BINHOST\fR variable is deprecated in
+favor of the \fBbinrepos.conf\fR configuration file (see \fBportage\fR(5)).
 .TP
 \fBPORTAGE_BINHOST_HEADER_URI\fR = \
 \fI"ftp://login:pass@××××××××××.site/pub/grp/i686/athlon\-xp/"\fR
diff --git a/man/portage.5 b/man/portage.5
index 7472972cc..890a22adb 100644
--- a/man/portage.5
+++ b/man/portage.5
@@ -47,6 +47,7 @@ virtuals
 .BR /etc/portage/
 .nf
 bashrc
+binrepos.conf
 categories
 color.map
 license_groups
@@ -620,6 +621,43 @@ any other bash script.
 
 Additional package-specific bashrc files can be created in /etc/portage/env.
 .TP
+.BR binrepos.conf
+Specifies remote binary package repository configuration information. This
+is intended to be used as a replacement for the \fBmake.conf\fR(5)
+\fBPORTAGE_BINHOST\fR variable.
+
+.I Format:
+.nf
+\- comments begin with # (no inline comments)
+\- configuration of each repository is specified in a section starting with \
+"[${repository_name}]"
+\- attributes are specified in "${attribute} = ${value}" format
+.fi
+
+.RS
+.I Attributes supported in sections of repositories:
+.RS
+.TP
+.B priority
+Specifies priority of given repository. When a package exists in multiple
+repositories, those with higher priority are preferred.
+.TP
+.B sync\-uri
+Specifies URI of repository used for `emerge \-\-getbinpkg`.
+.RE
+.RE
+
+.I Example:
+.nf
+[example-binhost]
+# repos with higher priorities are preferred when packages with equal
+# versions are found in multiple repos
+priority = 9999
+# packages are fetched from here
+sync-uri = https://example.com/binhost
+
+.fi
+.TP
 .BR categories
 A simple list of valid categories that may be used in repositories and PKGDIR
 (see \fBmake.conf\fR(5)). This allows for custom categories to be created.
-- 
2.25.3