1 |
commit: c36a4ec6694b8b9e22fb63298d1588589acb1ab2 |
2 |
Author: Zac Medico <zmedico <AT> gentoo <DOT> org> |
3 |
AuthorDate: Sun Sep 6 21:12:32 2020 +0000 |
4 |
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org> |
5 |
CommitDate: Tue Sep 8 01:37:42 2020 +0000 |
6 |
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=c36a4ec6 |
7 |
|
8 |
Add binrepos.conf to replace PORTAGE_BINHOST (bug 668334) |
9 |
|
10 |
Support /etc/portage/binrepos.conf as a replacement for the |
11 |
PORTAGE_BINHOST variable. Behavior is similar to repos.conf, |
12 |
initially supporting just the sync-uri attribute. Both binrepos.conf |
13 |
and PORTAGE_BINHOST can be used simultaneously, in the same way that |
14 |
repos.conf and PORTDIR_OVERLAY can be used simultaneously. |
15 |
|
16 |
The emerge --info output for binrepos.conf looks like this: |
17 |
|
18 |
Binary Repositories: |
19 |
|
20 |
example-binhost |
21 |
sync-uri: https://example.com/binhost |
22 |
|
23 |
Bug: https://bugs.gentoo.org/668334 |
24 |
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org> |
25 |
|
26 |
lib/_emerge/actions.py | 13 +++- |
27 |
lib/portage/binrepo/__init__.py | 0 |
28 |
lib/portage/binrepo/config.py | 131 ++++++++++++++++++++++++++++++++ |
29 |
lib/portage/const.py | 1 + |
30 |
lib/portage/dbapi/bintree.py | 14 +++- |
31 |
lib/portage/tests/emerge/test_simple.py | 14 +++- |
32 |
man/make.conf.5 | 3 +- |
33 |
man/portage.5 | 38 +++++++++ |
34 |
8 files changed, 206 insertions(+), 8 deletions(-) |
35 |
|
36 |
diff --git a/lib/_emerge/actions.py b/lib/_emerge/actions.py |
37 |
index f57269817..5e8a46957 100644 |
38 |
--- a/lib/_emerge/actions.py |
39 |
+++ b/lib/_emerge/actions.py |
40 |
@@ -32,7 +32,8 @@ portage.proxy.lazyimport.lazyimport(globals(), |
41 |
from portage import os |
42 |
from portage import shutil |
43 |
from portage import _encodings, _unicode_decode |
44 |
-from portage.const import _DEPCLEAN_LIB_CHECK_DEFAULT |
45 |
+from portage.binrepo.config import BinRepoConfigLoader |
46 |
+from portage.const import BINREPOS_CONF_FILE, _DEPCLEAN_LIB_CHECK_DEFAULT |
47 |
from portage.dbapi.dep_expand import dep_expand |
48 |
from portage.dbapi._expand_new_virt import expand_new_virt |
49 |
from portage.dbapi.IndexedPortdb import IndexedPortdb |
50 |
@@ -1836,6 +1837,16 @@ def action_info(settings, trees, myopts, myfiles): |
51 |
for repo in repos: |
52 |
append(repo.info_string()) |
53 |
|
54 |
+ binrepos_conf_path = os.path.join(settings['PORTAGE_CONFIGROOT'], BINREPOS_CONF_FILE) |
55 |
+ binrepos_conf = BinRepoConfigLoader((binrepos_conf_path,), settings) |
56 |
+ if binrepos_conf and any(repo.name for repo in binrepos_conf.values()): |
57 |
+ append("Binary Repositories:\n") |
58 |
+ for repo in reversed(list(binrepos_conf.values())): |
59 |
+ # Omit repos from the PORTAGE_BINHOST variable, since they |
60 |
+ # do not have a name to label them with. |
61 |
+ if repo.name: |
62 |
+ append(repo.info_string()) |
63 |
+ |
64 |
installed_sets = sorted(s for s in |
65 |
root_config.sets['selected'].getNonAtoms() if s.startswith(SETPREFIX)) |
66 |
if installed_sets: |
67 |
|
68 |
diff --git a/lib/portage/binrepo/__init__.py b/lib/portage/binrepo/__init__.py |
69 |
new file mode 100644 |
70 |
index 000000000..e69de29bb |
71 |
|
72 |
diff --git a/lib/portage/binrepo/config.py b/lib/portage/binrepo/config.py |
73 |
new file mode 100644 |
74 |
index 000000000..a4bce9073 |
75 |
--- /dev/null |
76 |
+++ b/lib/portage/binrepo/config.py |
77 |
@@ -0,0 +1,131 @@ |
78 |
+# Copyright 2020 Gentoo Authors |
79 |
+# Distributed under the terms of the GNU General Public License v2 |
80 |
+ |
81 |
+from collections import OrderedDict |
82 |
+from collections.abc import Mapping |
83 |
+from hashlib import md5 |
84 |
+ |
85 |
+from portage.localization import _ |
86 |
+from portage.util import _recursive_file_list, writemsg |
87 |
+from portage.util.configparser import (SafeConfigParser, ConfigParserError, |
88 |
+ read_configs) |
89 |
+ |
90 |
+ |
91 |
+class BinRepoConfig: |
92 |
+ __slots__ = ( |
93 |
+ 'name', |
94 |
+ 'name_fallback', |
95 |
+ 'priority', |
96 |
+ 'sync_uri', |
97 |
+ ) |
98 |
+ def __init__(self, opts): |
99 |
+ """ |
100 |
+ Create a BinRepoConfig with options in opts. |
101 |
+ """ |
102 |
+ for k in self.__slots__: |
103 |
+ setattr(self, k, opts.get(k.replace('_', '-'))) |
104 |
+ |
105 |
+ def info_string(self): |
106 |
+ """ |
107 |
+ Returns a formatted string containing informations about the repository. |
108 |
+ Used by emerge --info. |
109 |
+ """ |
110 |
+ indent = " " * 4 |
111 |
+ repo_msg = [] |
112 |
+ repo_msg.append(self.name or self.name_fallback) |
113 |
+ if self.priority is not None: |
114 |
+ repo_msg.append(indent + "priority: " + str(self.priority)) |
115 |
+ repo_msg.append(indent + "sync-uri: " + self.sync_uri) |
116 |
+ repo_msg.append("") |
117 |
+ return "\n".join(repo_msg) |
118 |
+ |
119 |
+ |
120 |
+class BinRepoConfigLoader(Mapping): |
121 |
+ def __init__(self, paths, settings): |
122 |
+ """Load config from files in paths""" |
123 |
+ |
124 |
+ # Defaults for value interpolation. |
125 |
+ parser_defaults = { |
126 |
+ "EPREFIX" : settings["EPREFIX"], |
127 |
+ "EROOT" : settings["EROOT"], |
128 |
+ "PORTAGE_CONFIGROOT" : settings["PORTAGE_CONFIGROOT"], |
129 |
+ "ROOT" : settings["ROOT"], |
130 |
+ } |
131 |
+ |
132 |
+ try: |
133 |
+ parser = self._parse(paths, parser_defaults) |
134 |
+ except ConfigParserError as e: |
135 |
+ writemsg( |
136 |
+ _("!!! Error while reading binrepo config file: %s\n") % e, |
137 |
+ noiselevel=-1) |
138 |
+ parser = SafeConfigParser(defaults=parser_defaults) |
139 |
+ |
140 |
+ repos = [] |
141 |
+ sync_uris = [] |
142 |
+ for section_name in parser.sections(): |
143 |
+ repo_data = dict(parser[section_name].items()) |
144 |
+ repo_data['name'] = section_name |
145 |
+ repo = BinRepoConfig(repo_data) |
146 |
+ if repo.sync_uri is None: |
147 |
+ writemsg(_("!!! Missing sync-uri setting for binrepo %s\n") % (repo.name,), noiselevel=-1) |
148 |
+ continue |
149 |
+ |
150 |
+ sync_uri = self._normalize_uri(repo.sync_uri) |
151 |
+ sync_uris.append(sync_uri) |
152 |
+ repo.sync_uri = sync_uri |
153 |
+ if repo.priority is not None: |
154 |
+ try: |
155 |
+ repo.priority = int(repo.priority) |
156 |
+ except ValueError: |
157 |
+ repo.priority = None |
158 |
+ repos.append(repo) |
159 |
+ |
160 |
+ sync_uris = set(sync_uris) |
161 |
+ current_priority = 0 |
162 |
+ for sync_uri in reversed(settings.get("PORTAGE_BINHOST", "").split()): |
163 |
+ sync_uri = self._normalize_uri(sync_uri) |
164 |
+ if sync_uri not in sync_uris: |
165 |
+ current_priority += 1 |
166 |
+ sync_uris.add(sync_uri) |
167 |
+ repos.append(BinRepoConfig({ |
168 |
+ 'name-fallback': self._digest_uri(sync_uri), |
169 |
+ 'name': None, |
170 |
+ 'priority': current_priority, |
171 |
+ 'sync-uri': sync_uri, |
172 |
+ })) |
173 |
+ |
174 |
+ self._data = OrderedDict((repo.name or repo.name_fallback, repo) for repo in |
175 |
+ sorted(repos, key=lambda repo: (repo.priority or 0, repo.name or repo.name_fallback))) |
176 |
+ |
177 |
+ @staticmethod |
178 |
+ def _digest_uri(uri): |
179 |
+ return md5(uri.encode('utf_8')).hexdigest() |
180 |
+ |
181 |
+ @staticmethod |
182 |
+ def _normalize_uri(uri): |
183 |
+ return uri.rstrip('/') |
184 |
+ |
185 |
+ @staticmethod |
186 |
+ def _parse(paths, defaults): |
187 |
+ parser = SafeConfigParser(defaults=defaults) |
188 |
+ recursive_paths = [] |
189 |
+ for p in paths: |
190 |
+ if isinstance(p, str): |
191 |
+ recursive_paths.extend(_recursive_file_list(p)) |
192 |
+ else: |
193 |
+ recursive_paths.append(p) |
194 |
+ |
195 |
+ read_configs(parser, recursive_paths) |
196 |
+ return parser |
197 |
+ |
198 |
+ def __iter__(self): |
199 |
+ return iter(self._data) |
200 |
+ |
201 |
+ def __contains__(self, key): |
202 |
+ return key in self._data |
203 |
+ |
204 |
+ def __getitem__(self, key): |
205 |
+ return self._data[key] |
206 |
+ |
207 |
+ def __len__(self): |
208 |
+ return len(self._data) |
209 |
|
210 |
diff --git a/lib/portage/const.py b/lib/portage/const.py |
211 |
index 9a7ea23bd..b895f0fa9 100644 |
212 |
--- a/lib/portage/const.py |
213 |
+++ b/lib/portage/const.py |
214 |
@@ -28,6 +28,7 @@ import os |
215 |
|
216 |
# variables used with config_root (these need to be relative) |
217 |
USER_CONFIG_PATH = "etc/portage" |
218 |
+BINREPOS_CONF_FILE = USER_CONFIG_PATH + "/binrepos.conf" |
219 |
MAKE_CONF_FILE = USER_CONFIG_PATH + "/make.conf" |
220 |
MODULES_FILE_PATH = USER_CONFIG_PATH + "/modules" |
221 |
CUSTOM_PROFILE_PATH = USER_CONFIG_PATH + "/profile" |
222 |
|
223 |
diff --git a/lib/portage/dbapi/bintree.py b/lib/portage/dbapi/bintree.py |
224 |
index ee30542a5..97018db6e 100644 |
225 |
--- a/lib/portage/dbapi/bintree.py |
226 |
+++ b/lib/portage/dbapi/bintree.py |
227 |
@@ -22,8 +22,9 @@ portage.proxy.lazyimport.lazyimport(globals(), |
228 |
'portage.versions:best,catpkgsplit,catsplit,_pkg_str', |
229 |
) |
230 |
|
231 |
+from portage.binrepo.config import BinRepoConfigLoader |
232 |
from portage.cache.mappings import slot_dict_class |
233 |
-from portage.const import CACHE_PATH, SUPPORTED_XPAK_EXTENSIONS |
234 |
+from portage.const import BINREPOS_CONF_FILE, CACHE_PATH, SUPPORTED_XPAK_EXTENSIONS |
235 |
from portage.dbapi.virtual import fakedbapi |
236 |
from portage.dep import Atom, use_reduce, paren_enclose |
237 |
from portage.exception import AlarmSignal, InvalidData, InvalidPackageName, \ |
238 |
@@ -364,6 +365,7 @@ class binarytree: |
239 |
self.move_slot_ent = self.dbapi.move_slot_ent |
240 |
self.populated = 0 |
241 |
self.tree = {} |
242 |
+ self._binrepos_conf = None |
243 |
self._remote_has_index = False |
244 |
self._remotepkgs = None # remote metadata indexed by cpv |
245 |
self._additional_pkgs = {} |
246 |
@@ -628,8 +630,10 @@ class binarytree: |
247 |
self._populate_additional(add_repos) |
248 |
|
249 |
if getbinpkgs: |
250 |
- if not self.settings.get("PORTAGE_BINHOST"): |
251 |
- writemsg(_("!!! PORTAGE_BINHOST unset, but use is requested.\n"), |
252 |
+ config_path = os.path.join(self.settings['PORTAGE_CONFIGROOT'], BINREPOS_CONF_FILE) |
253 |
+ self._binrepos_conf = BinRepoConfigLoader((config_path,), self.settings) |
254 |
+ if not self._binrepos_conf: |
255 |
+ writemsg(_("!!! %s is missing (or PORTAGE_BINHOST is unset), but use is requested.\n") % (config_path,), |
256 |
noiselevel=-1) |
257 |
else: |
258 |
self._populate_remote(getbinpkg_refresh=getbinpkg_refresh) |
259 |
@@ -903,7 +907,9 @@ class binarytree: |
260 |
|
261 |
self._remote_has_index = False |
262 |
self._remotepkgs = {} |
263 |
- for base_url in self.settings["PORTAGE_BINHOST"].split(): |
264 |
+ # Order by descending priority. |
265 |
+ for repo in reversed(list(self._binrepos_conf.values())): |
266 |
+ base_url = repo.sync_uri |
267 |
parsed_url = urlparse(base_url) |
268 |
host = parsed_url.netloc |
269 |
port = parsed_url.port |
270 |
|
271 |
diff --git a/lib/portage/tests/emerge/test_simple.py b/lib/portage/tests/emerge/test_simple.py |
272 |
index c24f5c603..8635b70e4 100644 |
273 |
--- a/lib/portage/tests/emerge/test_simple.py |
274 |
+++ b/lib/portage/tests/emerge/test_simple.py |
275 |
@@ -7,7 +7,7 @@ import sys |
276 |
import portage |
277 |
from portage import shutil, os |
278 |
from portage import _unicode_decode |
279 |
-from portage.const import (BASH_BINARY, PORTAGE_PYM_PATH, USER_CONFIG_PATH) |
280 |
+from portage.const import (BASH_BINARY, BINREPOS_CONF_FILE, PORTAGE_PYM_PATH, USER_CONFIG_PATH) |
281 |
from portage.cache.mappings import Mapping |
282 |
from portage.process import find_binary |
283 |
from portage.tests import TestCase |
284 |
@@ -419,13 +419,23 @@ call_has_and_best_version() { |
285 |
) |
286 |
|
287 |
# Test binhost support if FETCHCOMMAND is available. |
288 |
+ binrepos_conf_file = os.path.join(os.sep, eprefix, BINREPOS_CONF_FILE) |
289 |
+ with open(binrepos_conf_file, 'wt') as f: |
290 |
+ f.write('[test-binhost]\n') |
291 |
+ f.write('sync-uri = {}\n'.format(binhost_uri)) |
292 |
fetchcommand = portage.util.shlex_split(playground.settings['FETCHCOMMAND']) |
293 |
fetch_bin = portage.process.find_binary(fetchcommand[0]) |
294 |
if fetch_bin is not None: |
295 |
test_commands = test_commands + ( |
296 |
+ lambda: os.rename(pkgdir, binhost_dir), |
297 |
+ emerge_cmd + ("-e", "--getbinpkgonly", "dev-libs/A"), |
298 |
+ lambda: shutil.rmtree(pkgdir), |
299 |
+ lambda: os.rename(binhost_dir, pkgdir), |
300 |
+ # Remove binrepos.conf and test PORTAGE_BINHOST. |
301 |
+ lambda: os.unlink(binrepos_conf_file), |
302 |
lambda: os.rename(pkgdir, binhost_dir), |
303 |
({"PORTAGE_BINHOST": binhost_uri},) + \ |
304 |
- emerge_cmd + ("-e", "--getbinpkgonly", "dev-libs/A"), |
305 |
+ emerge_cmd + ("-fe", "--getbinpkgonly", "dev-libs/A"), |
306 |
lambda: shutil.rmtree(pkgdir), |
307 |
lambda: os.rename(binhost_dir, pkgdir), |
308 |
) |
309 |
|
310 |
diff --git a/man/make.conf.5 b/man/make.conf.5 |
311 |
index 8a1ea0603..403465fad 100644 |
312 |
--- a/man/make.conf.5 |
313 |
+++ b/man/make.conf.5 |
314 |
@@ -855,7 +855,8 @@ Each entry in the list must specify the full address of a directory |
315 |
serving tbz2's for your system (this directory must contain a 'Packages' index |
316 |
file). This is only used when running with |
317 |
the get binary pkg options are given to \fBemerge\fR. Review \fBemerge\fR(1) |
318 |
-for more information. |
319 |
+for more information. The \fBPORTAGE_BINHOST\fR variable is deprecated in |
320 |
+favor of the \fBbinrepos.conf\fR configuration file (see \fBportage\fR(5)). |
321 |
.TP |
322 |
\fBPORTAGE_BINHOST_HEADER_URI\fR = \ |
323 |
\fI"ftp://login:pass@××××××××××.site/pub/grp/i686/athlon\-xp/"\fR |
324 |
|
325 |
diff --git a/man/portage.5 b/man/portage.5 |
326 |
index 4cffb194a..4f183654c 100644 |
327 |
--- a/man/portage.5 |
328 |
+++ b/man/portage.5 |
329 |
@@ -47,6 +47,7 @@ virtuals |
330 |
.BR /etc/portage/ |
331 |
.nf |
332 |
bashrc |
333 |
+binrepos.conf |
334 |
categories |
335 |
color.map |
336 |
license_groups |
337 |
@@ -620,6 +621,43 @@ any other bash script. |
338 |
|
339 |
Additional package-specific bashrc files can be created in /etc/portage/env. |
340 |
.TP |
341 |
+.BR binrepos.conf |
342 |
+Specifies remote binary package repository configuration information. This |
343 |
+is intended to be used as a replacement for the \fBmake.conf\fR(5) |
344 |
+\fBPORTAGE_BINHOST\fR variable. |
345 |
+ |
346 |
+.I Format: |
347 |
+.nf |
348 |
+\- comments begin with # (no inline comments) |
349 |
+\- configuration of each repository is specified in a section starting with \ |
350 |
+"[${repository_name}]" |
351 |
+\- attributes are specified in "${attribute} = ${value}" format |
352 |
+.fi |
353 |
+ |
354 |
+.RS |
355 |
+.I Attributes supported in sections of repositories: |
356 |
+.RS |
357 |
+.TP |
358 |
+.B priority |
359 |
+Specifies priority of given repository. When a package exists in multiple |
360 |
+repositories, those with higher priority are preferred. |
361 |
+.TP |
362 |
+.B sync\-uri |
363 |
+Specifies URI of repository used for `emerge \-\-getbinpkg`. |
364 |
+.RE |
365 |
+.RE |
366 |
+ |
367 |
+.I Example: |
368 |
+.nf |
369 |
+[example-binhost] |
370 |
+# repos with higher priorities are preferred when packages with equal |
371 |
+# versions are found in multiple repos |
372 |
+priority = 9999 |
373 |
+# packages are fetched from here |
374 |
+sync-uri = https://example.com/binhost |
375 |
+ |
376 |
+.fi |
377 |
+.TP |
378 |
.BR categories |
379 |
A simple list of valid categories that may be used in repositories and PKGDIR |
380 |
(see \fBmake.conf\fR(5)). This allows for custom categories to be created. |