1 |
Add support for a new @profile set which allows the profile to pull |
2 |
in additional packages that do not belong to the @system set. |
3 |
|
4 |
The motivation to have @profile separate from @system is that |
5 |
@system packages may have incomplete dependency specifications |
6 |
(due to long-standing Gentoo policy), and incomplete dependency |
7 |
specifications have deleterious effects on the ability of emerge |
8 |
--jobs to parallelize builds. So, unlike @system, packages added to |
9 |
@profile do not hurt emerge --jobs parallelization. |
10 |
|
11 |
Packages are added to the @profile set in the same way that they are |
12 |
added to the @system set, except that atoms in the @profile set are |
13 |
not preceded with a '*' character. Also, the @profile package set |
14 |
is only supported when 'profile-set' is listed in the layout.conf |
15 |
profile-formats field of the containing repository. |
16 |
|
17 |
X-Gentoo-Bug: 532224 |
18 |
X-Gentoo-Bug-URL: https://bugs.gentoo.org/show_bug.cgi?id=532224 |
19 |
--- |
20 |
cnf/sets/portage.conf | 5 +- |
21 |
doc/config/sets.docbook | 19 +++- |
22 |
man/emerge.1 | 14 ++- |
23 |
man/portage.5 | 33 ++++-- |
24 |
pym/_emerge/actions.py | 14 ++- |
25 |
pym/portage/_sets/ProfilePackageSet.py | 33 ++++++ |
26 |
pym/portage/_sets/__init__.py | 2 +- |
27 |
pym/portage/package/ebuild/config.py | 4 +- |
28 |
pym/portage/repository/config.py | 2 +- |
29 |
.../tests/resolver/test_profile_package_set.py | 123 +++++++++++++++++++++ |
30 |
10 files changed, 225 insertions(+), 24 deletions(-) |
31 |
create mode 100644 pym/portage/_sets/ProfilePackageSet.py |
32 |
create mode 100644 pym/portage/tests/resolver/test_profile_package_set.py |
33 |
|
34 |
diff --git a/cnf/sets/portage.conf b/cnf/sets/portage.conf |
35 |
index 8ffcedc..cff7488 100644 |
36 |
--- a/cnf/sets/portage.conf |
37 |
+++ b/cnf/sets/portage.conf |
38 |
@@ -7,7 +7,10 @@ |
39 |
# Not much that could be changed for world, so better leave it alone |
40 |
[world] |
41 |
class = portage.sets.base.DummyPackageSet |
42 |
-packages = @selected @system |
43 |
+packages = @profile @selected @system |
44 |
+ |
45 |
+[profile] |
46 |
+class = portage.sets.ProfilePackageSet.ProfilePackageSet |
47 |
|
48 |
[selected] |
49 |
class = portage.sets.files.WorldSelectedSet |
50 |
diff --git a/doc/config/sets.docbook b/doc/config/sets.docbook |
51 |
index d3aa147..a557113 100644 |
52 |
--- a/doc/config/sets.docbook |
53 |
+++ b/doc/config/sets.docbook |
54 |
@@ -296,7 +296,23 @@ |
55 |
</para> |
56 |
</sect3> |
57 |
</sect2> |
58 |
- |
59 |
+ |
60 |
+ <sect2 id='config-set-classes-ProfilePackageSet'> |
61 |
+ <title>portage.sets.ProfilePackageSet.ProfilePackageSet</title> |
62 |
+ <para> |
63 |
+ This class implements the <parameter>profile</parameter> set, based on the |
64 |
+ <filename>packages</filename> files in the profile. |
65 |
+ There is no reason to use this in a user configuration as it is already |
66 |
+ confgured by default and doesn't support any options. |
67 |
+ </para> |
68 |
+ <sect3> |
69 |
+ <title>Single Set Configuration</title> |
70 |
+ <para> |
71 |
+ This class doesn't support any extra options. |
72 |
+ </para> |
73 |
+ </sect3> |
74 |
+ </sect2> |
75 |
+ |
76 |
<sect2 id='config-set-classes-SecuritySet' xreflabel='SecuritySet'> |
77 |
<title>portage.sets.security.SecuritySet</title> |
78 |
<para> |
79 |
@@ -601,6 +617,7 @@ |
80 |
</para> |
81 |
<itemizedlist> |
82 |
<listitem><para><varname>world</varname>: uses <classname>DummySet</classname></para></listitem> |
83 |
+ <listitem><para><varname>profile</varname>: uses <classname>ProfilePackageSet</classname></para></listitem> |
84 |
<listitem><para><varname>selected</varname>: uses <classname>WorldSelectedSet</classname></para></listitem> |
85 |
<listitem><para><varname>system</varname>: uses <classname>PackagesSystemSet</classname></para></listitem> |
86 |
<listitem><para><varname>security</varname>: uses <classname>NewAffectedSet</classname> with default options</para></listitem> |
87 |
diff --git a/man/emerge.1 b/man/emerge.1 |
88 |
index f64fd1b..b36f59c 100644 |
89 |
--- a/man/emerge.1 |
90 |
+++ b/man/emerge.1 |
91 |
@@ -61,15 +61,17 @@ would like to query the owners of one or more files or directories. |
92 |
.TP |
93 |
.BR set |
94 |
A \fIset\fR is a convenient shorthand for a large group of |
95 |
-packages. Five sets are currently always available: \fBselected-packages\fR, |
96 |
-\fBselected-sets\fR, \fBselected\fR, \fBsystem\fR and \fBworld\fR. |
97 |
+packages. Six sets are currently always available: \fBselected-packages\fR, |
98 |
+\fBselected-sets\fR, \fBselected\fR, \fBsystem\fR, \fBprofile\fR, and \fBworld\fR. |
99 |
\fBselected-packages\fR contains the user-selected "world" packages that |
100 |
are listed in \fB/var/lib/portage/world\fR, while \fBselected-sets\fR |
101 |
contains the nested sets that may be listed in \fB/var/lib/portage/world_sets\fR. |
102 |
-\fBsystem\fR refers to a set of packages deemed necessary for your system |
103 |
-to run properly. \fBselected\fR encompasses both the \fBselected-packages\fR |
104 |
-and \fBselected-sets\fR sets, while \fBworld\fR encompasses the \fBselected\fR |
105 |
-and \fBsystem\fR sets. [See |
106 |
+\fBsystem\fR and \fBprofile\fR both refer to sets of packages deemed |
107 |
+necessary for your system to run properly (the differences between these |
108 |
+two sets are documented in \fBportage\fR(5)). |
109 |
+\fBselected\fR encompasses both the \fBselected-packages\fR |
110 |
+and \fBselected-sets\fR sets, while \fBworld\fR encompasses the \fBselected\fR, |
111 |
+\fBsystem\fR and \fBprofile\fR sets. [See |
112 |
\fBFILES\fR below for more information.] Other sets can exist depending |
113 |
on the current configuration. The default set configuration is located |
114 |
in the \fB/usr/share/portage/config/sets\fR directory. |
115 |
diff --git a/man/portage.5 b/man/portage.5 |
116 |
index 2fa699c..88cf3bb 100644 |
117 |
--- a/man/portage.5 |
118 |
+++ b/man/portage.5 |
119 |
@@ -1,4 +1,4 @@ |
120 |
-.TH "PORTAGE" "5" "Feb 2014" "Portage VERSION" "Portage" |
121 |
+.TH "PORTAGE" "5" "December 2014" "Portage VERSION" "Portage" |
122 |
.SH NAME |
123 |
portage \- the heart of Gentoo |
124 |
.SH "DESCRIPTION" |
125 |
@@ -331,14 +331,25 @@ Special USE flags which may be needed when bootstrapping from stage1 to stage2. |
126 |
.PD 1 |
127 |
.TP |
128 |
.BR packages |
129 |
-Provides the list of packages that compose the special \fIsystem\fR set. |
130 |
+Provides the list of packages that compose the \fI@system\fR and |
131 |
+\fI@profile\fR package sets. The motivation to have \fI@profile\fR |
132 |
+separate from \fI@system\fR is that \fI@system\fR packages may have |
133 |
+incomplete dependency specifications (due to long-standing Gentoo |
134 |
+policy), and incomplete dependency specifications have deleterious |
135 |
+effects on the ability of \fBemerge\fR to parallelize builds. So, |
136 |
+unlike \fI@system\fR, packages included in \fI@profile\fR do not |
137 |
+hurt \fBemerge\fR's ability to parallelize. |
138 |
|
139 |
.I Format: |
140 |
.nf |
141 |
\- comments begin with # (no inline comments) |
142 |
\- one DEPEND atom per line |
143 |
-\- packages to be added to the system set begin with a * |
144 |
-\- atoms without * only appear for legacy reasons |
145 |
+\- packages to be added to the @system set begin with a * |
146 |
+\- packages to be added to the @profile set do not begin with a * |
147 |
+\- packages may only be added to the @profile set if the containing |
148 |
+ repository's layout.conf has 'profile-set' listed in the |
149 |
+ profile-formats field. Otherwise, packages that do not begin with |
150 |
+ '*' will simply be ignored for legacy reasons |
151 |
.fi |
152 |
.I Note: |
153 |
In a cascading profile setup, you can remove packages in children |
154 |
@@ -348,12 +359,14 @@ a '\-'. |
155 |
.I Example: |
156 |
.nf |
157 |
# i am a comment ! |
158 |
-# pull in a version of glibc less than 2.3 |
159 |
+# pull a version of glibc less than 2.3 into @system |
160 |
*<sys\-libs/glibc\-2.3 |
161 |
-# pull in any version of bash |
162 |
+# pull any version of bash into @system |
163 |
*app\-shells/bash |
164 |
-# pull in a version of readline earlier than 4.2 |
165 |
+# pull a version of readline earlier than 4.2 into @system |
166 |
*<sys\-libs/readline\-4.2 |
167 |
+# pull vim into @profile |
168 |
+app-editors/vim |
169 |
.fi |
170 |
.TP |
171 |
.BR packages.build |
172 |
@@ -1101,13 +1114,15 @@ The default setting for repoman's --echangelog option. |
173 |
The cache formats supported in the metadata tree. There is the old "pms" format |
174 |
and the newer/faster "md5-dict" format. Default is to detect dirs. |
175 |
.TP |
176 |
-.BR profile\-formats " = [pms|portage-1|portage-2|profile-bashrcs]" |
177 |
+.BR profile\-formats " = [pms|portage-1|portage-2|profile-bashrcs|profile-set]" |
178 |
Control functionality available to profiles in this repo such as which files |
179 |
may be dirs, or the syntax available in parent files. Use "portage-2" if you're |
180 |
unsure. The default is "portage-1-compat" mode which is meant to be compatible |
181 |
with old profiles, but is not allowed to be opted into directly. |
182 |
Setting profile-bashrcs will enable the per-profile bashrc mechanism |
183 |
-\fBpackage.bashrc\fR. |
184 |
+\fBpackage.bashrc\fR. Setting profile-set enables support for using the |
185 |
+profile \fBpackages\fR file to add atoms to the @profile package set. |
186 |
+See the profile \fBpackages\fR section for more information. |
187 |
.RE |
188 |
.RE |
189 |
|
190 |
diff --git a/pym/_emerge/actions.py b/pym/_emerge/actions.py |
191 |
index c7246a9..0ae2c16 100644 |
192 |
--- a/pym/_emerge/actions.py |
193 |
+++ b/pym/_emerge/actions.py |
194 |
@@ -650,7 +650,7 @@ def action_depclean(settings, trees, ldpath_mtimes, |
195 |
return rval |
196 |
|
197 |
set_atoms = {} |
198 |
- for k in ("system", "selected"): |
199 |
+ for k in ("profile", "system", "selected"): |
200 |
try: |
201 |
set_atoms[k] = root_config.setconfig.getSetAtoms(k) |
202 |
except portage.exception.PackageSetNotFound: |
203 |
@@ -660,6 +660,8 @@ def action_depclean(settings, trees, ldpath_mtimes, |
204 |
print("Packages installed: " + str(len(vardb.cpv_all()))) |
205 |
print("Packages in world: %d" % len(set_atoms["selected"])) |
206 |
print("Packages in system: %d" % len(set_atoms["system"])) |
207 |
+ if set_atoms["profile"]: |
208 |
+ print("Packages in profile: %d" % len(set_atoms["profile"])) |
209 |
print("Required packages: "+str(req_pkg_count)) |
210 |
if "--pretend" in myopts: |
211 |
print("Number to remove: "+str(len(cleanlist))) |
212 |
@@ -693,20 +695,24 @@ def calc_depclean(settings, trees, ldpath_mtimes, |
213 |
system_set = psets["system"] |
214 |
|
215 |
set_atoms = {} |
216 |
- for k in ("system", "selected"): |
217 |
+ for k in ("profile", "system", "selected"): |
218 |
try: |
219 |
set_atoms[k] = root_config.setconfig.getSetAtoms(k) |
220 |
except portage.exception.PackageSetNotFound: |
221 |
# A nested set could not be resolved, so ignore nested sets. |
222 |
set_atoms[k] = root_config.sets[k].getAtoms() |
223 |
|
224 |
- if not set_atoms["system"] or not set_atoms["selected"]: |
225 |
+ if (not set_atoms["system"] or |
226 |
+ not (set_atoms["selected"] or set_atoms["profile"])): |
227 |
|
228 |
if not set_atoms["system"]: |
229 |
writemsg_level("!!! You have no system list.\n", |
230 |
level=logging.ERROR, noiselevel=-1) |
231 |
|
232 |
- if not set_atoms["selected"]: |
233 |
+ # Skip this warning if @profile is non-empty, in order to |
234 |
+ # support using @profile as an alternative to @selected |
235 |
+ # for building a stage 4. |
236 |
+ if not (set_atoms["selected"] or set_atoms["profile"]): |
237 |
writemsg_level("!!! You have no world file.\n", |
238 |
level=logging.WARNING, noiselevel=-1) |
239 |
|
240 |
diff --git a/pym/portage/_sets/ProfilePackageSet.py b/pym/portage/_sets/ProfilePackageSet.py |
241 |
new file mode 100644 |
242 |
index 0000000..c2f5fee |
243 |
--- /dev/null |
244 |
+++ b/pym/portage/_sets/ProfilePackageSet.py |
245 |
@@ -0,0 +1,33 @@ |
246 |
+# Copyright 2014 Gentoo Foundation |
247 |
+# Distributed under the terms of the GNU General Public License v2 |
248 |
+ |
249 |
+from portage import os |
250 |
+from portage.util import grabfile_package, stack_lists |
251 |
+from portage._sets.base import PackageSet |
252 |
+ |
253 |
+class ProfilePackageSet(PackageSet): |
254 |
+ _operations = ["merge"] |
255 |
+ |
256 |
+ def __init__(self, profiles, debug=False): |
257 |
+ super(ProfilePackageSet, self).__init__() |
258 |
+ self._profiles = profiles |
259 |
+ if profiles: |
260 |
+ desc_profile = profiles[-1] |
261 |
+ if desc_profile.user_config and len(profiles) > 1: |
262 |
+ desc_profile = profiles[-2] |
263 |
+ description = desc_profile.location |
264 |
+ else: |
265 |
+ description = None |
266 |
+ self.description = "Profile packages for profile %s" % description |
267 |
+ |
268 |
+ def load(self): |
269 |
+ self._setAtoms(x for x in stack_lists( |
270 |
+ [grabfile_package(os.path.join(y.location, "packages"), |
271 |
+ verify_eapi=True) for y in self._profiles |
272 |
+ if "profile-set" in y.profile_formats], |
273 |
+ incremental=1) if x[:1] != "*") |
274 |
+ |
275 |
+ def singleBuilder(self, options, settings, trees): |
276 |
+ return ProfilePackageSet( |
277 |
+ settings._locations_manager.profiles_complex) |
278 |
+ singleBuilder = classmethod(singleBuilder) |
279 |
diff --git a/pym/portage/_sets/__init__.py b/pym/portage/_sets/__init__.py |
280 |
index a652227..d53387a 100644 |
281 |
--- a/pym/portage/_sets/__init__.py |
282 |
+++ b/pym/portage/_sets/__init__.py |
283 |
@@ -115,7 +115,7 @@ class SetConfig(object): |
284 |
parser.remove_section("world") |
285 |
parser.add_section("world") |
286 |
parser.set("world", "class", "portage.sets.base.DummyPackageSet") |
287 |
- parser.set("world", "packages", "@selected @system") |
288 |
+ parser.set("world", "packages", "@profile @selected @system") |
289 |
|
290 |
parser.remove_section("selected") |
291 |
parser.add_section("selected") |
292 |
diff --git a/pym/portage/package/ebuild/config.py b/pym/portage/package/ebuild/config.py |
293 |
index 59e239b..1a0377e 100644 |
294 |
--- a/pym/portage/package/ebuild/config.py |
295 |
+++ b/pym/portage/package/ebuild/config.py |
296 |
@@ -512,7 +512,6 @@ class config(object): |
297 |
if v is not None: |
298 |
portdir_sync = v |
299 |
|
300 |
- known_repos = frozenset(known_repos) |
301 |
self["PORTDIR"] = portdir |
302 |
self["PORTDIR_OVERLAY"] = portdir_overlay |
303 |
if portdir_sync: |
304 |
@@ -523,6 +522,9 @@ class config(object): |
305 |
else: |
306 |
self.repositories = repositories |
307 |
|
308 |
+ known_repos.extend(repo.location for repo in self.repositories) |
309 |
+ known_repos = frozenset(known_repos) |
310 |
+ |
311 |
self['PORTAGE_REPOSITORIES'] = self.repositories.config_string() |
312 |
self.backup_changes('PORTAGE_REPOSITORIES') |
313 |
|
314 |
diff --git a/pym/portage/repository/config.py b/pym/portage/repository/config.py |
315 |
index f45684b..9096d73 100644 |
316 |
--- a/pym/portage/repository/config.py |
317 |
+++ b/pym/portage/repository/config.py |
318 |
@@ -41,7 +41,7 @@ if sys.hexversion >= 0x3000000: |
319 |
_invalid_path_char_re = re.compile(r'[^a-zA-Z0-9._\-+:/]') |
320 |
|
321 |
_valid_profile_formats = frozenset( |
322 |
- ['pms', 'portage-1', 'portage-2', 'profile-bashrcs']) |
323 |
+ ['pms', 'portage-1', 'portage-2', 'profile-bashrcs', 'profile-set']) |
324 |
|
325 |
_portage1_profiles_allow_directories = frozenset( |
326 |
["portage-1-compat", "portage-1", 'portage-2']) |
327 |
diff --git a/pym/portage/tests/resolver/test_profile_package_set.py b/pym/portage/tests/resolver/test_profile_package_set.py |
328 |
new file mode 100644 |
329 |
index 0000000..88a2a82 |
330 |
--- /dev/null |
331 |
+++ b/pym/portage/tests/resolver/test_profile_package_set.py |
332 |
@@ -0,0 +1,123 @@ |
333 |
+# Copyright 2014 Gentoo Foundation |
334 |
+# Distributed under the terms of the GNU General Public License v2 |
335 |
+ |
336 |
+from __future__ import unicode_literals |
337 |
+ |
338 |
+import io |
339 |
+ |
340 |
+from portage import os, _encodings |
341 |
+from portage.tests import TestCase |
342 |
+from portage.tests.resolver.ResolverPlayground import ( |
343 |
+ ResolverPlayground, ResolverPlaygroundTestCase) |
344 |
+from portage.util import ensure_dirs |
345 |
+ |
346 |
+class ProfilePackageSetTestCase(TestCase): |
347 |
+ |
348 |
+ def testProfilePackageSet(self): |
349 |
+ |
350 |
+ repo_configs = { |
351 |
+ "test_repo": { |
352 |
+ "layout.conf": ("profile-formats = profile-set",), |
353 |
+ } |
354 |
+ } |
355 |
+ |
356 |
+ profiles = ( |
357 |
+ ( |
358 |
+ 'default/linux', |
359 |
+ { |
360 |
+ "eapi": ("5",), |
361 |
+ "packages": ( |
362 |
+ "*sys-libs/A", |
363 |
+ "app-misc/A", |
364 |
+ "app-misc/B", |
365 |
+ "app-misc/C", |
366 |
+ ), |
367 |
+ } |
368 |
+ ), |
369 |
+ ( |
370 |
+ 'default/linux/x86', |
371 |
+ { |
372 |
+ "eapi": ("5",), |
373 |
+ "packages": ( |
374 |
+ "-app-misc/B", |
375 |
+ ), |
376 |
+ "parent": ("..",) |
377 |
+ } |
378 |
+ ), |
379 |
+ ) |
380 |
+ |
381 |
+ ebuilds = { |
382 |
+ "sys-libs/A-1": { |
383 |
+ "EAPI": "5", |
384 |
+ }, |
385 |
+ "app-misc/A-1": { |
386 |
+ "EAPI": "5", |
387 |
+ }, |
388 |
+ "app-misc/B-1": { |
389 |
+ "EAPI": "5", |
390 |
+ }, |
391 |
+ "app-misc/C-1": { |
392 |
+ "EAPI": "5", |
393 |
+ }, |
394 |
+ } |
395 |
+ |
396 |
+ installed = { |
397 |
+ "sys-libs/A-1": { |
398 |
+ "EAPI": "5", |
399 |
+ }, |
400 |
+ "app-misc/A-1": { |
401 |
+ "EAPI": "5", |
402 |
+ }, |
403 |
+ "app-misc/B-1": { |
404 |
+ "EAPI": "5", |
405 |
+ }, |
406 |
+ "app-misc/C-1": { |
407 |
+ "EAPI": "5", |
408 |
+ }, |
409 |
+ } |
410 |
+ |
411 |
+ test_cases = ( |
412 |
+ |
413 |
+ ResolverPlaygroundTestCase( |
414 |
+ ["@world"], |
415 |
+ options={"--update": True, "--deep": True}, |
416 |
+ mergelist = [], |
417 |
+ success = True, |
418 |
+ ), |
419 |
+ |
420 |
+ ResolverPlaygroundTestCase( |
421 |
+ [], |
422 |
+ options={"--depclean": True}, |
423 |
+ success=True, |
424 |
+ cleanlist=["app-misc/B-1"] |
425 |
+ ), |
426 |
+ |
427 |
+ ) |
428 |
+ |
429 |
+ playground = ResolverPlayground(debug=False, ebuilds=ebuilds, |
430 |
+ installed=installed, repo_configs=repo_configs) |
431 |
+ try: |
432 |
+ repo_dir = (playground.settings.repositories. |
433 |
+ get_location_for_name("test_repo")) |
434 |
+ profile_root = os.path.join(repo_dir, "profiles") |
435 |
+ |
436 |
+ for p, data in profiles: |
437 |
+ prof_path = os.path.join(profile_root, p) |
438 |
+ ensure_dirs(prof_path) |
439 |
+ for k, v in data.items(): |
440 |
+ with io.open(os.path.join(prof_path, k), mode="w", |
441 |
+ encoding=_encodings["repo.content"]) as f: |
442 |
+ for line in v: |
443 |
+ f.write("%s\n" % line) |
444 |
+ |
445 |
+ # The config must be reloaded in order to account |
446 |
+ # for the above profile customizations. |
447 |
+ playground.reload_config() |
448 |
+ |
449 |
+ for test_case in test_cases: |
450 |
+ playground.run_TestCase(test_case) |
451 |
+ self.assertEqual(test_case.test_success, True, |
452 |
+ test_case.fail_msg) |
453 |
+ |
454 |
+ finally: |
455 |
+ playground.cleanup() |
456 |
-- |
457 |
2.0.4 |