Gentoo Archives: gentoo-commits

From: Brian Dolbec <dolsen@g.o>
To: gentoo-commits@l.g.o
Subject: [gentoo-commits] proj/portage:repoman commit in: pym/_emerge/, man/, pym/portage/tests/resolver/
Date: Tue, 27 Jun 2017 20:06:13
Message-Id: 1494785505.40505ceeadc769f4f01c66e52a19ce0bf2f59761.dolsen@gentoo
1 commit: 40505ceeadc769f4f01c66e52a19ce0bf2f59761
2 Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
3 AuthorDate: Wed May 10 03:44:56 2017 +0000
4 Commit: Brian Dolbec <dolsen <AT> gentoo <DOT> org>
5 CommitDate: Sun May 14 18:11:45 2017 +0000
6 URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=40505cee
7
8 emerge: terminate backtracking early for autounmask changes (bug 615680)
9
10 Since autounmask changes are a strong indicator that backtracking
11 will ultimately fail to produce a solution, terminate early for
12 autounmask changes, and add a --autounmask-backtrack=<y|n> option
13 to modify this behavior. The --autounmask-continue option implies
14 --autounmask-backtrack=y behavior, for backward compatibility.
15
16 When backtracking terminates early, the following warning message
17 is displayed after the autounmask change(s):
18
19 * In order to avoid wasting time, backtracking has terminated early
20 * due to the above autounmask change(s). The --autounmask-backtrack=y
21 * option can be used to force further backtracking, but there is no
22 * guarantee that it will produce a solution.
23
24 With this change, five of the existing cases fail unless
25 --autounmask-backtrack=y is added to the options. For each of
26 these cases, comments below the test case document how it behaves
27 with and without --autounmask-backtrack=y enabled.
28
29 X-Gentoo-bug: 615680
30 X-Gentoo-bug-url: https://bugs.gentoo.org/show_bug.cgi?id=615680
31 Acked-by: Brian Dolbec <dolsen <AT> gentoo.org>
32
33 man/emerge.1 | 10 ++-
34 pym/_emerge/depgraph.py | 80 ++++++++++++++++++----
35 pym/_emerge/main.py | 6 ++
36 pym/portage/tests/resolver/test_autounmask.py | 57 ++++++++++++++-
37 .../tests/resolver/test_autounmask_use_breakage.py | 40 +++++++++++
38 .../test_slot_conflict_unsatisfied_deep_deps.py | 61 +++++++++++++++++
39 6 files changed, 237 insertions(+), 17 deletions(-)
40
41 diff --git a/man/emerge.1 b/man/emerge.1
42 index f1a9d4f3f..94edc9095 100644
43 --- a/man/emerge.1
44 +++ b/man/emerge.1
45 @@ -363,12 +363,20 @@ the specified configuration file(s), or enable the
46 \fBEMERGE_DEFAULT_OPTS\fR variable may be used to
47 disable this option by default in \fBmake.conf\fR(5).
48 .TP
49 +.BR "\-\-autounmask\-backtrack < y | n >"
50 +Allow backtracking after autounmask has detected that
51 +configuration changes are necessary. This option is not
52 +recommended, since it can cause a large amount of time to
53 +be wasted by backtracking calculations, even though there
54 +is no guarantee that it will produce a solution. This
55 +option is disabled by default.
56 +.TP
57 .BR "\-\-autounmask\-continue [ y | n ]"
58 Automatically apply autounmask changes to configuration
59 files, and continue to execute the specified command. If
60 the dependency calculation is not entirely successful, then
61 emerge will simply abort without modifying any configuration
62 -files.
63 +files. This option implies \fB\-\-autounmask\-backtrack=y\fR.
64 \fBWARNING:\fR
65 This option is intended to be used only with great caution,
66 since it is possible for it to make nonsensical configuration
67
68 diff --git a/pym/_emerge/depgraph.py b/pym/_emerge/depgraph.py
69 index e1119af3c..53910dd25 100644
70 --- a/pym/_emerge/depgraph.py
71 +++ b/pym/_emerge/depgraph.py
72 @@ -444,6 +444,7 @@ class _dynamic_depgraph_config(object):
73 self._autounmask = depgraph._frozen_config.myopts.get('--autounmask') != 'n'
74 self._displayed_autounmask = False
75 self._success_without_autounmask = False
76 + self._autounmask_backtrack_disabled = False
77 self._required_use_unsatisfied = False
78 self._traverse_ignored_deps = False
79 self._complete_mode = False
80 @@ -1129,7 +1130,8 @@ class depgraph(object):
81
82 self._show_merge_list()
83
84 - self._dynamic_config._slot_conflict_handler = slot_conflict_handler(self)
85 + if self._dynamic_config._slot_conflict_handler is None:
86 + self._dynamic_config._slot_conflict_handler = slot_conflict_handler(self)
87 handler = self._dynamic_config._slot_conflict_handler
88
89 conflict = handler.get_conflict()
90 @@ -4243,17 +4245,7 @@ class depgraph(object):
91 # set below is reserved for cases where there are *zero* other
92 # problems. For reference, see backtrack_depgraph, where it skips the
93 # get_best_run() call when success_without_autounmask is True.
94 -
95 - digraph_nodes = self._dynamic_config.digraph.nodes
96 -
97 - if any(x in digraph_nodes for x in
98 - self._dynamic_config._needed_unstable_keywords) or \
99 - any(x in digraph_nodes for x in
100 - self._dynamic_config._needed_p_mask_changes) or \
101 - any(x in digraph_nodes for x in
102 - self._dynamic_config._needed_use_config_changes) or \
103 - any(x in digraph_nodes for x in
104 - self._dynamic_config._needed_license_changes) :
105 + if self._have_autounmask_changes():
106 #We failed if the user needs to change the configuration
107 self._dynamic_config._success_without_autounmask = True
108 if (self._frozen_config.myopts.get("--autounmask-continue") is True and
109 @@ -8564,6 +8556,17 @@ class depgraph(object):
110 "experimental or unstable packages.\n",
111 noiselevel=-1)
112
113 + if self._dynamic_config._autounmask_backtrack_disabled:
114 + msg = [
115 + "In order to avoid wasting time, backtracking has terminated early",
116 + "due to the above autounmask change(s). The --autounmask-backtrack=y",
117 + "option can be used to force further backtracking, but there is no",
118 + "guarantee that it will produce a solution.",
119 + ]
120 + writemsg("\n", noiselevel=-1)
121 + for line in msg:
122 + writemsg(" %s %s\n" % (colorize("WARN", "*"), line),
123 + noiselevel=-1)
124
125 def display_problems(self):
126 """
127 @@ -9072,8 +9075,57 @@ class depgraph(object):
128 not self._dynamic_config._skip_restart
129
130 def need_config_change(self):
131 - return self._dynamic_config._success_without_autounmask or \
132 - self._dynamic_config._required_use_unsatisfied
133 + """
134 + Returns true if backtracking should terminate due to a needed
135 + configuration change.
136 + """
137 + if (self._dynamic_config._success_without_autounmask or
138 + self._dynamic_config._required_use_unsatisfied):
139 + return True
140 +
141 + if (self._dynamic_config._slot_conflict_handler is None and
142 + not self._accept_blocker_conflicts() and
143 + any(self._dynamic_config._package_tracker.slot_conflicts())):
144 + self._dynamic_config._slot_conflict_handler = slot_conflict_handler(self)
145 + if self._dynamic_config._slot_conflict_handler.changes:
146 + # Terminate backtracking early if the slot conflict
147 + # handler finds some changes to suggest. The case involving
148 + # sci-libs/L and sci-libs/M in SlotCollisionTestCase will
149 + # otherwise fail with --autounmask-backtrack=n, since
150 + # backtracking will eventually lead to some autounmask
151 + # changes. Changes suggested by the slot conflict handler
152 + # are more likely to be useful.
153 + return True
154 +
155 + if (self._dynamic_config._allow_backtracking and
156 + self._frozen_config.myopts.get("--autounmask-backtrack") != 'y' and
157 + self._have_autounmask_changes()):
158 +
159 + if (self._frozen_config.myopts.get("--autounmask-continue") is True and
160 + self._frozen_config.myopts.get("--autounmask-backtrack") != 'n'):
161 + # --autounmask-continue implies --autounmask-backtrack=y behavior,
162 + # for backward compatibility.
163 + return False
164 +
165 + # This disables backtracking when there are autounmask
166 + # config changes. The display_problems method will notify
167 + # the user that --autounmask-backtrack=y can be used to
168 + # force backtracking in this case.
169 + self._dynamic_config._autounmask_backtrack_disabled = True
170 + return True
171 +
172 + return False
173 +
174 + def _have_autounmask_changes(self):
175 + digraph_nodes = self._dynamic_config.digraph.nodes
176 + return (any(x in digraph_nodes for x in
177 + self._dynamic_config._needed_unstable_keywords) or
178 + any(x in digraph_nodes for x in
179 + self._dynamic_config._needed_p_mask_changes) or
180 + any(x in digraph_nodes for x in
181 + self._dynamic_config._needed_use_config_changes) or
182 + any(x in digraph_nodes for x in
183 + self._dynamic_config._needed_license_changes))
184
185 def need_config_reload(self):
186 return self._dynamic_config._need_config_reload
187
188 diff --git a/pym/_emerge/main.py b/pym/_emerge/main.py
189 index 76e963ac9..808496722 100644
190 --- a/pym/_emerge/main.py
191 +++ b/pym/_emerge/main.py
192 @@ -326,6 +326,12 @@ def parse_opts(tmpcmdline, silent=False):
193 "choices" : true_y_or_n
194 },
195
196 + "--autounmask-backtrack": {
197 + "help": ("continue backtracking when there are autounmask "
198 + "configuration changes"),
199 + "choices":("y", "n")
200 + },
201 +
202 "--autounmask-continue": {
203 "help" : "write autounmask changes and continue",
204 "choices" : true_y_or_n
205
206 diff --git a/pym/portage/tests/resolver/test_autounmask.py b/pym/portage/tests/resolver/test_autounmask.py
207 index 75fb36843..e2a7de028 100644
208 --- a/pym/portage/tests/resolver/test_autounmask.py
209 +++ b/pym/portage/tests/resolver/test_autounmask.py
210 @@ -81,20 +81,73 @@ class AutounmaskTestCase(TestCase):
211 #Make sure we restart if needed.
212 ResolverPlaygroundTestCase(
213 ["dev-libs/A:1", "dev-libs/B"],
214 - options={"--autounmask": True},
215 + options={"--autounmask": True, "--autounmask-backtrack": "y"},
216 all_permutations=True,
217 success=False,
218 mergelist=["dev-libs/C-1", "dev-libs/B-1", "dev-libs/A-1"],
219 use_changes={ "dev-libs/B-1": {"foo": True} }),
220 +
221 + # With --autounmask-backtrack=y:
222 + #[ebuild N ] dev-libs/C-1
223 + #[ebuild N ] dev-libs/B-1 USE="foo -bar"
224 + #[ebuild N ] dev-libs/A-1
225 + #
226 + #The following USE changes are necessary to proceed:
227 + # (see "package.use" in the portage(5) man page for more details)
228 + ## required by dev-libs/A-1::test_repo
229 + ## required by dev-libs/A:1 (argument)
230 + #>=dev-libs/B-1 foo
231 +
232 + # Without --autounmask-backtrack=y:
233 + #[ebuild N ] dev-libs/B-1 USE="foo -bar"
234 + #[ebuild N ] dev-libs/A-1
235 + #
236 + #The following USE changes are necessary to proceed:
237 + # (see "package.use" in the portage(5) man page for more details)
238 + ## required by dev-libs/A-1::test_repo
239 + ## required by dev-libs/A:1 (argument)
240 + #>=dev-libs/B-1 foo
241 +
242 ResolverPlaygroundTestCase(
243 ["dev-libs/A:1", "dev-libs/A:2", "dev-libs/B"],
244 - options={"--autounmask": True},
245 + options={"--autounmask": True, "--autounmask-backtrack": "y"},
246 all_permutations=True,
247 success=False,
248 mergelist=["dev-libs/D-1", "dev-libs/C-1", "dev-libs/B-1", "dev-libs/A-1", "dev-libs/A-2"],
249 ignore_mergelist_order=True,
250 use_changes={ "dev-libs/B-1": {"foo": True, "bar": True} }),
251
252 + # With --autounmask-backtrack=y:
253 + #[ebuild N ] dev-libs/C-1
254 + #[ebuild N ] dev-libs/D-1
255 + #[ebuild N ] dev-libs/B-1 USE="bar foo"
256 + #[ebuild N ] dev-libs/A-2
257 + #[ebuild N ] dev-libs/A-1
258 + #
259 + #The following USE changes are necessary to proceed:
260 + # (see "package.use" in the portage(5) man page for more details)
261 + ## required by dev-libs/A-2::test_repo
262 + ## required by dev-libs/A:2 (argument)
263 + #>=dev-libs/B-1 bar foo
264 +
265 + # Without --autounmask-backtrack=y:
266 + #[ebuild N ] dev-libs/B-1 USE="bar foo"
267 + #[ebuild N ] dev-libs/A-1
268 + #[ebuild N ] dev-libs/A-2
269 + #
270 + #The following USE changes are necessary to proceed:
271 + # (see "package.use" in the portage(5) man page for more details)
272 + ## required by dev-libs/A-1::test_repo
273 + ## required by dev-libs/A:1 (argument)
274 + #>=dev-libs/B-1 foo bar
275 +
276 + # NOTE: The --autounmask-backtrack=n behavior is acceptable, but
277 + # it would be nicer if it added the dev-libs/C-1 and dev-libs/D-1
278 + # deps to the depgraph without backtracking. It could add two
279 + # instances of dev-libs/B-1 to the graph with different USE flags,
280 + # and then use _solve_non_slot_operator_slot_conflicts to eliminate
281 + # the redundant instance.
282 +
283 #Test keywording.
284 #The simple case.
285
286
287 diff --git a/pym/portage/tests/resolver/test_autounmask_use_breakage.py b/pym/portage/tests/resolver/test_autounmask_use_breakage.py
288 index 3654aa6a3..173941629 100644
289 --- a/pym/portage/tests/resolver/test_autounmask_use_breakage.py
290 +++ b/pym/portage/tests/resolver/test_autounmask_use_breakage.py
291 @@ -46,12 +46,52 @@ class AutounmaskUseBreakageTestCase(TestCase):
292 # due to autounmask USE breakage.
293 ResolverPlaygroundTestCase(
294 ["app-misc/C", "app-misc/B", "app-misc/A"],
295 + options={"--autounmask-backtrack": "y"},
296 all_permutations = True,
297 success = False,
298 ambiguous_slot_collision_solutions = True,
299 slot_collision_solutions = [None, []]
300 ),
301
302 + # With --autounmask-backtrack=y:
303 + #emerge: there are no ebuilds built with USE flags to satisfy "app-misc/D[foo]".
304 + #!!! One of the following packages is required to complete your request:
305 + #- app-misc/D-0::test_repo (Change USE: +foo)
306 + #(dependency required by "app-misc/B-0::test_repo" [ebuild])
307 + #(dependency required by "app-misc/B" [argument])
308 +
309 + # Without --autounmask-backtrack=y:
310 + #[ebuild N ] app-misc/D-0 USE="foo"
311 + #[ebuild N ] app-misc/D-1 USE="-bar"
312 + #[ebuild N ] app-misc/C-0
313 + #[ebuild N ] app-misc/B-0
314 + #[ebuild N ] app-misc/A-0
315 + #
316 + #!!! Multiple package instances within a single package slot have been pulled
317 + #!!! into the dependency graph, resulting in a slot conflict:
318 + #
319 + #app-misc/D:0
320 + #
321 + # (app-misc/D-0:0/0::test_repo, ebuild scheduled for merge) pulled in by
322 + # app-misc/D[-foo] required by (app-misc/A-0:0/0::test_repo, ebuild scheduled for merge)
323 + # ^^^^
324 + # app-misc/D[foo] required by (app-misc/B-0:0/0::test_repo, ebuild scheduled for merge)
325 + # ^^^
326 + #
327 + # (app-misc/D-1:0/0::test_repo, ebuild scheduled for merge) pulled in by
328 + # >=app-misc/D-1 required by (app-misc/C-0:0/0::test_repo, ebuild scheduled for merge)
329 + # ^^ ^
330 + #
331 + #The following USE changes are necessary to proceed:
332 + # (see "package.use" in the portage(5) man page for more details)
333 + ## required by app-misc/B-0::test_repo
334 + ## required by app-misc/B (argument)
335 + #=app-misc/D-0 foo
336 +
337 + # NOTE: The --autounmask-backtrack=n output is preferable here,
338 + # because it highlights the unsolvable dependency conflict.
339 + # It would be better if it eliminated the autounmask suggestion,
340 + # since that suggestion won't solve the conflict.
341 )
342
343 playground = ResolverPlayground(ebuilds=ebuilds, debug=False)
344
345 diff --git a/pym/portage/tests/resolver/test_slot_conflict_unsatisfied_deep_deps.py b/pym/portage/tests/resolver/test_slot_conflict_unsatisfied_deep_deps.py
346 index 13f7e67e3..846ba0e59 100644
347 --- a/pym/portage/tests/resolver/test_slot_conflict_unsatisfied_deep_deps.py
348 +++ b/pym/portage/tests/resolver/test_slot_conflict_unsatisfied_deep_deps.py
349 @@ -79,6 +79,7 @@ class SlotConflictUnsatisfiedDeepDepsTestCase(TestCase):
350 ["@world"],
351 options={
352 "--autounmask": "y",
353 + "--autounmask-backtrack": "y",
354 "--complete-graph": True,
355 "--selective": True,
356 "--deep": 1
357 @@ -89,11 +90,63 @@ class SlotConflictUnsatisfiedDeepDepsTestCase(TestCase):
358 unsatisfied_deps=["dev-libs/initially-unsatisfied"],
359 success=False),
360
361 + # With --autounmask-backtrack=y:
362 + #[ebuild N ~] dev-libs/A-2
363 + #[ebuild N ] dev-libs/C-1
364 + #[ebuild N ] dev-libs/D-1
365 + #[ebuild N ] dev-libs/B-1
366 + #
367 + #The following keyword changes are necessary to proceed:
368 + # (see "package.accept_keywords" in the portage(5) man page for more details)
369 + ## required by dev-libs/C-1::test_repo
370 + ## required by @selected
371 + ## required by @world (argument)
372 + #=dev-libs/A-2 ~x86
373 + #
374 + #!!! Problems have been detected with your world file
375 + #!!! Please run emaint --check world
376 + #
377 + #
378 + #!!! Ebuilds for the following packages are either all
379 + #!!! masked or don't exist:
380 + #dev-libs/broken
381 + #
382 + #emerge: there are no ebuilds to satisfy "dev-libs/initially-unsatisfied".
383 + #(dependency required by "dev-libs/broken-1::test_repo" [installed])
384 + #(dependency required by "@selected" [set])
385 + #(dependency required by "@world" [argument])
386 +
387 + # Without --autounmask-backtrack=y:
388 + #!!! Multiple package instances within a single package slot have been pulled
389 + #!!! into the dependency graph, resulting in a slot conflict:
390 + #
391 + #dev-libs/A:0
392 + #
393 + # (dev-libs/A-1:0/0::test_repo, ebuild scheduled for merge) pulled in by
394 + # (no parents that aren't satisfied by other packages in this slot)
395 + #
396 + # (dev-libs/A-2:0/0::test_repo, ebuild scheduled for merge) pulled in by
397 + # >=dev-libs/A-2 required by (dev-libs/C-1:0/0::test_repo, ebuild scheduled for merge)
398 + # ^^ ^
399 + #
400 + #The following keyword changes are necessary to proceed:
401 + # (see "package.accept_keywords" in the portage(5) man page for more details)
402 + ## required by dev-libs/C-1::test_repo
403 + ## required by @selected
404 + ## required by @world (argument)
405 + #=dev-libs/A-2 ~x86
406 + #
407 + #emerge: there are no ebuilds to satisfy "dev-libs/initially-unsatisfied".
408 + #(dependency required by "dev-libs/broken-1::test_repo" [installed])
409 + #(dependency required by "@selected" [set])
410 + #(dependency required by "@world" [argument])
411 +
412 # Test --deep = True
413 ResolverPlaygroundTestCase(
414 ["@world"],
415 options={
416 "--autounmask": "y",
417 + "--autounmask-backtrack": "y",
418 "--complete-graph": True,
419 "--selective": True,
420 "--deep": True
421 @@ -103,6 +156,14 @@ class SlotConflictUnsatisfiedDeepDepsTestCase(TestCase):
422 unstable_keywords=["dev-libs/A-2"],
423 unsatisfied_deps=["dev-libs/initially-unsatisfied"],
424 success=False),
425 +
426 + # The effects of --autounmask-backtrack are the same as the previous test case.
427 + # Both test cases can randomly succeed with --autounmask-backtrack=n, when
428 + # "backtracking due to unsatisfied dep" randomly occurs before the autounmask
429 + # unstable keyword change. It would be possible to eliminate backtracking here
430 + # by recognizing that there are no alternatives to satisfy the dev-libs/broken
431 + # atom in the world file. Then the test cases will consistently succeed with
432 + # --autounmask-backtrack=n.
433 )
434
435 playground = ResolverPlayground(ebuilds=ebuilds, installed=installed,