Gentoo Archives: gentoo-commits

From: Magnus Granberg <zorry@g.o>
To: gentoo-commits@l.g.o
Subject: [gentoo-commits] dev/zorry:master commit in: gobs/pym/
Date: Tue, 01 May 2012 00:02:45
Message-Id: 1335830528.5fd52e5cf2f85ec872780338fcd2ad165f3123d9.zorry@gentoo
1 commit: 5fd52e5cf2f85ec872780338fcd2ad165f3123d9
2 Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
3 AuthorDate: Tue May 1 00:02:08 2012 +0000
4 Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
5 CommitDate: Tue May 1 00:02:08 2012 +0000
6 URL: http://git.overlays.gentoo.org/gitweb/?p=dev/zorry.git;a=commit;h=5fd52e5c
7
8 Updated Scheduler.py
9
10 ---
11 gobs/pym/Scheduler.py | 381 +++++++++++++++++++------------------------------
12 1 files changed, 146 insertions(+), 235 deletions(-)
13
14 diff --git a/gobs/pym/Scheduler.py b/gobs/pym/Scheduler.py
15 index 005f861..229c595 100644
16 --- a/gobs/pym/Scheduler.py
17 +++ b/gobs/pym/Scheduler.py
18 @@ -1,4 +1,4 @@
19 -# Copyright 1999-2011 Gentoo Foundation
20 +# Copyright 1999-2012 Gentoo Foundation
21 # Distributed under the terms of the GNU General Public License v2
22
23 from __future__ import print_function
24 @@ -7,10 +7,8 @@ from collections import deque
25 import gc
26 import gzip
27 import logging
28 -import shutil
29 import signal
30 import sys
31 -import tempfile
32 import textwrap
33 import time
34 import warnings
35 @@ -28,9 +26,12 @@ from portage.output import colorize, create_color_func, red
36 bad = create_color_func("BAD")
37 from portage._sets import SETPREFIX
38 from portage._sets.base import InternalPackageSet
39 -from portage.util import writemsg, writemsg_level
40 +from portage.util import ensure_dirs, writemsg, writemsg_level
41 +from portage.util.SlotObject import SlotObject
42 from portage.package.ebuild.digestcheck import digestcheck
43 from portage.package.ebuild.digestgen import digestgen
44 +from portage.package.ebuild.doebuild import (_check_temp_dir,
45 + _prepare_self_update)
46 from portage.package.ebuild.prepare_build_dirs import prepare_build_dirs
47
48 import _emerge
49 @@ -44,6 +45,7 @@ from _emerge.create_depgraph_params import create_depgraph_params
50 from _emerge.create_world_atom import create_world_atom
51 from _emerge.DepPriority import DepPriority
52 from _emerge.depgraph import depgraph, resume_depgraph
53 +from _emerge.EbuildBuildDir import EbuildBuildDir
54 from _emerge.EbuildFetcher import EbuildFetcher
55 from _emerge.EbuildPhase import EbuildPhase
56 from _emerge.emergelog import emergelog
57 @@ -52,12 +54,9 @@ from _emerge._find_deep_system_runtime_deps import _find_deep_system_runtime_dep
58 from _emerge._flush_elog_mod_echo import _flush_elog_mod_echo
59 from _emerge.JobStatusDisplay import JobStatusDisplay
60 from _emerge.MergeListItem import MergeListItem
61 -from _emerge.MiscFunctionsProcess import MiscFunctionsProcess
62 from _emerge.Package import Package
63 from _emerge.PackageMerge import PackageMerge
64 from _emerge.PollScheduler import PollScheduler
65 -from _emerge.RootConfig import RootConfig
66 -from _emerge.SlotObject import SlotObject
67 from _emerge.SequentialTaskQueue import SequentialTaskQueue
68
69 from gobs.build_log import gobs_buildlog
70 @@ -79,17 +78,12 @@ class Scheduler(PollScheduler):
71 frozenset(["--pretend",
72 "--fetchonly", "--fetch-all-uri"])
73
74 - _opts_no_restart = frozenset(["--buildpkgonly",
75 + _opts_no_self_update = frozenset(["--buildpkgonly",
76 "--fetchonly", "--fetch-all-uri", "--pretend"])
77
78 - _bad_resume_opts = set(["--ask", "--changelog",
79 - "--resume", "--skipfirst"])
80 -
81 - class _iface_class(SlotObject):
82 + class _iface_class(PollScheduler._sched_iface_class):
83 __slots__ = ("fetch",
84 - "output", "register", "schedule",
85 - "scheduleSetup", "scheduleUnpack", "scheduleYield",
86 - "unregister")
87 + "scheduleSetup", "scheduleUnpack")
88
89 class _fetch_iface_class(SlotObject):
90 __slots__ = ("log_file", "schedule")
91 @@ -153,7 +147,7 @@ class Scheduler(PollScheduler):
92 DeprecationWarning, stacklevel=2)
93
94 self.settings = settings
95 - self.target_root = settings["ROOT"]
96 + self.target_root = settings["EROOT"]
97 self.trees = trees
98 self.myopts = myopts
99 self._spinner = spinner
100 @@ -163,7 +157,7 @@ class Scheduler(PollScheduler):
101 self._build_opts = self._build_opts_class()
102
103 for k in self._build_opts.__slots__:
104 - setattr(self._build_opts, k, "--" + k.replace("_", "-") in myopts)
105 + setattr(self._build_opts, k, myopts.get("--" + k.replace("_", "-")))
106 self._build_opts.buildpkg_exclude = InternalPackageSet( \
107 initial_atoms=" ".join(myopts.get("--buildpkg-exclude", [])).split(), \
108 allow_wildcard=True, allow_repo=True)
109 @@ -209,10 +203,7 @@ class Scheduler(PollScheduler):
110 if max_jobs is None:
111 max_jobs = 1
112 self._set_max_jobs(max_jobs)
113 -
114 - # The root where the currently running
115 - # portage instance is installed.
116 - self._running_root = trees["/"]["root_config"]
117 + self._running_root = trees[trees._running_eroot]["root_config"]
118 self.edebug = 0
119 if settings.get("PORTAGE_DEBUG", "") == "1":
120 self.edebug = 1
121 @@ -226,13 +217,11 @@ class Scheduler(PollScheduler):
122 fetch_iface = self._fetch_iface_class(log_file=self._fetch_log,
123 schedule=self._schedule_fetch)
124 self._sched_iface = self._iface_class(
125 - fetch=fetch_iface, output=self._task_output,
126 - register=self._register,
127 - schedule=self._schedule_wait,
128 + fetch=fetch_iface,
129 scheduleSetup=self._schedule_setup,
130 scheduleUnpack=self._schedule_unpack,
131 - scheduleYield=self._schedule_yield,
132 - unregister=self._unregister)
133 + **dict((k, getattr(self.sched_iface, k))
134 + for k in self.sched_iface.__slots__))
135
136 self._prefetchers = weakref.WeakValueDictionary()
137 self._pkg_queue = []
138 @@ -296,10 +285,37 @@ class Scheduler(PollScheduler):
139 self._running_portage = self._pkg(cpv, "installed",
140 self._running_root, installed=True)
141
142 + def _handle_self_update(self):
143 +
144 + if self._opts_no_self_update.intersection(self.myopts):
145 + return os.EX_OK
146 +
147 + for x in self._mergelist:
148 + if not isinstance(x, Package):
149 + continue
150 + if x.operation != "merge":
151 + continue
152 + if x.root != self._running_root.root:
153 + continue
154 + if not portage.dep.match_from_list(
155 + portage.const.PORTAGE_PACKAGE_ATOM, [x]):
156 + continue
157 + if self._running_portage is None or \
158 + self._running_portage.cpv != x.cpv or \
159 + '9999' in x.cpv or \
160 + 'git' in x.inherited or \
161 + 'git-2' in x.inherited:
162 + rval = _check_temp_dir(self.settings)
163 + if rval != os.EX_OK:
164 + return rval
165 + _prepare_self_update(self.settings)
166 + break
167 +
168 + return os.EX_OK
169 +
170 def _terminate_tasks(self):
171 self._status_display.quiet = True
172 - while self._running_tasks:
173 - task_id, task = self._running_tasks.popitem()
174 + for task in list(self._running_tasks.values()):
175 task.cancel()
176 for q in self._task_queues.values():
177 q.clear()
178 @@ -311,10 +327,11 @@ class Scheduler(PollScheduler):
179 """
180 self._set_graph_config(graph_config)
181 self._blocker_db = {}
182 + dynamic_deps = self.myopts.get("--dynamic-deps", "y") != "n"
183 for root in self.trees:
184 if graph_config is None:
185 fake_vartree = FakeVartree(self.trees[root]["root_config"],
186 - pkg_cache=self._pkg_cache)
187 + pkg_cache=self._pkg_cache, dynamic_deps=dynamic_deps)
188 fake_vartree.sync()
189 else:
190 fake_vartree = graph_config.trees[root]['vartree']
191 @@ -331,52 +348,6 @@ class Scheduler(PollScheduler):
192 self._set_graph_config(None)
193 gc.collect()
194
195 - def _poll(self, timeout=None):
196 -
197 - self._schedule()
198 -
199 - if timeout is None:
200 - while True:
201 - if not self._poll_event_handlers:
202 - self._schedule()
203 - if not self._poll_event_handlers:
204 - raise StopIteration(
205 - "timeout is None and there are no poll() event handlers")
206 - previous_count = len(self._poll_event_queue)
207 - PollScheduler._poll(self, timeout=self._max_display_latency)
208 - self._status_display.display()
209 - if previous_count != len(self._poll_event_queue):
210 - break
211 -
212 - elif timeout <= self._max_display_latency:
213 - PollScheduler._poll(self, timeout=timeout)
214 - if timeout == 0:
215 - # The display is updated by _schedule() above, so it would be
216 - # redundant to update it here when timeout is 0.
217 - pass
218 - else:
219 - self._status_display.display()
220 -
221 - else:
222 - remaining_timeout = timeout
223 - start_time = time.time()
224 - while True:
225 - previous_count = len(self._poll_event_queue)
226 - PollScheduler._poll(self,
227 - timeout=min(self._max_display_latency, remaining_timeout))
228 - self._status_display.display()
229 - if previous_count != len(self._poll_event_queue):
230 - break
231 - elapsed_time = time.time() - start_time
232 - if elapsed_time < 0:
233 - # The system clock has changed such that start_time
234 - # is now in the future, so just assume that the
235 - # timeout has already elapsed.
236 - break
237 - remaining_timeout = timeout - 1000 * elapsed_time
238 - if remaining_timeout <= 0:
239 - break
240 -
241 def _set_max_jobs(self, max_jobs):
242 self._max_jobs = max_jobs
243 self._task_queues.jobs.max_jobs = max_jobs
244 @@ -388,11 +359,11 @@ class Scheduler(PollScheduler):
245 Check if background mode is enabled and adjust states as necessary.
246
247 @rtype: bool
248 - @returns: True if background mode is enabled, False otherwise.
249 + @return: True if background mode is enabled, False otherwise.
250 """
251 background = (self._max_jobs is True or \
252 self._max_jobs > 1 or "--quiet" in self.myopts \
253 - or "--quiet-build" in self.myopts) and \
254 + or self.myopts.get("--quiet-build") == "y") and \
255 not bool(self._opts_no_background.intersection(self.myopts))
256
257 if background:
258 @@ -405,7 +376,7 @@ class Scheduler(PollScheduler):
259 msg = [""]
260 for pkg in interactive_tasks:
261 pkg_str = " " + colorize("INFORM", str(pkg.cpv))
262 - if pkg.root != "/":
263 + if pkg.root_config.settings["ROOT"] != "/":
264 pkg_str += " for " + pkg.root
265 msg.append(pkg_str)
266 msg.append("")
267 @@ -748,7 +719,6 @@ class Scheduler(PollScheduler):
268 self._status_msg("Starting parallel fetch")
269
270 prefetchers = self._prefetchers
271 - getbinpkg = "--getbinpkg" in self.myopts
272
273 for pkg in self._mergelist:
274 # mergelist can contain solved Blocker instances
275 @@ -756,15 +726,13 @@ class Scheduler(PollScheduler):
276 continue
277 prefetcher = self._create_prefetcher(pkg)
278 if prefetcher is not None:
279 - self._task_queues.fetch.add(prefetcher)
280 + # This will start the first prefetcher immediately, so that
281 + # self._task() won't discard it. This avoids a case where
282 + # the first prefetcher is discarded, causing the second
283 + # prefetcher to occupy the fetch queue before the first
284 + # fetcher has an opportunity to execute.
285 prefetchers[pkg] = prefetcher
286 -
287 - # Start the first prefetcher immediately so that self._task()
288 - # won't discard it. This avoids a case where the first
289 - # prefetcher is discarded, causing the second prefetcher to
290 - # occupy the fetch queue before the first fetcher has an
291 - # opportunity to execute.
292 - self._task_queues.fetch.schedule()
293 + self._task_queues.fetch.add(prefetcher)
294
295 def _create_prefetcher(self, pkg):
296 """
297 @@ -792,100 +760,6 @@ class Scheduler(PollScheduler):
298
299 return prefetcher
300
301 - def _is_restart_scheduled(self):
302 - """
303 - Check if the merge list contains a replacement
304 - for the current running instance, that will result
305 - in restart after merge.
306 - @rtype: bool
307 - @returns: True if a restart is scheduled, False otherwise.
308 - """
309 - if self._opts_no_restart.intersection(self.myopts):
310 - return False
311 -
312 - mergelist = self._mergelist
313 -
314 - for i, pkg in enumerate(mergelist):
315 - if self._is_restart_necessary(pkg) and \
316 - i != len(mergelist) - 1:
317 - return True
318 -
319 - return False
320 -
321 - def _is_restart_necessary(self, pkg):
322 - """
323 - @return: True if merging the given package
324 - requires restart, False otherwise.
325 - """
326 -
327 - # Figure out if we need a restart.
328 - if pkg.root == self._running_root.root and \
329 - portage.match_from_list(
330 - portage.const.PORTAGE_PACKAGE_ATOM, [pkg]):
331 - if self._running_portage is None:
332 - return True
333 - elif pkg.cpv != self._running_portage.cpv or \
334 - '9999' in pkg.cpv or \
335 - 'git' in pkg.inherited or \
336 - 'git-2' in pkg.inherited:
337 - return True
338 - return False
339 -
340 - def _restart_if_necessary(self, pkg):
341 - """
342 - Use execv() to restart emerge. This happens
343 - if portage upgrades itself and there are
344 - remaining packages in the list.
345 - """
346 -
347 - if self._opts_no_restart.intersection(self.myopts):
348 - return
349 -
350 - if not self._is_restart_necessary(pkg):
351 - return
352 -
353 - if pkg == self._mergelist[-1]:
354 - return
355 -
356 - self._main_loop_cleanup()
357 -
358 - logger = self._logger
359 - pkg_count = self._pkg_count
360 - mtimedb = self._mtimedb
361 - bad_resume_opts = self._bad_resume_opts
362 -
363 - logger.log(" ::: completed emerge (%s of %s) %s to %s" % \
364 - (pkg_count.curval, pkg_count.maxval, pkg.cpv, pkg.root))
365 -
366 - logger.log(" *** RESTARTING " + \
367 - "emerge via exec() after change of " + \
368 - "portage version.")
369 -
370 - mtimedb["resume"]["mergelist"].remove(list(pkg))
371 - mtimedb.commit()
372 - portage.run_exitfuncs()
373 - # Don't trust sys.argv[0] here because eselect-python may modify it.
374 - emerge_binary = os.path.join(portage.const.PORTAGE_BIN_PATH, 'emerge')
375 - mynewargv = [emerge_binary, "--resume"]
376 - resume_opts = self.myopts.copy()
377 - # For automatic resume, we need to prevent
378 - # any of bad_resume_opts from leaking in
379 - # via EMERGE_DEFAULT_OPTS.
380 - resume_opts["--ignore-default-opts"] = True
381 - for myopt, myarg in resume_opts.items():
382 - if myopt not in bad_resume_opts:
383 - if myarg is True:
384 - mynewargv.append(myopt)
385 - elif isinstance(myarg, list):
386 - # arguments like --exclude that use 'append' action
387 - for x in myarg:
388 - mynewargv.append("%s=%s" % (myopt, x))
389 - else:
390 - mynewargv.append("%s=%s" % (myopt, myarg))
391 - # priority only needs to be adjusted on the first run
392 - os.environ["PORTAGE_NICENESS"] = "0"
393 - os.execv(mynewargv[0], mynewargv)
394 -
395 def _run_pkg_pretend(self):
396 """
397 Since pkg_pretend output may be important, this method sends all
398 @@ -919,11 +793,48 @@ class Scheduler(PollScheduler):
399 root_config = x.root_config
400 settings = self.pkgsettings[root_config.root]
401 settings.setcpv(x)
402 - tmpdir = tempfile.mkdtemp()
403 - tmpdir_orig = settings["PORTAGE_TMPDIR"]
404 - settings["PORTAGE_TMPDIR"] = tmpdir
405 +
406 + # setcpv/package.env allows for per-package PORTAGE_TMPDIR so we
407 + # have to validate it for each package
408 + rval = _check_temp_dir(settings)
409 + if rval != os.EX_OK:
410 + return rval
411 +
412 + build_dir_path = os.path.join(
413 + os.path.realpath(settings["PORTAGE_TMPDIR"]),
414 + "portage", x.category, x.pf)
415 + existing_buildir = os.path.isdir(build_dir_path)
416 + settings["PORTAGE_BUILDDIR"] = build_dir_path
417 + build_dir = EbuildBuildDir(scheduler=sched_iface,
418 + settings=settings)
419 + build_dir.lock()
420 + current_task = None
421
422 try:
423 +
424 + # Clean up the existing build dir, in case pkg_pretend
425 + # checks for available space (bug #390711).
426 + if existing_buildir:
427 + if x.built:
428 + tree = "bintree"
429 + infloc = os.path.join(build_dir_path, "build-info")
430 + ebuild_path = os.path.join(infloc, x.pf + ".ebuild")
431 + else:
432 + tree = "porttree"
433 + portdb = root_config.trees["porttree"].dbapi
434 + ebuild_path = portdb.findname(x.cpv, myrepo=x.repo)
435 + if ebuild_path is None:
436 + raise AssertionError(
437 + "ebuild not found for '%s'" % x.cpv)
438 + portage.package.ebuild.doebuild.doebuild_environment(
439 + ebuild_path, "clean", settings=settings,
440 + db=self.trees[settings['EROOT']][tree].dbapi)
441 + clean_phase = EbuildPhase(background=False,
442 + phase='clean', scheduler=sched_iface, settings=settings)
443 + current_task = clean_phase
444 + clean_phase.start()
445 + clean_phase.wait()
446 +
447 if x.built:
448 tree = "bintree"
449 bintree = root_config.trees["bintree"].dbapi.bintree
450 @@ -942,6 +853,7 @@ class Scheduler(PollScheduler):
451
452 verifier = BinpkgVerifier(pkg=x,
453 scheduler=sched_iface)
454 + current_task = verifier
455 verifier.start()
456 if verifier.wait() != os.EX_OK:
457 failures += 1
458 @@ -950,8 +862,8 @@ class Scheduler(PollScheduler):
459 if fetched:
460 bintree.inject(x.cpv, filename=fetched)
461 tbz2_file = bintree.getname(x.cpv)
462 - infloc = os.path.join(tmpdir, x.category, x.pf, "build-info")
463 - os.makedirs(infloc)
464 + infloc = os.path.join(build_dir_path, "build-info")
465 + ensure_dirs(infloc)
466 portage.xpak.tbz2(tbz2_file).unpackinfo(infloc)
467 ebuild_path = os.path.join(infloc, x.pf + ".ebuild")
468 settings.configdict["pkg"]["EMERGE_FROM"] = "binary"
469 @@ -971,7 +883,8 @@ class Scheduler(PollScheduler):
470
471 portage.package.ebuild.doebuild.doebuild_environment(ebuild_path,
472 "pretend", settings=settings,
473 - db=self.trees[settings["ROOT"]][tree].dbapi)
474 + db=self.trees[settings['EROOT']][tree].dbapi)
475 +
476 prepare_build_dirs(root_config.root, settings, cleanup=0)
477
478 vardb = root_config.trees['vartree'].dbapi
479 @@ -983,14 +896,21 @@ class Scheduler(PollScheduler):
480 phase="pretend", scheduler=sched_iface,
481 settings=settings)
482
483 + current_task = pretend_phase
484 pretend_phase.start()
485 ret = pretend_phase.wait()
486 if ret != os.EX_OK:
487 failures += 1
488 portage.elog.elog_process(x.cpv, settings)
489 finally:
490 - shutil.rmtree(tmpdir)
491 - settings["PORTAGE_TMPDIR"] = tmpdir_orig
492 + if current_task is not None and current_task.isAlive():
493 + current_task.cancel()
494 + current_task.wait()
495 + clean_phase = EbuildPhase(background=False,
496 + phase='clean', scheduler=sched_iface, settings=settings)
497 + clean_phase.start()
498 + clean_phase.wait()
499 + build_dir.unlock()
500
501 if failures:
502 return 1
503 @@ -1010,6 +930,10 @@ class Scheduler(PollScheduler):
504 except self._unknown_internal_error:
505 return 1
506
507 + rval = self._handle_self_update()
508 + if rval != os.EX_OK:
509 + return rval
510 +
511 for root in self.trees:
512 root_config = self.trees[root]["root_config"]
513
514 @@ -1138,12 +1062,9 @@ class Scheduler(PollScheduler):
515 # If only one package failed then just show it's
516 # whole log for easy viewing.
517 failed_pkg = self._failed_pkgs_all[-1]
518 - build_dir = failed_pkg.build_dir
519 log_file = None
520 log_file_real = None
521
522 - log_paths = [failed_pkg.build_log]
523 -
524 log_path = self._locate_failure_log(failed_pkg)
525 if log_path is not None:
526 try:
527 @@ -1239,9 +1160,6 @@ class Scheduler(PollScheduler):
528
529 def _locate_failure_log(self, failed_pkg):
530
531 - build_dir = failed_pkg.build_dir
532 - log_file = None
533 -
534 log_paths = [failed_pkg.build_log]
535
536 for log_path in log_paths:
537 @@ -1283,7 +1201,7 @@ class Scheduler(PollScheduler):
538
539 # Skip this if $ROOT != / since it shouldn't matter if there
540 # are unsatisfied system runtime deps in this case.
541 - if pkg.root != '/':
542 + if pkg.root_config.settings["ROOT"] != "/":
543 return
544
545 completed_tasks = self._completed_tasks
546 @@ -1365,8 +1283,6 @@ class Scheduler(PollScheduler):
547 init_buildlog.add_buildlog_main(settings, pkg, trees)
548 return
549
550 - self._restart_if_necessary(pkg)
551 -
552 # Call mtimedb.commit() after each merge so that
553 # --resume still works after being interrupted
554 # by reboot, sigkill or similar.
555 @@ -1408,7 +1324,8 @@ class Scheduler(PollScheduler):
556
557 self._failed_pkgs.append(self._failed_pkg(
558 build_dir=build_dir, build_log=build_log,
559 - pkg=pkg, returncode=build.returncode))
560 + pkg=build.pkg,
561 + returncode=build.returncode))
562 if not self._terminated_tasks:
563 self._failed_pkg_msg(self._failed_pkgs[-1], "emerge", "for")
564 self._status_display.failed = len(self._failed_pkgs)
565 @@ -1430,12 +1347,16 @@ class Scheduler(PollScheduler):
566
567 def _merge(self):
568
569 + if self._opts_no_background.intersection(self.myopts):
570 + self._set_max_jobs(1)
571 +
572 self._add_prefetchers()
573 self._add_packages()
574 - pkg_queue = self._pkg_queue
575 failed_pkgs = self._failed_pkgs
576 portage.locks._quiet = self._background
577 portage.elog.add_listener(self._elog_listener)
578 + display_timeout_id = self.sched_iface.timeout_add(
579 + self._max_display_latency, self._status_display.display)
580 rval = os.EX_OK
581
582 try:
583 @@ -1444,6 +1365,7 @@ class Scheduler(PollScheduler):
584 self._main_loop_cleanup()
585 portage.locks._quiet = False
586 portage.elog.remove_listener(self._elog_listener)
587 + self.sched_iface.source_remove(display_timeout_id)
588 if failed_pkgs:
589 rval = failed_pkgs[-1].returncode
590
591 @@ -1524,7 +1446,7 @@ class Scheduler(PollScheduler):
592 merge order
593 @type later: set
594 @rtype: bool
595 - @returns: True if the package is dependent, False otherwise.
596 + @return: True if the package is dependent, False otherwise.
597 """
598
599 graph = self._digraph
600 @@ -1572,24 +1494,7 @@ class Scheduler(PollScheduler):
601 return temp_settings
602
603 def _deallocate_config(self, settings):
604 - self._config_pool[settings["ROOT"]].append(settings)
605 -
606 - def _main_loop(self):
607 -
608 - # Only allow 1 job max if a restart is scheduled
609 - # due to portage update.
610 - if self._is_restart_scheduled() or \
611 - self._opts_no_background.intersection(self.myopts):
612 - self._set_max_jobs(1)
613 -
614 - while self._schedule():
615 - self._poll_loop()
616 -
617 - while True:
618 - self._schedule()
619 - if not self._is_work_scheduled():
620 - break
621 - self._poll_loop()
622 + self._config_pool[settings['EROOT']].append(settings)
623
624 def _keep_scheduling(self):
625 return bool(not self._terminated_tasks and self._pkg_queue and \
626 @@ -1602,6 +1507,8 @@ class Scheduler(PollScheduler):
627
628 while True:
629
630 + state_change = 0
631 +
632 # When the number of jobs and merges drops to zero,
633 # process a single merge from _merge_wait_queue if
634 # it's not empty. We only process one since these are
635 @@ -1612,37 +1519,34 @@ class Scheduler(PollScheduler):
636 not self._task_queues.merge):
637 task = self._merge_wait_queue.popleft()
638 task.addExitListener(self._merge_wait_exit_handler)
639 + self._merge_wait_scheduled.append(task)
640 self._task_queues.merge.add(task)
641 self._status_display.merges = len(self._task_queues.merge)
642 - self._merge_wait_scheduled.append(task)
643 + state_change += 1
644
645 - self._schedule_tasks_imp()
646 - self._status_display.display()
647 + if self._schedule_tasks_imp():
648 + state_change += 1
649
650 - state_change = 0
651 - for q in self._task_queues.values():
652 - if q.schedule():
653 - state_change += 1
654 + self._status_display.display()
655
656 # Cancel prefetchers if they're the only reason
657 # the main poll loop is still running.
658 if self._failed_pkgs and not self._build_opts.fetchonly and \
659 not self._is_work_scheduled() and \
660 self._task_queues.fetch:
661 + # Since this happens asynchronously, it doesn't count in
662 + # state_change (counting it triggers an infinite loop).
663 self._task_queues.fetch.clear()
664 - state_change += 1
665
666 if not (state_change or \
667 (self._merge_wait_queue and not self._jobs and
668 not self._task_queues.merge)):
669 break
670
671 - return self._keep_scheduling()
672 -
673 def _job_delay(self):
674 """
675 @rtype: bool
676 - @returns: True if job scheduling should be delayed, False otherwise.
677 + @return: True if job scheduling should be delayed, False otherwise.
678 """
679
680 if self._jobs and self._max_load is not None:
681 @@ -1660,7 +1564,7 @@ class Scheduler(PollScheduler):
682 def _schedule_tasks_imp(self):
683 """
684 @rtype: bool
685 - @returns: True if state changed, False otherwise.
686 + @return: True if state changed, False otherwise.
687 """
688
689 state_change = 0
690 @@ -1728,7 +1632,14 @@ class Scheduler(PollScheduler):
691 "installed", pkg.root_config, installed=True,
692 operation="uninstall")
693
694 - prefetcher = self._prefetchers.pop(pkg, None)
695 + try:
696 + prefetcher = self._prefetchers.pop(pkg, None)
697 + except KeyError:
698 + # KeyError observed with PyPy 1.8, despite None given as default.
699 + # Note that PyPy 1.8 has the same WeakValueDictionary code as
700 + # CPython 2.7, so it may be possible for CPython to raise KeyError
701 + # here as well.
702 + prefetcher = None
703 if prefetcher is not None and not prefetcher.isAlive():
704 try:
705 self._task_queues.fetch._task_queue.remove(prefetcher)
706 @@ -1757,7 +1668,7 @@ class Scheduler(PollScheduler):
707 pkg = failed_pkg.pkg
708 msg = "%s to %s %s" % \
709 (bad("Failed"), action, colorize("INFORM", pkg.cpv))
710 - if pkg.root != "/":
711 + if pkg.root_config.settings["ROOT"] != "/":
712 msg += " %s %s" % (preposition, pkg.root)
713
714 log_path = self._locate_failure_log(failed_pkg)
715 @@ -1810,7 +1721,7 @@ class Scheduler(PollScheduler):
716 Use the current resume list to calculate a new one,
717 dropping any packages with unsatisfied deps.
718 @rtype: bool
719 - @returns: True if successful, False otherwise.
720 + @return: True if successful, False otherwise.
721 """
722 print(colorize("GOOD", "*** Resuming merge..."))
723
724 @@ -1887,7 +1798,7 @@ class Scheduler(PollScheduler):
725 pkg = task
726 msg = "emerge --keep-going:" + \
727 " %s" % (pkg.cpv,)
728 - if pkg.root != "/":
729 + if pkg.root_config.settings["ROOT"] != "/":
730 msg += " for %s" % (pkg.root,)
731 msg += " dropped due to unsatisfied dependency."
732 for line in textwrap.wrap(msg, msg_width):