1 |
commit: 5fd52e5cf2f85ec872780338fcd2ad165f3123d9 |
2 |
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org> |
3 |
AuthorDate: Tue May 1 00:02:08 2012 +0000 |
4 |
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org> |
5 |
CommitDate: Tue May 1 00:02:08 2012 +0000 |
6 |
URL: http://git.overlays.gentoo.org/gitweb/?p=dev/zorry.git;a=commit;h=5fd52e5c |
7 |
|
8 |
Updated Scheduler.py |
9 |
|
10 |
--- |
11 |
gobs/pym/Scheduler.py | 381 +++++++++++++++++++------------------------------ |
12 |
1 files changed, 146 insertions(+), 235 deletions(-) |
13 |
|
14 |
diff --git a/gobs/pym/Scheduler.py b/gobs/pym/Scheduler.py |
15 |
index 005f861..229c595 100644 |
16 |
--- a/gobs/pym/Scheduler.py |
17 |
+++ b/gobs/pym/Scheduler.py |
18 |
@@ -1,4 +1,4 @@ |
19 |
-# Copyright 1999-2011 Gentoo Foundation |
20 |
+# Copyright 1999-2012 Gentoo Foundation |
21 |
# Distributed under the terms of the GNU General Public License v2 |
22 |
|
23 |
from __future__ import print_function |
24 |
@@ -7,10 +7,8 @@ from collections import deque |
25 |
import gc |
26 |
import gzip |
27 |
import logging |
28 |
-import shutil |
29 |
import signal |
30 |
import sys |
31 |
-import tempfile |
32 |
import textwrap |
33 |
import time |
34 |
import warnings |
35 |
@@ -28,9 +26,12 @@ from portage.output import colorize, create_color_func, red |
36 |
bad = create_color_func("BAD") |
37 |
from portage._sets import SETPREFIX |
38 |
from portage._sets.base import InternalPackageSet |
39 |
-from portage.util import writemsg, writemsg_level |
40 |
+from portage.util import ensure_dirs, writemsg, writemsg_level |
41 |
+from portage.util.SlotObject import SlotObject |
42 |
from portage.package.ebuild.digestcheck import digestcheck |
43 |
from portage.package.ebuild.digestgen import digestgen |
44 |
+from portage.package.ebuild.doebuild import (_check_temp_dir, |
45 |
+ _prepare_self_update) |
46 |
from portage.package.ebuild.prepare_build_dirs import prepare_build_dirs |
47 |
|
48 |
import _emerge |
49 |
@@ -44,6 +45,7 @@ from _emerge.create_depgraph_params import create_depgraph_params |
50 |
from _emerge.create_world_atom import create_world_atom |
51 |
from _emerge.DepPriority import DepPriority |
52 |
from _emerge.depgraph import depgraph, resume_depgraph |
53 |
+from _emerge.EbuildBuildDir import EbuildBuildDir |
54 |
from _emerge.EbuildFetcher import EbuildFetcher |
55 |
from _emerge.EbuildPhase import EbuildPhase |
56 |
from _emerge.emergelog import emergelog |
57 |
@@ -52,12 +54,9 @@ from _emerge._find_deep_system_runtime_deps import _find_deep_system_runtime_dep |
58 |
from _emerge._flush_elog_mod_echo import _flush_elog_mod_echo |
59 |
from _emerge.JobStatusDisplay import JobStatusDisplay |
60 |
from _emerge.MergeListItem import MergeListItem |
61 |
-from _emerge.MiscFunctionsProcess import MiscFunctionsProcess |
62 |
from _emerge.Package import Package |
63 |
from _emerge.PackageMerge import PackageMerge |
64 |
from _emerge.PollScheduler import PollScheduler |
65 |
-from _emerge.RootConfig import RootConfig |
66 |
-from _emerge.SlotObject import SlotObject |
67 |
from _emerge.SequentialTaskQueue import SequentialTaskQueue |
68 |
|
69 |
from gobs.build_log import gobs_buildlog |
70 |
@@ -79,17 +78,12 @@ class Scheduler(PollScheduler): |
71 |
frozenset(["--pretend", |
72 |
"--fetchonly", "--fetch-all-uri"]) |
73 |
|
74 |
- _opts_no_restart = frozenset(["--buildpkgonly", |
75 |
+ _opts_no_self_update = frozenset(["--buildpkgonly", |
76 |
"--fetchonly", "--fetch-all-uri", "--pretend"]) |
77 |
|
78 |
- _bad_resume_opts = set(["--ask", "--changelog", |
79 |
- "--resume", "--skipfirst"]) |
80 |
- |
81 |
- class _iface_class(SlotObject): |
82 |
+ class _iface_class(PollScheduler._sched_iface_class): |
83 |
__slots__ = ("fetch", |
84 |
- "output", "register", "schedule", |
85 |
- "scheduleSetup", "scheduleUnpack", "scheduleYield", |
86 |
- "unregister") |
87 |
+ "scheduleSetup", "scheduleUnpack") |
88 |
|
89 |
class _fetch_iface_class(SlotObject): |
90 |
__slots__ = ("log_file", "schedule") |
91 |
@@ -153,7 +147,7 @@ class Scheduler(PollScheduler): |
92 |
DeprecationWarning, stacklevel=2) |
93 |
|
94 |
self.settings = settings |
95 |
- self.target_root = settings["ROOT"] |
96 |
+ self.target_root = settings["EROOT"] |
97 |
self.trees = trees |
98 |
self.myopts = myopts |
99 |
self._spinner = spinner |
100 |
@@ -163,7 +157,7 @@ class Scheduler(PollScheduler): |
101 |
self._build_opts = self._build_opts_class() |
102 |
|
103 |
for k in self._build_opts.__slots__: |
104 |
- setattr(self._build_opts, k, "--" + k.replace("_", "-") in myopts) |
105 |
+ setattr(self._build_opts, k, myopts.get("--" + k.replace("_", "-"))) |
106 |
self._build_opts.buildpkg_exclude = InternalPackageSet( \ |
107 |
initial_atoms=" ".join(myopts.get("--buildpkg-exclude", [])).split(), \ |
108 |
allow_wildcard=True, allow_repo=True) |
109 |
@@ -209,10 +203,7 @@ class Scheduler(PollScheduler): |
110 |
if max_jobs is None: |
111 |
max_jobs = 1 |
112 |
self._set_max_jobs(max_jobs) |
113 |
- |
114 |
- # The root where the currently running |
115 |
- # portage instance is installed. |
116 |
- self._running_root = trees["/"]["root_config"] |
117 |
+ self._running_root = trees[trees._running_eroot]["root_config"] |
118 |
self.edebug = 0 |
119 |
if settings.get("PORTAGE_DEBUG", "") == "1": |
120 |
self.edebug = 1 |
121 |
@@ -226,13 +217,11 @@ class Scheduler(PollScheduler): |
122 |
fetch_iface = self._fetch_iface_class(log_file=self._fetch_log, |
123 |
schedule=self._schedule_fetch) |
124 |
self._sched_iface = self._iface_class( |
125 |
- fetch=fetch_iface, output=self._task_output, |
126 |
- register=self._register, |
127 |
- schedule=self._schedule_wait, |
128 |
+ fetch=fetch_iface, |
129 |
scheduleSetup=self._schedule_setup, |
130 |
scheduleUnpack=self._schedule_unpack, |
131 |
- scheduleYield=self._schedule_yield, |
132 |
- unregister=self._unregister) |
133 |
+ **dict((k, getattr(self.sched_iface, k)) |
134 |
+ for k in self.sched_iface.__slots__)) |
135 |
|
136 |
self._prefetchers = weakref.WeakValueDictionary() |
137 |
self._pkg_queue = [] |
138 |
@@ -296,10 +285,37 @@ class Scheduler(PollScheduler): |
139 |
self._running_portage = self._pkg(cpv, "installed", |
140 |
self._running_root, installed=True) |
141 |
|
142 |
+ def _handle_self_update(self): |
143 |
+ |
144 |
+ if self._opts_no_self_update.intersection(self.myopts): |
145 |
+ return os.EX_OK |
146 |
+ |
147 |
+ for x in self._mergelist: |
148 |
+ if not isinstance(x, Package): |
149 |
+ continue |
150 |
+ if x.operation != "merge": |
151 |
+ continue |
152 |
+ if x.root != self._running_root.root: |
153 |
+ continue |
154 |
+ if not portage.dep.match_from_list( |
155 |
+ portage.const.PORTAGE_PACKAGE_ATOM, [x]): |
156 |
+ continue |
157 |
+ if self._running_portage is None or \ |
158 |
+ self._running_portage.cpv != x.cpv or \ |
159 |
+ '9999' in x.cpv or \ |
160 |
+ 'git' in x.inherited or \ |
161 |
+ 'git-2' in x.inherited: |
162 |
+ rval = _check_temp_dir(self.settings) |
163 |
+ if rval != os.EX_OK: |
164 |
+ return rval |
165 |
+ _prepare_self_update(self.settings) |
166 |
+ break |
167 |
+ |
168 |
+ return os.EX_OK |
169 |
+ |
170 |
def _terminate_tasks(self): |
171 |
self._status_display.quiet = True |
172 |
- while self._running_tasks: |
173 |
- task_id, task = self._running_tasks.popitem() |
174 |
+ for task in list(self._running_tasks.values()): |
175 |
task.cancel() |
176 |
for q in self._task_queues.values(): |
177 |
q.clear() |
178 |
@@ -311,10 +327,11 @@ class Scheduler(PollScheduler): |
179 |
""" |
180 |
self._set_graph_config(graph_config) |
181 |
self._blocker_db = {} |
182 |
+ dynamic_deps = self.myopts.get("--dynamic-deps", "y") != "n" |
183 |
for root in self.trees: |
184 |
if graph_config is None: |
185 |
fake_vartree = FakeVartree(self.trees[root]["root_config"], |
186 |
- pkg_cache=self._pkg_cache) |
187 |
+ pkg_cache=self._pkg_cache, dynamic_deps=dynamic_deps) |
188 |
fake_vartree.sync() |
189 |
else: |
190 |
fake_vartree = graph_config.trees[root]['vartree'] |
191 |
@@ -331,52 +348,6 @@ class Scheduler(PollScheduler): |
192 |
self._set_graph_config(None) |
193 |
gc.collect() |
194 |
|
195 |
- def _poll(self, timeout=None): |
196 |
- |
197 |
- self._schedule() |
198 |
- |
199 |
- if timeout is None: |
200 |
- while True: |
201 |
- if not self._poll_event_handlers: |
202 |
- self._schedule() |
203 |
- if not self._poll_event_handlers: |
204 |
- raise StopIteration( |
205 |
- "timeout is None and there are no poll() event handlers") |
206 |
- previous_count = len(self._poll_event_queue) |
207 |
- PollScheduler._poll(self, timeout=self._max_display_latency) |
208 |
- self._status_display.display() |
209 |
- if previous_count != len(self._poll_event_queue): |
210 |
- break |
211 |
- |
212 |
- elif timeout <= self._max_display_latency: |
213 |
- PollScheduler._poll(self, timeout=timeout) |
214 |
- if timeout == 0: |
215 |
- # The display is updated by _schedule() above, so it would be |
216 |
- # redundant to update it here when timeout is 0. |
217 |
- pass |
218 |
- else: |
219 |
- self._status_display.display() |
220 |
- |
221 |
- else: |
222 |
- remaining_timeout = timeout |
223 |
- start_time = time.time() |
224 |
- while True: |
225 |
- previous_count = len(self._poll_event_queue) |
226 |
- PollScheduler._poll(self, |
227 |
- timeout=min(self._max_display_latency, remaining_timeout)) |
228 |
- self._status_display.display() |
229 |
- if previous_count != len(self._poll_event_queue): |
230 |
- break |
231 |
- elapsed_time = time.time() - start_time |
232 |
- if elapsed_time < 0: |
233 |
- # The system clock has changed such that start_time |
234 |
- # is now in the future, so just assume that the |
235 |
- # timeout has already elapsed. |
236 |
- break |
237 |
- remaining_timeout = timeout - 1000 * elapsed_time |
238 |
- if remaining_timeout <= 0: |
239 |
- break |
240 |
- |
241 |
def _set_max_jobs(self, max_jobs): |
242 |
self._max_jobs = max_jobs |
243 |
self._task_queues.jobs.max_jobs = max_jobs |
244 |
@@ -388,11 +359,11 @@ class Scheduler(PollScheduler): |
245 |
Check if background mode is enabled and adjust states as necessary. |
246 |
|
247 |
@rtype: bool |
248 |
- @returns: True if background mode is enabled, False otherwise. |
249 |
+ @return: True if background mode is enabled, False otherwise. |
250 |
""" |
251 |
background = (self._max_jobs is True or \ |
252 |
self._max_jobs > 1 or "--quiet" in self.myopts \ |
253 |
- or "--quiet-build" in self.myopts) and \ |
254 |
+ or self.myopts.get("--quiet-build") == "y") and \ |
255 |
not bool(self._opts_no_background.intersection(self.myopts)) |
256 |
|
257 |
if background: |
258 |
@@ -405,7 +376,7 @@ class Scheduler(PollScheduler): |
259 |
msg = [""] |
260 |
for pkg in interactive_tasks: |
261 |
pkg_str = " " + colorize("INFORM", str(pkg.cpv)) |
262 |
- if pkg.root != "/": |
263 |
+ if pkg.root_config.settings["ROOT"] != "/": |
264 |
pkg_str += " for " + pkg.root |
265 |
msg.append(pkg_str) |
266 |
msg.append("") |
267 |
@@ -748,7 +719,6 @@ class Scheduler(PollScheduler): |
268 |
self._status_msg("Starting parallel fetch") |
269 |
|
270 |
prefetchers = self._prefetchers |
271 |
- getbinpkg = "--getbinpkg" in self.myopts |
272 |
|
273 |
for pkg in self._mergelist: |
274 |
# mergelist can contain solved Blocker instances |
275 |
@@ -756,15 +726,13 @@ class Scheduler(PollScheduler): |
276 |
continue |
277 |
prefetcher = self._create_prefetcher(pkg) |
278 |
if prefetcher is not None: |
279 |
- self._task_queues.fetch.add(prefetcher) |
280 |
+ # This will start the first prefetcher immediately, so that |
281 |
+ # self._task() won't discard it. This avoids a case where |
282 |
+ # the first prefetcher is discarded, causing the second |
283 |
+ # prefetcher to occupy the fetch queue before the first |
284 |
+ # fetcher has an opportunity to execute. |
285 |
prefetchers[pkg] = prefetcher |
286 |
- |
287 |
- # Start the first prefetcher immediately so that self._task() |
288 |
- # won't discard it. This avoids a case where the first |
289 |
- # prefetcher is discarded, causing the second prefetcher to |
290 |
- # occupy the fetch queue before the first fetcher has an |
291 |
- # opportunity to execute. |
292 |
- self._task_queues.fetch.schedule() |
293 |
+ self._task_queues.fetch.add(prefetcher) |
294 |
|
295 |
def _create_prefetcher(self, pkg): |
296 |
""" |
297 |
@@ -792,100 +760,6 @@ class Scheduler(PollScheduler): |
298 |
|
299 |
return prefetcher |
300 |
|
301 |
- def _is_restart_scheduled(self): |
302 |
- """ |
303 |
- Check if the merge list contains a replacement |
304 |
- for the current running instance, that will result |
305 |
- in restart after merge. |
306 |
- @rtype: bool |
307 |
- @returns: True if a restart is scheduled, False otherwise. |
308 |
- """ |
309 |
- if self._opts_no_restart.intersection(self.myopts): |
310 |
- return False |
311 |
- |
312 |
- mergelist = self._mergelist |
313 |
- |
314 |
- for i, pkg in enumerate(mergelist): |
315 |
- if self._is_restart_necessary(pkg) and \ |
316 |
- i != len(mergelist) - 1: |
317 |
- return True |
318 |
- |
319 |
- return False |
320 |
- |
321 |
- def _is_restart_necessary(self, pkg): |
322 |
- """ |
323 |
- @return: True if merging the given package |
324 |
- requires restart, False otherwise. |
325 |
- """ |
326 |
- |
327 |
- # Figure out if we need a restart. |
328 |
- if pkg.root == self._running_root.root and \ |
329 |
- portage.match_from_list( |
330 |
- portage.const.PORTAGE_PACKAGE_ATOM, [pkg]): |
331 |
- if self._running_portage is None: |
332 |
- return True |
333 |
- elif pkg.cpv != self._running_portage.cpv or \ |
334 |
- '9999' in pkg.cpv or \ |
335 |
- 'git' in pkg.inherited or \ |
336 |
- 'git-2' in pkg.inherited: |
337 |
- return True |
338 |
- return False |
339 |
- |
340 |
- def _restart_if_necessary(self, pkg): |
341 |
- """ |
342 |
- Use execv() to restart emerge. This happens |
343 |
- if portage upgrades itself and there are |
344 |
- remaining packages in the list. |
345 |
- """ |
346 |
- |
347 |
- if self._opts_no_restart.intersection(self.myopts): |
348 |
- return |
349 |
- |
350 |
- if not self._is_restart_necessary(pkg): |
351 |
- return |
352 |
- |
353 |
- if pkg == self._mergelist[-1]: |
354 |
- return |
355 |
- |
356 |
- self._main_loop_cleanup() |
357 |
- |
358 |
- logger = self._logger |
359 |
- pkg_count = self._pkg_count |
360 |
- mtimedb = self._mtimedb |
361 |
- bad_resume_opts = self._bad_resume_opts |
362 |
- |
363 |
- logger.log(" ::: completed emerge (%s of %s) %s to %s" % \ |
364 |
- (pkg_count.curval, pkg_count.maxval, pkg.cpv, pkg.root)) |
365 |
- |
366 |
- logger.log(" *** RESTARTING " + \ |
367 |
- "emerge via exec() after change of " + \ |
368 |
- "portage version.") |
369 |
- |
370 |
- mtimedb["resume"]["mergelist"].remove(list(pkg)) |
371 |
- mtimedb.commit() |
372 |
- portage.run_exitfuncs() |
373 |
- # Don't trust sys.argv[0] here because eselect-python may modify it. |
374 |
- emerge_binary = os.path.join(portage.const.PORTAGE_BIN_PATH, 'emerge') |
375 |
- mynewargv = [emerge_binary, "--resume"] |
376 |
- resume_opts = self.myopts.copy() |
377 |
- # For automatic resume, we need to prevent |
378 |
- # any of bad_resume_opts from leaking in |
379 |
- # via EMERGE_DEFAULT_OPTS. |
380 |
- resume_opts["--ignore-default-opts"] = True |
381 |
- for myopt, myarg in resume_opts.items(): |
382 |
- if myopt not in bad_resume_opts: |
383 |
- if myarg is True: |
384 |
- mynewargv.append(myopt) |
385 |
- elif isinstance(myarg, list): |
386 |
- # arguments like --exclude that use 'append' action |
387 |
- for x in myarg: |
388 |
- mynewargv.append("%s=%s" % (myopt, x)) |
389 |
- else: |
390 |
- mynewargv.append("%s=%s" % (myopt, myarg)) |
391 |
- # priority only needs to be adjusted on the first run |
392 |
- os.environ["PORTAGE_NICENESS"] = "0" |
393 |
- os.execv(mynewargv[0], mynewargv) |
394 |
- |
395 |
def _run_pkg_pretend(self): |
396 |
""" |
397 |
Since pkg_pretend output may be important, this method sends all |
398 |
@@ -919,11 +793,48 @@ class Scheduler(PollScheduler): |
399 |
root_config = x.root_config |
400 |
settings = self.pkgsettings[root_config.root] |
401 |
settings.setcpv(x) |
402 |
- tmpdir = tempfile.mkdtemp() |
403 |
- tmpdir_orig = settings["PORTAGE_TMPDIR"] |
404 |
- settings["PORTAGE_TMPDIR"] = tmpdir |
405 |
+ |
406 |
+ # setcpv/package.env allows for per-package PORTAGE_TMPDIR so we |
407 |
+ # have to validate it for each package |
408 |
+ rval = _check_temp_dir(settings) |
409 |
+ if rval != os.EX_OK: |
410 |
+ return rval |
411 |
+ |
412 |
+ build_dir_path = os.path.join( |
413 |
+ os.path.realpath(settings["PORTAGE_TMPDIR"]), |
414 |
+ "portage", x.category, x.pf) |
415 |
+ existing_buildir = os.path.isdir(build_dir_path) |
416 |
+ settings["PORTAGE_BUILDDIR"] = build_dir_path |
417 |
+ build_dir = EbuildBuildDir(scheduler=sched_iface, |
418 |
+ settings=settings) |
419 |
+ build_dir.lock() |
420 |
+ current_task = None |
421 |
|
422 |
try: |
423 |
+ |
424 |
+ # Clean up the existing build dir, in case pkg_pretend |
425 |
+ # checks for available space (bug #390711). |
426 |
+ if existing_buildir: |
427 |
+ if x.built: |
428 |
+ tree = "bintree" |
429 |
+ infloc = os.path.join(build_dir_path, "build-info") |
430 |
+ ebuild_path = os.path.join(infloc, x.pf + ".ebuild") |
431 |
+ else: |
432 |
+ tree = "porttree" |
433 |
+ portdb = root_config.trees["porttree"].dbapi |
434 |
+ ebuild_path = portdb.findname(x.cpv, myrepo=x.repo) |
435 |
+ if ebuild_path is None: |
436 |
+ raise AssertionError( |
437 |
+ "ebuild not found for '%s'" % x.cpv) |
438 |
+ portage.package.ebuild.doebuild.doebuild_environment( |
439 |
+ ebuild_path, "clean", settings=settings, |
440 |
+ db=self.trees[settings['EROOT']][tree].dbapi) |
441 |
+ clean_phase = EbuildPhase(background=False, |
442 |
+ phase='clean', scheduler=sched_iface, settings=settings) |
443 |
+ current_task = clean_phase |
444 |
+ clean_phase.start() |
445 |
+ clean_phase.wait() |
446 |
+ |
447 |
if x.built: |
448 |
tree = "bintree" |
449 |
bintree = root_config.trees["bintree"].dbapi.bintree |
450 |
@@ -942,6 +853,7 @@ class Scheduler(PollScheduler): |
451 |
|
452 |
verifier = BinpkgVerifier(pkg=x, |
453 |
scheduler=sched_iface) |
454 |
+ current_task = verifier |
455 |
verifier.start() |
456 |
if verifier.wait() != os.EX_OK: |
457 |
failures += 1 |
458 |
@@ -950,8 +862,8 @@ class Scheduler(PollScheduler): |
459 |
if fetched: |
460 |
bintree.inject(x.cpv, filename=fetched) |
461 |
tbz2_file = bintree.getname(x.cpv) |
462 |
- infloc = os.path.join(tmpdir, x.category, x.pf, "build-info") |
463 |
- os.makedirs(infloc) |
464 |
+ infloc = os.path.join(build_dir_path, "build-info") |
465 |
+ ensure_dirs(infloc) |
466 |
portage.xpak.tbz2(tbz2_file).unpackinfo(infloc) |
467 |
ebuild_path = os.path.join(infloc, x.pf + ".ebuild") |
468 |
settings.configdict["pkg"]["EMERGE_FROM"] = "binary" |
469 |
@@ -971,7 +883,8 @@ class Scheduler(PollScheduler): |
470 |
|
471 |
portage.package.ebuild.doebuild.doebuild_environment(ebuild_path, |
472 |
"pretend", settings=settings, |
473 |
- db=self.trees[settings["ROOT"]][tree].dbapi) |
474 |
+ db=self.trees[settings['EROOT']][tree].dbapi) |
475 |
+ |
476 |
prepare_build_dirs(root_config.root, settings, cleanup=0) |
477 |
|
478 |
vardb = root_config.trees['vartree'].dbapi |
479 |
@@ -983,14 +896,21 @@ class Scheduler(PollScheduler): |
480 |
phase="pretend", scheduler=sched_iface, |
481 |
settings=settings) |
482 |
|
483 |
+ current_task = pretend_phase |
484 |
pretend_phase.start() |
485 |
ret = pretend_phase.wait() |
486 |
if ret != os.EX_OK: |
487 |
failures += 1 |
488 |
portage.elog.elog_process(x.cpv, settings) |
489 |
finally: |
490 |
- shutil.rmtree(tmpdir) |
491 |
- settings["PORTAGE_TMPDIR"] = tmpdir_orig |
492 |
+ if current_task is not None and current_task.isAlive(): |
493 |
+ current_task.cancel() |
494 |
+ current_task.wait() |
495 |
+ clean_phase = EbuildPhase(background=False, |
496 |
+ phase='clean', scheduler=sched_iface, settings=settings) |
497 |
+ clean_phase.start() |
498 |
+ clean_phase.wait() |
499 |
+ build_dir.unlock() |
500 |
|
501 |
if failures: |
502 |
return 1 |
503 |
@@ -1010,6 +930,10 @@ class Scheduler(PollScheduler): |
504 |
except self._unknown_internal_error: |
505 |
return 1 |
506 |
|
507 |
+ rval = self._handle_self_update() |
508 |
+ if rval != os.EX_OK: |
509 |
+ return rval |
510 |
+ |
511 |
for root in self.trees: |
512 |
root_config = self.trees[root]["root_config"] |
513 |
|
514 |
@@ -1138,12 +1062,9 @@ class Scheduler(PollScheduler): |
515 |
# If only one package failed then just show it's |
516 |
# whole log for easy viewing. |
517 |
failed_pkg = self._failed_pkgs_all[-1] |
518 |
- build_dir = failed_pkg.build_dir |
519 |
log_file = None |
520 |
log_file_real = None |
521 |
|
522 |
- log_paths = [failed_pkg.build_log] |
523 |
- |
524 |
log_path = self._locate_failure_log(failed_pkg) |
525 |
if log_path is not None: |
526 |
try: |
527 |
@@ -1239,9 +1160,6 @@ class Scheduler(PollScheduler): |
528 |
|
529 |
def _locate_failure_log(self, failed_pkg): |
530 |
|
531 |
- build_dir = failed_pkg.build_dir |
532 |
- log_file = None |
533 |
- |
534 |
log_paths = [failed_pkg.build_log] |
535 |
|
536 |
for log_path in log_paths: |
537 |
@@ -1283,7 +1201,7 @@ class Scheduler(PollScheduler): |
538 |
|
539 |
# Skip this if $ROOT != / since it shouldn't matter if there |
540 |
# are unsatisfied system runtime deps in this case. |
541 |
- if pkg.root != '/': |
542 |
+ if pkg.root_config.settings["ROOT"] != "/": |
543 |
return |
544 |
|
545 |
completed_tasks = self._completed_tasks |
546 |
@@ -1365,8 +1283,6 @@ class Scheduler(PollScheduler): |
547 |
init_buildlog.add_buildlog_main(settings, pkg, trees) |
548 |
return |
549 |
|
550 |
- self._restart_if_necessary(pkg) |
551 |
- |
552 |
# Call mtimedb.commit() after each merge so that |
553 |
# --resume still works after being interrupted |
554 |
# by reboot, sigkill or similar. |
555 |
@@ -1408,7 +1324,8 @@ class Scheduler(PollScheduler): |
556 |
|
557 |
self._failed_pkgs.append(self._failed_pkg( |
558 |
build_dir=build_dir, build_log=build_log, |
559 |
- pkg=pkg, returncode=build.returncode)) |
560 |
+ pkg=build.pkg, |
561 |
+ returncode=build.returncode)) |
562 |
if not self._terminated_tasks: |
563 |
self._failed_pkg_msg(self._failed_pkgs[-1], "emerge", "for") |
564 |
self._status_display.failed = len(self._failed_pkgs) |
565 |
@@ -1430,12 +1347,16 @@ class Scheduler(PollScheduler): |
566 |
|
567 |
def _merge(self): |
568 |
|
569 |
+ if self._opts_no_background.intersection(self.myopts): |
570 |
+ self._set_max_jobs(1) |
571 |
+ |
572 |
self._add_prefetchers() |
573 |
self._add_packages() |
574 |
- pkg_queue = self._pkg_queue |
575 |
failed_pkgs = self._failed_pkgs |
576 |
portage.locks._quiet = self._background |
577 |
portage.elog.add_listener(self._elog_listener) |
578 |
+ display_timeout_id = self.sched_iface.timeout_add( |
579 |
+ self._max_display_latency, self._status_display.display) |
580 |
rval = os.EX_OK |
581 |
|
582 |
try: |
583 |
@@ -1444,6 +1365,7 @@ class Scheduler(PollScheduler): |
584 |
self._main_loop_cleanup() |
585 |
portage.locks._quiet = False |
586 |
portage.elog.remove_listener(self._elog_listener) |
587 |
+ self.sched_iface.source_remove(display_timeout_id) |
588 |
if failed_pkgs: |
589 |
rval = failed_pkgs[-1].returncode |
590 |
|
591 |
@@ -1524,7 +1446,7 @@ class Scheduler(PollScheduler): |
592 |
merge order |
593 |
@type later: set |
594 |
@rtype: bool |
595 |
- @returns: True if the package is dependent, False otherwise. |
596 |
+ @return: True if the package is dependent, False otherwise. |
597 |
""" |
598 |
|
599 |
graph = self._digraph |
600 |
@@ -1572,24 +1494,7 @@ class Scheduler(PollScheduler): |
601 |
return temp_settings |
602 |
|
603 |
def _deallocate_config(self, settings): |
604 |
- self._config_pool[settings["ROOT"]].append(settings) |
605 |
- |
606 |
- def _main_loop(self): |
607 |
- |
608 |
- # Only allow 1 job max if a restart is scheduled |
609 |
- # due to portage update. |
610 |
- if self._is_restart_scheduled() or \ |
611 |
- self._opts_no_background.intersection(self.myopts): |
612 |
- self._set_max_jobs(1) |
613 |
- |
614 |
- while self._schedule(): |
615 |
- self._poll_loop() |
616 |
- |
617 |
- while True: |
618 |
- self._schedule() |
619 |
- if not self._is_work_scheduled(): |
620 |
- break |
621 |
- self._poll_loop() |
622 |
+ self._config_pool[settings['EROOT']].append(settings) |
623 |
|
624 |
def _keep_scheduling(self): |
625 |
return bool(not self._terminated_tasks and self._pkg_queue and \ |
626 |
@@ -1602,6 +1507,8 @@ class Scheduler(PollScheduler): |
627 |
|
628 |
while True: |
629 |
|
630 |
+ state_change = 0 |
631 |
+ |
632 |
# When the number of jobs and merges drops to zero, |
633 |
# process a single merge from _merge_wait_queue if |
634 |
# it's not empty. We only process one since these are |
635 |
@@ -1612,37 +1519,34 @@ class Scheduler(PollScheduler): |
636 |
not self._task_queues.merge): |
637 |
task = self._merge_wait_queue.popleft() |
638 |
task.addExitListener(self._merge_wait_exit_handler) |
639 |
+ self._merge_wait_scheduled.append(task) |
640 |
self._task_queues.merge.add(task) |
641 |
self._status_display.merges = len(self._task_queues.merge) |
642 |
- self._merge_wait_scheduled.append(task) |
643 |
+ state_change += 1 |
644 |
|
645 |
- self._schedule_tasks_imp() |
646 |
- self._status_display.display() |
647 |
+ if self._schedule_tasks_imp(): |
648 |
+ state_change += 1 |
649 |
|
650 |
- state_change = 0 |
651 |
- for q in self._task_queues.values(): |
652 |
- if q.schedule(): |
653 |
- state_change += 1 |
654 |
+ self._status_display.display() |
655 |
|
656 |
# Cancel prefetchers if they're the only reason |
657 |
# the main poll loop is still running. |
658 |
if self._failed_pkgs and not self._build_opts.fetchonly and \ |
659 |
not self._is_work_scheduled() and \ |
660 |
self._task_queues.fetch: |
661 |
+ # Since this happens asynchronously, it doesn't count in |
662 |
+ # state_change (counting it triggers an infinite loop). |
663 |
self._task_queues.fetch.clear() |
664 |
- state_change += 1 |
665 |
|
666 |
if not (state_change or \ |
667 |
(self._merge_wait_queue and not self._jobs and |
668 |
not self._task_queues.merge)): |
669 |
break |
670 |
|
671 |
- return self._keep_scheduling() |
672 |
- |
673 |
def _job_delay(self): |
674 |
""" |
675 |
@rtype: bool |
676 |
- @returns: True if job scheduling should be delayed, False otherwise. |
677 |
+ @return: True if job scheduling should be delayed, False otherwise. |
678 |
""" |
679 |
|
680 |
if self._jobs and self._max_load is not None: |
681 |
@@ -1660,7 +1564,7 @@ class Scheduler(PollScheduler): |
682 |
def _schedule_tasks_imp(self): |
683 |
""" |
684 |
@rtype: bool |
685 |
- @returns: True if state changed, False otherwise. |
686 |
+ @return: True if state changed, False otherwise. |
687 |
""" |
688 |
|
689 |
state_change = 0 |
690 |
@@ -1728,7 +1632,14 @@ class Scheduler(PollScheduler): |
691 |
"installed", pkg.root_config, installed=True, |
692 |
operation="uninstall") |
693 |
|
694 |
- prefetcher = self._prefetchers.pop(pkg, None) |
695 |
+ try: |
696 |
+ prefetcher = self._prefetchers.pop(pkg, None) |
697 |
+ except KeyError: |
698 |
+ # KeyError observed with PyPy 1.8, despite None given as default. |
699 |
+ # Note that PyPy 1.8 has the same WeakValueDictionary code as |
700 |
+ # CPython 2.7, so it may be possible for CPython to raise KeyError |
701 |
+ # here as well. |
702 |
+ prefetcher = None |
703 |
if prefetcher is not None and not prefetcher.isAlive(): |
704 |
try: |
705 |
self._task_queues.fetch._task_queue.remove(prefetcher) |
706 |
@@ -1757,7 +1668,7 @@ class Scheduler(PollScheduler): |
707 |
pkg = failed_pkg.pkg |
708 |
msg = "%s to %s %s" % \ |
709 |
(bad("Failed"), action, colorize("INFORM", pkg.cpv)) |
710 |
- if pkg.root != "/": |
711 |
+ if pkg.root_config.settings["ROOT"] != "/": |
712 |
msg += " %s %s" % (preposition, pkg.root) |
713 |
|
714 |
log_path = self._locate_failure_log(failed_pkg) |
715 |
@@ -1810,7 +1721,7 @@ class Scheduler(PollScheduler): |
716 |
Use the current resume list to calculate a new one, |
717 |
dropping any packages with unsatisfied deps. |
718 |
@rtype: bool |
719 |
- @returns: True if successful, False otherwise. |
720 |
+ @return: True if successful, False otherwise. |
721 |
""" |
722 |
print(colorize("GOOD", "*** Resuming merge...")) |
723 |
|
724 |
@@ -1887,7 +1798,7 @@ class Scheduler(PollScheduler): |
725 |
pkg = task |
726 |
msg = "emerge --keep-going:" + \ |
727 |
" %s" % (pkg.cpv,) |
728 |
- if pkg.root != "/": |
729 |
+ if pkg.root_config.settings["ROOT"] != "/": |
730 |
msg += " for %s" % (pkg.root,) |
731 |
msg += " dropped due to unsatisfied dependency." |
732 |
for line in textwrap.wrap(msg, msg_width): |