| 1 |
Author: zmedico |
| 2 |
Date: 2008-07-04 00:11:03 +0000 (Fri, 04 Jul 2008) |
| 3 |
New Revision: 10918 |
| 4 |
|
| 5 |
Modified: |
| 6 |
main/trunk/pym/_emerge/__init__.py |
| 7 |
Log: |
| 8 |
* Split out a SequentialTaskQueue class to encapsulate the parallel-fetch |
| 9 |
prefetcher queue. |
| 10 |
|
| 11 |
* Fix broken return value status handling in Scheduler.merge(). |
| 12 |
|
| 13 |
|
| 14 |
Modified: main/trunk/pym/_emerge/__init__.py |
| 15 |
=================================================================== |
| 16 |
--- main/trunk/pym/_emerge/__init__.py 2008-07-03 22:48:21 UTC (rev 10917) |
| 17 |
+++ main/trunk/pym/_emerge/__init__.py 2008-07-04 00:11:03 UTC (rev 10918) |
| 18 |
@@ -6966,6 +6966,49 @@ |
| 19 |
poll_events.append((fd, select.POLLIN)) |
| 20 |
return poll_events |
| 21 |
|
| 22 |
+class SequentialTaskQueue(SlotObject): |
| 23 |
+ |
| 24 |
+ __slots__ = ("max_jobs", "running_tasks", "_task_queue") |
| 25 |
+ |
| 26 |
+ def __init__(self, **kwargs): |
| 27 |
+ SlotObject.__init__(self, **kwargs) |
| 28 |
+ from collections import deque |
| 29 |
+ self._task_queue = deque() |
| 30 |
+ self.running_tasks = set() |
| 31 |
+ if self.max_jobs is None: |
| 32 |
+ self.max_jobs = 1 |
| 33 |
+ |
| 34 |
+ def add(self, task): |
| 35 |
+ self._task_queue.append(task) |
| 36 |
+ |
| 37 |
+ def schedule(self): |
| 38 |
+ task_queue = self._task_queue |
| 39 |
+ running_tasks = self.running_tasks |
| 40 |
+ max_jobs = self.max_jobs |
| 41 |
+ state_changed = False |
| 42 |
+ |
| 43 |
+ for task in list(running_tasks): |
| 44 |
+ if not task.registered and task.poll() is not None: |
| 45 |
+ running_tasks.remove(task) |
| 46 |
+ state_changed = True |
| 47 |
+ |
| 48 |
+ while task_queue and (len(running_tasks) < max_jobs): |
| 49 |
+ task = task_queue.popleft() |
| 50 |
+ cancelled = getattr(task, "cancelled", None) |
| 51 |
+ if not cancelled: |
| 52 |
+ task.start() |
| 53 |
+ running_tasks.add(task) |
| 54 |
+ state_changed = True |
| 55 |
+ |
| 56 |
+ return state_changed |
| 57 |
+ |
| 58 |
+ def clear(self): |
| 59 |
+ self._task_queue.clear() |
| 60 |
+ running_tasks = self.running_tasks |
| 61 |
+ while running_tasks: |
| 62 |
+ task = running_tasks.pop() |
| 63 |
+ task.cancel() |
| 64 |
+ |
| 65 |
class Scheduler(object): |
| 66 |
|
| 67 |
_opts_ignore_blockers = \ |
| 68 |
@@ -7038,10 +7081,10 @@ |
| 69 |
except AttributeError: |
| 70 |
self._poll = PollSelectFallback() |
| 71 |
|
| 72 |
- from collections import deque |
| 73 |
- self._task_queue = deque() |
| 74 |
- self._running_tasks = set() |
| 75 |
- self._max_jobs = 1 |
| 76 |
+ self._prefetch_queue = SequentialTaskQueue() |
| 77 |
+ self._add_task = self._prefetch_queue.add |
| 78 |
+ self._schedule_tasks = self._prefetch_queue.schedule |
| 79 |
+ |
| 80 |
self._prefetchers = weakref.WeakValueDictionary() |
| 81 |
self._failed_fetches = [] |
| 82 |
self._parallel_fetch = False |
| 83 |
@@ -7071,9 +7114,6 @@ |
| 84 |
except EnvironmentError: |
| 85 |
pass |
| 86 |
|
| 87 |
- def _add_task(self, task): |
| 88 |
- self._task_queue.append(task) |
| 89 |
- |
| 90 |
class _pkg_failure(portage.exception.PortageException): |
| 91 |
""" |
| 92 |
An instance of this class is raised by unmerge() when |
| 93 |
@@ -7282,7 +7322,7 @@ |
| 94 |
mtimedb = self._mtimedb |
| 95 |
|
| 96 |
while True: |
| 97 |
- self._merge() |
| 98 |
+ rval = self._merge() |
| 99 |
self._show_failed_fetches() |
| 100 |
del self._failed_fetches[:] |
| 101 |
|
| 102 |
@@ -7350,11 +7390,7 @@ |
| 103 |
return e.status |
| 104 |
finally: |
| 105 |
# clean up child process if necessary |
| 106 |
- self._task_queue.clear() |
| 107 |
- running_tasks = self._running_tasks |
| 108 |
- while running_tasks: |
| 109 |
- task = running_tasks.pop() |
| 110 |
- task.cancel() |
| 111 |
+ self._prefetch_queue.clear() |
| 112 |
return os.EX_OK |
| 113 |
|
| 114 |
def _save_resume_list(self): |
| 115 |
@@ -7411,7 +7447,7 @@ |
| 116 |
|
| 117 |
def _schedule(self): |
| 118 |
event_handlers = self._poll_event_handlers |
| 119 |
- running_tasks = self._running_tasks |
| 120 |
+ running_tasks = self._prefetch_queue.running_tasks |
| 121 |
poll = self._poll.poll |
| 122 |
|
| 123 |
self._schedule_tasks() |
| 124 |
@@ -7426,27 +7462,6 @@ |
| 125 |
# handler, so it's time to yield. |
| 126 |
break |
| 127 |
|
| 128 |
- def _schedule_tasks(self): |
| 129 |
- task_queue = self._task_queue |
| 130 |
- running_tasks = self._running_tasks |
| 131 |
- max_jobs = self._max_jobs |
| 132 |
- state_changed = False |
| 133 |
- |
| 134 |
- for task in list(running_tasks): |
| 135 |
- if not task.registered and task.poll() is not None: |
| 136 |
- running_tasks.remove(task) |
| 137 |
- state_changed = True |
| 138 |
- |
| 139 |
- while task_queue and (len(running_tasks) < max_jobs): |
| 140 |
- task = task_queue.popleft() |
| 141 |
- cancelled = getattr(task, "cancelled", None) |
| 142 |
- if not cancelled: |
| 143 |
- task.start() |
| 144 |
- running_tasks.add(task) |
| 145 |
- state_changed = True |
| 146 |
- |
| 147 |
- return state_changed |
| 148 |
- |
| 149 |
def _world_atom(self, pkg): |
| 150 |
""" |
| 151 |
Add the package to the world file, but only if |
| 152 |
|
| 153 |
-- |
| 154 |
gentoo-commits@l.g.o mailing list |
Archives