1 |
Add a terminate method to FetchIterator so that it doesn't |
2 |
delay termination of emirrordist via SIGINT. Otherwise it |
3 |
it is possible for the __iter__ method to loop for a very |
4 |
long time after SIGINT has been delivered. For example, |
5 |
this could happen if there are many ebuilds with stale |
6 |
cache and RESTRICT=mirror. This issue was discovered |
7 |
during testing of changes to the MirrorDistTask.terminate |
8 |
implementation. |
9 |
--- |
10 |
pym/portage/_emirrordist/FetchIterator.py | 21 +++++++++++++++++++++ |
11 |
pym/portage/_emirrordist/MirrorDistTask.py | 6 +++++- |
12 |
2 files changed, 26 insertions(+), 1 deletion(-) |
13 |
|
14 |
diff --git a/pym/portage/_emirrordist/FetchIterator.py b/pym/portage/_emirrordist/FetchIterator.py |
15 |
index 16a0b04..3841979 100644 |
16 |
--- a/pym/portage/_emirrordist/FetchIterator.py |
17 |
+++ b/pym/portage/_emirrordist/FetchIterator.py |
18 |
@@ -1,6 +1,8 @@ |
19 |
# Copyright 2013 Gentoo Foundation |
20 |
# Distributed under the terms of the GNU General Public License v2 |
21 |
|
22 |
+import threading |
23 |
+ |
24 |
from portage import os |
25 |
from portage.checksum import (_apply_hash_filter, |
26 |
_filter_unaccelarated_hashes, _hash_filter) |
27 |
@@ -13,6 +15,19 @@ class FetchIterator(object): |
28 |
def __init__(self, config): |
29 |
self._config = config |
30 |
self._log_failure = config.log_failure |
31 |
+ self._terminated = threading.Event() |
32 |
+ |
33 |
+ def terminate(self): |
34 |
+ """ |
35 |
+ Schedules early termination of the __iter__ method, which is |
36 |
+ useful because under some conditions it's possible for __iter__ |
37 |
+ to loop for a long time without yielding to the caller. For |
38 |
+ example, it's useful when there are many ebuilds with stale |
39 |
+ cache and RESTRICT=mirror. |
40 |
+ |
41 |
+ This method is thread-safe (and safe for signal handlers). |
42 |
+ """ |
43 |
+ self._terminated.set() |
44 |
|
45 |
def _iter_every_cp(self): |
46 |
# List categories individually, in order to start yielding quicker, |
47 |
@@ -37,6 +52,9 @@ class FetchIterator(object): |
48 |
|
49 |
for cp in self._iter_every_cp(): |
50 |
|
51 |
+ if self._terminated.is_set(): |
52 |
+ return |
53 |
+ |
54 |
for tree in portdb.porttrees: |
55 |
|
56 |
# Reset state so the Manifest is pulled once |
57 |
@@ -46,6 +64,9 @@ class FetchIterator(object): |
58 |
|
59 |
for cpv in portdb.cp_list(cp, mytree=tree): |
60 |
|
61 |
+ if self._terminated.is_set(): |
62 |
+ return |
63 |
+ |
64 |
try: |
65 |
restrict, = portdb.aux_get(cpv, ("RESTRICT",), |
66 |
mytree=tree) |
67 |
diff --git a/pym/portage/_emirrordist/MirrorDistTask.py b/pym/portage/_emirrordist/MirrorDistTask.py |
68 |
index 0702eb1..8da9f06 100644 |
69 |
--- a/pym/portage/_emirrordist/MirrorDistTask.py |
70 |
+++ b/pym/portage/_emirrordist/MirrorDistTask.py |
71 |
@@ -32,9 +32,11 @@ class MirrorDistTask(CompositeTask): |
72 |
self._config = config |
73 |
self._term_rlock = threading.RLock() |
74 |
self._term_callback_handle = None |
75 |
+ self._fetch_iterator = None |
76 |
|
77 |
def _start(self): |
78 |
- fetch = TaskScheduler(iter(FetchIterator(self._config)), |
79 |
+ self._fetch_iterator = FetchIterator(self._config) |
80 |
+ fetch = TaskScheduler(iter(self._fetch_iterator), |
81 |
max_jobs=self._config.options.jobs, |
82 |
max_load=self._config.options.load_average, |
83 |
event_loop=self._config.event_loop) |
84 |
@@ -226,6 +228,8 @@ class MirrorDistTask(CompositeTask): |
85 |
self._term_callback) |
86 |
|
87 |
def _term_callback(self): |
88 |
+ if self._fetch_iterator is not None: |
89 |
+ self._fetch_iterator.terminate() |
90 |
self.cancel() |
91 |
self.wait() |
92 |
|
93 |
-- |
94 |
2.10.2 |