1 |
Since key refresh is prone to failure, retry using exponential |
2 |
backoff with random jitter. This adds the following sync-openpgp-* |
3 |
configuration settings: |
4 |
|
5 |
sync-openpgp-key-refresh-retry-count = 40 |
6 |
|
7 |
Maximum number of times to retry key refresh if it fails. Between |
8 |
each key refresh attempt, there is an exponential delay with a |
9 |
constant multiplier and a uniform random multiplier between 0 and 1. |
10 |
|
11 |
sync-openpgp-key-refresh-retry-delay-exp-base = 2 |
12 |
|
13 |
The base of the exponential expression. The exponent is the number |
14 |
of previous refresh attempts. |
15 |
|
16 |
sync-openpgp-key-refresh-retry-delay-max = 60 |
17 |
|
18 |
Maximum delay between each retry attempt, in units of seconds. This |
19 |
places a limit on the length of the exponential delay. |
20 |
|
21 |
sync-openpgp-key-refresh-retry-delay-mult = 4 |
22 |
|
23 |
Multiplier for the exponential delay. |
24 |
|
25 |
sync-openpgp-key-refresh-retry-overall-timeout = 1200 |
26 |
|
27 |
Combined time limit for all refresh attempts, in units of seconds. |
28 |
|
29 |
Bug: https://bugs.gentoo.org/649276 |
30 |
--- |
31 |
cnf/repos.conf | 5 ++ |
32 |
man/portage.5 | 19 ++++++++ |
33 |
pym/portage/repository/config.py | 22 +++++++++ |
34 |
pym/portage/sync/modules/rsync/rsync.py | 16 ++++++- |
35 |
pym/portage/sync/syncbase.py | 85 ++++++++++++++++++++++++++++++++- |
36 |
5 files changed, 144 insertions(+), 3 deletions(-) |
37 |
|
38 |
diff --git a/cnf/repos.conf b/cnf/repos.conf |
39 |
index 984ecd220..5759b8b43 100644 |
40 |
--- a/cnf/repos.conf |
41 |
+++ b/cnf/repos.conf |
42 |
@@ -9,6 +9,11 @@ auto-sync = yes |
43 |
sync-rsync-verify-metamanifest = yes |
44 |
sync-rsync-verify-max-age = 24 |
45 |
sync-openpgp-key-path = /var/lib/gentoo/gkeys/keyrings/gentoo/release/pubring.gpg |
46 |
+sync-openpgp-key-refresh-retry-count = 40 |
47 |
+sync-openpgp-key-refresh-retry-overall-timeout = 1200 |
48 |
+sync-openpgp-key-refresh-retry-delay-exp-base = 2 |
49 |
+sync-openpgp-key-refresh-retry-delay-max = 60 |
50 |
+sync-openpgp-key-refresh-retry-delay-mult = 4 |
51 |
|
52 |
# for daily squashfs snapshots |
53 |
#sync-type = squashdelta |
54 |
diff --git a/man/portage.5 b/man/portage.5 |
55 |
index 549c51c73..d3e258a43 100644 |
56 |
--- a/man/portage.5 |
57 |
+++ b/man/portage.5 |
58 |
@@ -1081,6 +1081,25 @@ only for protocols supporting cryptographic verification, provided |
59 |
that the respective verification option is enabled. If unset, the user's |
60 |
keyring is used. |
61 |
.TP |
62 |
+.B sync\-openpgp\-key\-refresh\-retry\-count = 40 |
63 |
+Maximum number of times to retry key refresh if it fails. Between each |
64 |
+key refresh attempt, there is an exponential delay with a constant |
65 |
+multiplier and a uniform random multiplier between 0 and 1. |
66 |
+.TP |
67 |
+.B sync\-openpgp\-key\-refresh\-retry\-delay\-exp\-base = 2 |
68 |
+The base of the exponential expression. The exponent is the number of |
69 |
+previous refresh attempts. |
70 |
+.TP |
71 |
+.B sync\-openpgp\-key\-refresh\-retry\-delay\-max = 60 |
72 |
+Maximum delay between each retry attempt, in units of seconds. This |
73 |
+places a limit on the length of the exponential delay. |
74 |
+.TP |
75 |
+.B sync\-openpgp\-key\-refresh\-retry\-delay\-mult = 4 |
76 |
+Multiplier for the exponential delay. |
77 |
+.TP |
78 |
+.B sync\-openpgp\-key\-refresh\-retry\-overall\-timeout = 1200 |
79 |
+Combined time limit for all refresh attempts, in units of seconds. |
80 |
+.TP |
81 |
.B sync-rsync-vcs-ignore = true|false |
82 |
Ignore vcs directories that may be present in the repository. It is the |
83 |
user's responsibility to set sync-rsync-extra-opts to protect vcs |
84 |
diff --git a/pym/portage/repository/config.py b/pym/portage/repository/config.py |
85 |
index b5db4855f..1d897bb90 100644 |
86 |
--- a/pym/portage/repository/config.py |
87 |
+++ b/pym/portage/repository/config.py |
88 |
@@ -87,6 +87,11 @@ class RepoConfig(object): |
89 |
'update_changelog', '_eapis_banned', '_eapis_deprecated', |
90 |
'_masters_orig', 'module_specific_options', 'manifest_required_hashes', |
91 |
'sync_openpgp_key_path', |
92 |
+ 'sync_openpgp_key_refresh_retry_count', |
93 |
+ 'sync_openpgp_key_refresh_retry_delay_max', |
94 |
+ 'sync_openpgp_key_refresh_retry_delay_exp_base', |
95 |
+ 'sync_openpgp_key_refresh_retry_delay_mult', |
96 |
+ 'sync_openpgp_key_refresh_retry_overall_timeout', |
97 |
) |
98 |
|
99 |
def __init__(self, name, repo_opts, local_config=True): |
100 |
@@ -186,6 +191,13 @@ class RepoConfig(object): |
101 |
self.sync_openpgp_key_path = repo_opts.get( |
102 |
'sync-openpgp-key-path', None) |
103 |
|
104 |
+ for k in ('sync_openpgp_key_refresh_retry_count', |
105 |
+ 'sync_openpgp_key_refresh_retry_delay_max', |
106 |
+ 'sync_openpgp_key_refresh_retry_delay_exp_base', |
107 |
+ 'sync_openpgp_key_refresh_retry_delay_mult', |
108 |
+ 'sync_openpgp_key_refresh_retry_overall_timeout'): |
109 |
+ setattr(self, k, repo_opts.get(k.replace('_', '-'), None)) |
110 |
+ |
111 |
self.module_specific_options = {} |
112 |
|
113 |
# Not implemented. |
114 |
@@ -523,6 +535,11 @@ class RepoConfigLoader(object): |
115 |
'force', 'masters', 'priority', 'strict_misc_digests', |
116 |
'sync_depth', 'sync_hooks_only_on_change', |
117 |
'sync_openpgp_key_path', |
118 |
+ 'sync_openpgp_key_refresh_retry_count', |
119 |
+ 'sync_openpgp_key_refresh_retry_delay_max', |
120 |
+ 'sync_openpgp_key_refresh_retry_delay_exp_base', |
121 |
+ 'sync_openpgp_key_refresh_retry_delay_mult', |
122 |
+ 'sync_openpgp_key_refresh_retry_overall_timeout', |
123 |
'sync_type', 'sync_umask', 'sync_uri', 'sync_user', |
124 |
'module_specific_options'): |
125 |
v = getattr(repos_conf_opts, k, None) |
126 |
@@ -946,6 +963,11 @@ class RepoConfigLoader(object): |
127 |
bool_keys = ("strict_misc_digests",) |
128 |
str_or_int_keys = ("auto_sync", "clone_depth", "format", "location", |
129 |
"main_repo", "priority", "sync_depth", "sync_openpgp_key_path", |
130 |
+ "sync_openpgp_key_refresh_retry_count", |
131 |
+ "sync_openpgp_key_refresh_retry_delay_max", |
132 |
+ "sync_openpgp_key_refresh_retry_delay_exp_base", |
133 |
+ "sync_openpgp_key_refresh_retry_delay_mult", |
134 |
+ "sync_openpgp_key_refresh_retry_overall_timeout", |
135 |
"sync_type", "sync_umask", "sync_uri", 'sync_user') |
136 |
str_tuple_keys = ("aliases", "eclass_overrides", "force") |
137 |
repo_config_tuple_keys = ("masters",) |
138 |
diff --git a/pym/portage/sync/modules/rsync/rsync.py b/pym/portage/sync/modules/rsync/rsync.py |
139 |
index ac841545d..763f41699 100644 |
140 |
--- a/pym/portage/sync/modules/rsync/rsync.py |
141 |
+++ b/pym/portage/sync/modules/rsync/rsync.py |
142 |
@@ -7,6 +7,7 @@ import time |
143 |
import signal |
144 |
import socket |
145 |
import datetime |
146 |
+import functools |
147 |
import io |
148 |
import re |
149 |
import random |
150 |
@@ -22,7 +23,9 @@ good = create_color_func("GOOD") |
151 |
bad = create_color_func("BAD") |
152 |
warn = create_color_func("WARN") |
153 |
from portage.const import VCS_DIRS, TIMESTAMP_FORMAT, RSYNC_PACKAGE_ATOM |
154 |
+from portage.util._eventloop.global_event_loop import global_event_loop |
155 |
from portage.util import writemsg, writemsg_stdout |
156 |
+from portage.util.futures.futures import TimeoutError |
157 |
from portage.sync.getaddrinfo_validate import getaddrinfo_validate |
158 |
from _emerge.UserQuery import UserQuery |
159 |
from portage.sync.syncbase import NewBase |
160 |
@@ -139,14 +142,23 @@ class RsyncSync(NewBase): |
161 |
# will not be performed and the user will have to fix it and try again, |
162 |
# so we may as well bail out before actual rsync happens. |
163 |
if openpgp_env is not None and self.repo.sync_openpgp_key_path is not None: |
164 |
+ |
165 |
try: |
166 |
out.einfo('Using keys from %s' % (self.repo.sync_openpgp_key_path,)) |
167 |
with io.open(self.repo.sync_openpgp_key_path, 'rb') as f: |
168 |
openpgp_env.import_key(f) |
169 |
out.ebegin('Refreshing keys from keyserver') |
170 |
- openpgp_env.refresh_keys() |
171 |
+ retry_decorator = self._key_refresh_retry_decorator() |
172 |
+ if retry_decorator is None: |
173 |
+ openpgp_env.refresh_keys() |
174 |
+ else: |
175 |
+ loop = global_event_loop() |
176 |
+ func_coroutine = functools.partial(loop.run_in_executor, |
177 |
+ None, openpgp_env.refresh_keys) |
178 |
+ decorated_func = retry_decorator(func_coroutine) |
179 |
+ loop.run_until_complete(decorated_func()) |
180 |
out.eend(0) |
181 |
- except GematoException as e: |
182 |
+ except (GematoException, TimeoutError) as e: |
183 |
writemsg_level("!!! Manifest verification impossible due to keyring problem:\n%s\n" |
184 |
% (e,), |
185 |
level=logging.ERROR, noiselevel=-1) |
186 |
diff --git a/pym/portage/sync/syncbase.py b/pym/portage/sync/syncbase.py |
187 |
index 43b667fb0..9515dd8d8 100644 |
188 |
--- a/pym/portage/sync/syncbase.py |
189 |
+++ b/pym/portage/sync/syncbase.py |
190 |
@@ -6,12 +6,14 @@ Base class for performing sync operations. |
191 |
This class contains common initialization code and functions. |
192 |
''' |
193 |
|
194 |
- |
195 |
+from __future__ import unicode_literals |
196 |
import logging |
197 |
import os |
198 |
|
199 |
import portage |
200 |
from portage.util import writemsg_level |
201 |
+from portage.util.backoff import RandomExponentialBackoff |
202 |
+from portage.util.futures.retry import retry |
203 |
from . import _SUBMODULE_PATH_MAP |
204 |
|
205 |
class SyncBase(object): |
206 |
@@ -106,6 +108,87 @@ class SyncBase(object): |
207 |
'''Get information about the head commit''' |
208 |
raise NotImplementedError |
209 |
|
210 |
+ def _key_refresh_retry_decorator(self): |
211 |
+ ''' |
212 |
+ Return a retry decorator, or None if retry is disabled. |
213 |
+ |
214 |
+ If retry fails, the function reraises the exception raised |
215 |
+ by the decorated function. If retry times out and no exception |
216 |
+ is available to reraise, the function raises TimeoutError. |
217 |
+ ''' |
218 |
+ errors = [] |
219 |
+ |
220 |
+ if self.repo.sync_openpgp_key_refresh_retry_count is None: |
221 |
+ return None |
222 |
+ try: |
223 |
+ retry_count = int(self.repo.sync_openpgp_key_refresh_retry_count) |
224 |
+ except Exception as e: |
225 |
+ errors.append('sync-openpgp-key-refresh-retry-count: {}'.format(e)) |
226 |
+ else: |
227 |
+ if retry_count <= 0: |
228 |
+ return None |
229 |
+ |
230 |
+ if self.repo.sync_openpgp_key_refresh_retry_overall_timeout is None: |
231 |
+ retry_overall_timeout = None |
232 |
+ else: |
233 |
+ try: |
234 |
+ retry_overall_timeout = float(self.repo.sync_openpgp_key_refresh_retry_overall_timeout) |
235 |
+ except Exception as e: |
236 |
+ errors.append('sync-openpgp-key-refresh-retry-overall-timeout: {}'.format(e)) |
237 |
+ else: |
238 |
+ if retry_overall_timeout < 0: |
239 |
+ errors.append('sync-openpgp-key-refresh-retry-overall-timeout: ' |
240 |
+ 'value must be greater than or equal to zero: {}'.format(retry_overall_timeout)) |
241 |
+ elif retry_overall_timeout == 0: |
242 |
+ retry_overall_timeout = None |
243 |
+ |
244 |
+ if self.repo.sync_openpgp_key_refresh_retry_delay_mult is None: |
245 |
+ retry_delay_mult = None |
246 |
+ else: |
247 |
+ try: |
248 |
+ retry_delay_mult = float(self.repo.sync_openpgp_key_refresh_retry_delay_mult) |
249 |
+ except Exception as e: |
250 |
+ errors.append('sync-openpgp-key-refresh-retry-delay-mult: {}'.format(e)) |
251 |
+ else: |
252 |
+ if retry_delay_mult <= 0: |
253 |
+ errors.append('sync-openpgp-key-refresh-retry-mult: ' |
254 |
+ 'value must be greater than zero: {}'.format(retry_delay_mult)) |
255 |
+ |
256 |
+ if self.repo.sync_openpgp_key_refresh_retry_delay_exp_base is None: |
257 |
+ retry_delay_exp_base = None |
258 |
+ else: |
259 |
+ try: |
260 |
+ retry_delay_exp_base = float(self.repo.sync_openpgp_key_refresh_retry_delay_exp_base) |
261 |
+ except Exception as e: |
262 |
+ errors.append('sync-openpgp-key-refresh-retry-delay-exp: {}'.format(e)) |
263 |
+ else: |
264 |
+ if retry_delay_exp_base <= 0: |
265 |
+ errors.append('sync-openpgp-key-refresh-retry-delay-exp: ' |
266 |
+ 'value must be greater than zero: {}'.format(retry_delay_mult)) |
267 |
+ |
268 |
+ if errors: |
269 |
+ lines = [] |
270 |
+ lines.append('') |
271 |
+ lines.append('!!! Retry disabled for openpgp key refresh:') |
272 |
+ lines.append('') |
273 |
+ for msg in errors: |
274 |
+ lines.append(' {}'.format(msg)) |
275 |
+ lines.append('') |
276 |
+ |
277 |
+ for line in lines: |
278 |
+ writemsg_level("{}\n".format(line), |
279 |
+ level=logging.ERROR, noiselevel=-1) |
280 |
+ |
281 |
+ return None |
282 |
+ |
283 |
+ return retry( |
284 |
+ reraise=True, |
285 |
+ try_max=retry_count, |
286 |
+ overall_timeout=(retry_overall_timeout if retry_overall_timeout > 0 else None), |
287 |
+ delay_func=RandomExponentialBackoff( |
288 |
+ multiplier=(1 if retry_delay_mult is None else retry_delay_mult), |
289 |
+ base=(2 if retry_delay_exp_base is None else retry_delay_exp_base))) |
290 |
+ |
291 |
|
292 |
class NewBase(SyncBase): |
293 |
'''Subclasses Syncbase adding a new() and runs it |
294 |
-- |
295 |
2.13.6 |