1 |
commit: 5f29f03d0d9714082fabbbae5cead8857fbc9093 |
2 |
Author: Zac Medico <zmedico <AT> gentoo <DOT> org> |
3 |
AuthorDate: Fri Mar 30 08:12:19 2018 +0000 |
4 |
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org> |
5 |
CommitDate: Mon Apr 2 16:53:24 2018 +0000 |
6 |
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=5f29f03d |
7 |
|
8 |
rsync: add key refresh retry (bug 649276) |
9 |
|
10 |
Since key refresh is prone to failure, retry using exponential |
11 |
backoff with random jitter. This adds the following sync-openpgp-* |
12 |
configuration settings: |
13 |
|
14 |
sync-openpgp-key-refresh-retry-count = 40 |
15 |
|
16 |
Maximum number of times to retry key refresh if it fails. Between |
17 |
each key refresh attempt, there is an exponential delay with a |
18 |
constant multiplier and a uniform random multiplier between 0 and 1. |
19 |
|
20 |
sync-openpgp-key-refresh-retry-delay-exp-base = 2 |
21 |
|
22 |
The base of the exponential expression. The exponent is the number |
23 |
of previous refresh attempts. |
24 |
|
25 |
sync-openpgp-key-refresh-retry-delay-max = 60 |
26 |
|
27 |
Maximum delay between each retry attempt, in units of seconds. This |
28 |
places a limit on the length of the exponential delay. |
29 |
|
30 |
sync-openpgp-key-refresh-retry-delay-mult = 4 |
31 |
|
32 |
Multiplier for the exponential delay. |
33 |
|
34 |
sync-openpgp-key-refresh-retry-overall-timeout = 1200 |
35 |
|
36 |
Combined time limit for all refresh attempts, in units of seconds. |
37 |
|
38 |
Bug: https://bugs.gentoo.org/649276 |
39 |
|
40 |
cnf/repos.conf | 5 ++ |
41 |
man/portage.5 | 21 +++++++- |
42 |
pym/portage/repository/config.py | 22 +++++++++ |
43 |
pym/portage/sync/modules/rsync/rsync.py | 16 +++++- |
44 |
pym/portage/sync/syncbase.py | 87 ++++++++++++++++++++++++++++++++- |
45 |
5 files changed, 146 insertions(+), 5 deletions(-) |
46 |
|
47 |
diff --git a/cnf/repos.conf b/cnf/repos.conf |
48 |
index 984ecd220..5759b8b43 100644 |
49 |
--- a/cnf/repos.conf |
50 |
+++ b/cnf/repos.conf |
51 |
@@ -9,6 +9,11 @@ auto-sync = yes |
52 |
sync-rsync-verify-metamanifest = yes |
53 |
sync-rsync-verify-max-age = 24 |
54 |
sync-openpgp-key-path = /var/lib/gentoo/gkeys/keyrings/gentoo/release/pubring.gpg |
55 |
+sync-openpgp-key-refresh-retry-count = 40 |
56 |
+sync-openpgp-key-refresh-retry-overall-timeout = 1200 |
57 |
+sync-openpgp-key-refresh-retry-delay-exp-base = 2 |
58 |
+sync-openpgp-key-refresh-retry-delay-max = 60 |
59 |
+sync-openpgp-key-refresh-retry-delay-mult = 4 |
60 |
|
61 |
# for daily squashfs snapshots |
62 |
#sync-type = squashdelta |
63 |
|
64 |
diff --git a/man/portage.5 b/man/portage.5 |
65 |
index 549c51c73..2c3a75ccd 100644 |
66 |
--- a/man/portage.5 |
67 |
+++ b/man/portage.5 |
68 |
@@ -1,4 +1,4 @@ |
69 |
-.TH "PORTAGE" "31" "May 2017" "Portage VERSION" "Portage" |
70 |
+.TH "PORTAGE" "31" "Apr 2018" "Portage VERSION" "Portage" |
71 |
.SH NAME |
72 |
portage \- the heart of Gentoo |
73 |
.SH "DESCRIPTION" |
74 |
@@ -1081,6 +1081,25 @@ only for protocols supporting cryptographic verification, provided |
75 |
that the respective verification option is enabled. If unset, the user's |
76 |
keyring is used. |
77 |
.TP |
78 |
+.B sync\-openpgp\-key\-refresh\-retry\-count = 40 |
79 |
+Maximum number of times to retry key refresh if it fails. Between each |
80 |
+key refresh attempt, there is an exponential delay with a constant |
81 |
+multiplier and a uniform random multiplier between 0 and 1. |
82 |
+.TP |
83 |
+.B sync\-openpgp\-key\-refresh\-retry\-delay\-exp\-base = 2 |
84 |
+The base of the exponential expression. The exponent is the number of |
85 |
+previous refresh attempts. |
86 |
+.TP |
87 |
+.B sync\-openpgp\-key\-refresh\-retry\-delay\-max = 60 |
88 |
+Maximum delay between each retry attempt, in units of seconds. This |
89 |
+places a limit on the length of the exponential delay. |
90 |
+.TP |
91 |
+.B sync\-openpgp\-key\-refresh\-retry\-delay\-mult = 4 |
92 |
+Multiplier for the exponential delay. |
93 |
+.TP |
94 |
+.B sync\-openpgp\-key\-refresh\-retry\-overall\-timeout = 1200 |
95 |
+Combined time limit for all refresh attempts, in units of seconds. |
96 |
+.TP |
97 |
.B sync-rsync-vcs-ignore = true|false |
98 |
Ignore vcs directories that may be present in the repository. It is the |
99 |
user's responsibility to set sync-rsync-extra-opts to protect vcs |
100 |
|
101 |
diff --git a/pym/portage/repository/config.py b/pym/portage/repository/config.py |
102 |
index b5db4855f..1d897bb90 100644 |
103 |
--- a/pym/portage/repository/config.py |
104 |
+++ b/pym/portage/repository/config.py |
105 |
@@ -87,6 +87,11 @@ class RepoConfig(object): |
106 |
'update_changelog', '_eapis_banned', '_eapis_deprecated', |
107 |
'_masters_orig', 'module_specific_options', 'manifest_required_hashes', |
108 |
'sync_openpgp_key_path', |
109 |
+ 'sync_openpgp_key_refresh_retry_count', |
110 |
+ 'sync_openpgp_key_refresh_retry_delay_max', |
111 |
+ 'sync_openpgp_key_refresh_retry_delay_exp_base', |
112 |
+ 'sync_openpgp_key_refresh_retry_delay_mult', |
113 |
+ 'sync_openpgp_key_refresh_retry_overall_timeout', |
114 |
) |
115 |
|
116 |
def __init__(self, name, repo_opts, local_config=True): |
117 |
@@ -186,6 +191,13 @@ class RepoConfig(object): |
118 |
self.sync_openpgp_key_path = repo_opts.get( |
119 |
'sync-openpgp-key-path', None) |
120 |
|
121 |
+ for k in ('sync_openpgp_key_refresh_retry_count', |
122 |
+ 'sync_openpgp_key_refresh_retry_delay_max', |
123 |
+ 'sync_openpgp_key_refresh_retry_delay_exp_base', |
124 |
+ 'sync_openpgp_key_refresh_retry_delay_mult', |
125 |
+ 'sync_openpgp_key_refresh_retry_overall_timeout'): |
126 |
+ setattr(self, k, repo_opts.get(k.replace('_', '-'), None)) |
127 |
+ |
128 |
self.module_specific_options = {} |
129 |
|
130 |
# Not implemented. |
131 |
@@ -523,6 +535,11 @@ class RepoConfigLoader(object): |
132 |
'force', 'masters', 'priority', 'strict_misc_digests', |
133 |
'sync_depth', 'sync_hooks_only_on_change', |
134 |
'sync_openpgp_key_path', |
135 |
+ 'sync_openpgp_key_refresh_retry_count', |
136 |
+ 'sync_openpgp_key_refresh_retry_delay_max', |
137 |
+ 'sync_openpgp_key_refresh_retry_delay_exp_base', |
138 |
+ 'sync_openpgp_key_refresh_retry_delay_mult', |
139 |
+ 'sync_openpgp_key_refresh_retry_overall_timeout', |
140 |
'sync_type', 'sync_umask', 'sync_uri', 'sync_user', |
141 |
'module_specific_options'): |
142 |
v = getattr(repos_conf_opts, k, None) |
143 |
@@ -946,6 +963,11 @@ class RepoConfigLoader(object): |
144 |
bool_keys = ("strict_misc_digests",) |
145 |
str_or_int_keys = ("auto_sync", "clone_depth", "format", "location", |
146 |
"main_repo", "priority", "sync_depth", "sync_openpgp_key_path", |
147 |
+ "sync_openpgp_key_refresh_retry_count", |
148 |
+ "sync_openpgp_key_refresh_retry_delay_max", |
149 |
+ "sync_openpgp_key_refresh_retry_delay_exp_base", |
150 |
+ "sync_openpgp_key_refresh_retry_delay_mult", |
151 |
+ "sync_openpgp_key_refresh_retry_overall_timeout", |
152 |
"sync_type", "sync_umask", "sync_uri", 'sync_user') |
153 |
str_tuple_keys = ("aliases", "eclass_overrides", "force") |
154 |
repo_config_tuple_keys = ("masters",) |
155 |
|
156 |
diff --git a/pym/portage/sync/modules/rsync/rsync.py b/pym/portage/sync/modules/rsync/rsync.py |
157 |
index ac841545d..763f41699 100644 |
158 |
--- a/pym/portage/sync/modules/rsync/rsync.py |
159 |
+++ b/pym/portage/sync/modules/rsync/rsync.py |
160 |
@@ -7,6 +7,7 @@ import time |
161 |
import signal |
162 |
import socket |
163 |
import datetime |
164 |
+import functools |
165 |
import io |
166 |
import re |
167 |
import random |
168 |
@@ -22,7 +23,9 @@ good = create_color_func("GOOD") |
169 |
bad = create_color_func("BAD") |
170 |
warn = create_color_func("WARN") |
171 |
from portage.const import VCS_DIRS, TIMESTAMP_FORMAT, RSYNC_PACKAGE_ATOM |
172 |
+from portage.util._eventloop.global_event_loop import global_event_loop |
173 |
from portage.util import writemsg, writemsg_stdout |
174 |
+from portage.util.futures.futures import TimeoutError |
175 |
from portage.sync.getaddrinfo_validate import getaddrinfo_validate |
176 |
from _emerge.UserQuery import UserQuery |
177 |
from portage.sync.syncbase import NewBase |
178 |
@@ -139,14 +142,23 @@ class RsyncSync(NewBase): |
179 |
# will not be performed and the user will have to fix it and try again, |
180 |
# so we may as well bail out before actual rsync happens. |
181 |
if openpgp_env is not None and self.repo.sync_openpgp_key_path is not None: |
182 |
+ |
183 |
try: |
184 |
out.einfo('Using keys from %s' % (self.repo.sync_openpgp_key_path,)) |
185 |
with io.open(self.repo.sync_openpgp_key_path, 'rb') as f: |
186 |
openpgp_env.import_key(f) |
187 |
out.ebegin('Refreshing keys from keyserver') |
188 |
- openpgp_env.refresh_keys() |
189 |
+ retry_decorator = self._key_refresh_retry_decorator() |
190 |
+ if retry_decorator is None: |
191 |
+ openpgp_env.refresh_keys() |
192 |
+ else: |
193 |
+ loop = global_event_loop() |
194 |
+ func_coroutine = functools.partial(loop.run_in_executor, |
195 |
+ None, openpgp_env.refresh_keys) |
196 |
+ decorated_func = retry_decorator(func_coroutine) |
197 |
+ loop.run_until_complete(decorated_func()) |
198 |
out.eend(0) |
199 |
- except GematoException as e: |
200 |
+ except (GematoException, TimeoutError) as e: |
201 |
writemsg_level("!!! Manifest verification impossible due to keyring problem:\n%s\n" |
202 |
% (e,), |
203 |
level=logging.ERROR, noiselevel=-1) |
204 |
|
205 |
diff --git a/pym/portage/sync/syncbase.py b/pym/portage/sync/syncbase.py |
206 |
index 43b667fb0..7d4d33272 100644 |
207 |
--- a/pym/portage/sync/syncbase.py |
208 |
+++ b/pym/portage/sync/syncbase.py |
209 |
@@ -1,4 +1,4 @@ |
210 |
-# Copyright 2014-2015 Gentoo Foundation |
211 |
+# Copyright 2014-2018 Gentoo Foundation |
212 |
# Distributed under the terms of the GNU General Public License v2 |
213 |
|
214 |
''' |
215 |
@@ -6,12 +6,14 @@ Base class for performing sync operations. |
216 |
This class contains common initialization code and functions. |
217 |
''' |
218 |
|
219 |
- |
220 |
+from __future__ import unicode_literals |
221 |
import logging |
222 |
import os |
223 |
|
224 |
import portage |
225 |
from portage.util import writemsg_level |
226 |
+from portage.util.backoff import RandomExponentialBackoff |
227 |
+from portage.util.futures.retry import retry |
228 |
from . import _SUBMODULE_PATH_MAP |
229 |
|
230 |
class SyncBase(object): |
231 |
@@ -106,6 +108,87 @@ class SyncBase(object): |
232 |
'''Get information about the head commit''' |
233 |
raise NotImplementedError |
234 |
|
235 |
+ def _key_refresh_retry_decorator(self): |
236 |
+ ''' |
237 |
+ Return a retry decorator, or None if retry is disabled. |
238 |
+ |
239 |
+ If retry fails, the function reraises the exception raised |
240 |
+ by the decorated function. If retry times out and no exception |
241 |
+ is available to reraise, the function raises TimeoutError. |
242 |
+ ''' |
243 |
+ errors = [] |
244 |
+ |
245 |
+ if self.repo.sync_openpgp_key_refresh_retry_count is None: |
246 |
+ return None |
247 |
+ try: |
248 |
+ retry_count = int(self.repo.sync_openpgp_key_refresh_retry_count) |
249 |
+ except Exception as e: |
250 |
+ errors.append('sync-openpgp-key-refresh-retry-count: {}'.format(e)) |
251 |
+ else: |
252 |
+ if retry_count <= 0: |
253 |
+ return None |
254 |
+ |
255 |
+ if self.repo.sync_openpgp_key_refresh_retry_overall_timeout is None: |
256 |
+ retry_overall_timeout = None |
257 |
+ else: |
258 |
+ try: |
259 |
+ retry_overall_timeout = float(self.repo.sync_openpgp_key_refresh_retry_overall_timeout) |
260 |
+ except Exception as e: |
261 |
+ errors.append('sync-openpgp-key-refresh-retry-overall-timeout: {}'.format(e)) |
262 |
+ else: |
263 |
+ if retry_overall_timeout < 0: |
264 |
+ errors.append('sync-openpgp-key-refresh-retry-overall-timeout: ' |
265 |
+ 'value must be greater than or equal to zero: {}'.format(retry_overall_timeout)) |
266 |
+ elif retry_overall_timeout == 0: |
267 |
+ retry_overall_timeout = None |
268 |
+ |
269 |
+ if self.repo.sync_openpgp_key_refresh_retry_delay_mult is None: |
270 |
+ retry_delay_mult = None |
271 |
+ else: |
272 |
+ try: |
273 |
+ retry_delay_mult = float(self.repo.sync_openpgp_key_refresh_retry_delay_mult) |
274 |
+ except Exception as e: |
275 |
+ errors.append('sync-openpgp-key-refresh-retry-delay-mult: {}'.format(e)) |
276 |
+ else: |
277 |
+ if retry_delay_mult <= 0: |
278 |
+ errors.append('sync-openpgp-key-refresh-retry-mult: ' |
279 |
+ 'value must be greater than zero: {}'.format(retry_delay_mult)) |
280 |
+ |
281 |
+ if self.repo.sync_openpgp_key_refresh_retry_delay_exp_base is None: |
282 |
+ retry_delay_exp_base = None |
283 |
+ else: |
284 |
+ try: |
285 |
+ retry_delay_exp_base = float(self.repo.sync_openpgp_key_refresh_retry_delay_exp_base) |
286 |
+ except Exception as e: |
287 |
+ errors.append('sync-openpgp-key-refresh-retry-delay-exp: {}'.format(e)) |
288 |
+ else: |
289 |
+ if retry_delay_exp_base <= 0: |
290 |
+ errors.append('sync-openpgp-key-refresh-retry-delay-exp: ' |
291 |
+ 'value must be greater than zero: {}'.format(retry_delay_mult)) |
292 |
+ |
293 |
+ if errors: |
294 |
+ lines = [] |
295 |
+ lines.append('') |
296 |
+ lines.append('!!! Retry disabled for openpgp key refresh:') |
297 |
+ lines.append('') |
298 |
+ for msg in errors: |
299 |
+ lines.append(' {}'.format(msg)) |
300 |
+ lines.append('') |
301 |
+ |
302 |
+ for line in lines: |
303 |
+ writemsg_level("{}\n".format(line), |
304 |
+ level=logging.ERROR, noiselevel=-1) |
305 |
+ |
306 |
+ return None |
307 |
+ |
308 |
+ return retry( |
309 |
+ reraise=True, |
310 |
+ try_max=retry_count, |
311 |
+ overall_timeout=(retry_overall_timeout if retry_overall_timeout > 0 else None), |
312 |
+ delay_func=RandomExponentialBackoff( |
313 |
+ multiplier=(1 if retry_delay_mult is None else retry_delay_mult), |
314 |
+ base=(2 if retry_delay_exp_base is None else retry_delay_exp_base))) |
315 |
+ |
316 |
|
317 |
class NewBase(SyncBase): |
318 |
'''Subclasses Syncbase adding a new() and runs it |