1 |
commit: eed71e755fca18ebc271b3a3b72e70c0822f55d2 |
2 |
Author: Brian Dolbec <dolsen <AT> gentoo <DOT> org> |
3 |
AuthorDate: Mon Jul 2 21:57:23 2018 +0000 |
4 |
Commit: Brian Dolbec <dolsen <AT> gentoo <DOT> org> |
5 |
CommitDate: Sat Jul 7 05:22:12 2018 +0000 |
6 |
URL: https://gitweb.gentoo.org/proj/gentoo-keys.git/commit/?id=eed71e75 |
7 |
|
8 |
gkeys: Implement use of the new Fetch class |
9 |
|
10 |
Signed-off-by: Brian Dolbec <dolsen <AT> gentoo.org> |
11 |
|
12 |
Signed-off-by: Brian Dolbec <dolsen <AT> gentoo.org> |
13 |
|
14 |
gkeys/gkeys/actions.py | 51 +++++------------------------------- |
15 |
gkeys/gkeys/base.py | 6 +++++ |
16 |
gkeys/gkeys/seedhandler.py | 64 ++++++++++++++++++++++++++++++++++------------ |
17 |
3 files changed, 61 insertions(+), 60 deletions(-) |
18 |
|
19 |
diff --git a/gkeys/gkeys/actions.py b/gkeys/gkeys/actions.py |
20 |
index 64d6123..0f03880 100644 |
21 |
--- a/gkeys/gkeys/actions.py |
22 |
+++ b/gkeys/gkeys/actions.py |
23 |
@@ -29,6 +29,7 @@ from snakeoil.demandload import demandload |
24 |
|
25 |
demandload( |
26 |
"gkeys.base:Args", |
27 |
+ "gkeys.fetch:Fetch", |
28 |
"json:load", |
29 |
) |
30 |
|
31 |
@@ -718,9 +719,7 @@ class Actions(ActionBase): |
32 |
|
33 |
def installed(self, args): |
34 |
'''Lists the installed key directories''' |
35 |
- if args.category: |
36 |
- keyring = self.config.get_key('keyring') |
37 |
- else: |
38 |
+ if not args.category: |
39 |
return (False, ["Please specify a category."]) |
40 |
catdir = self._set_category(args.category) |
41 |
self.logger.debug("ACTIONS: installed; catdir = %s" % catdir) |
42 |
@@ -818,45 +817,10 @@ class Actions(ActionBase): |
43 |
climit = 0 |
44 |
sig_path = None |
45 |
if isurl: |
46 |
- from sslfetch.connections import Connector |
47 |
- connector_output = { |
48 |
- 'info': self.logger.info, |
49 |
- 'debug': self.logger.debug, |
50 |
- 'error': self.logger.error, |
51 |
- 'exception': self.logger.exception, |
52 |
- # we want any warnings to be printed to the terminal |
53 |
- # so assign it to logging.error |
54 |
- 'warning': self.logger.error, |
55 |
- 'kwargs-info': {}, |
56 |
- 'kwargs-debug': {}, |
57 |
- 'kwargs-error': {}, |
58 |
- 'kwargs-exception': {}, |
59 |
- 'kwargs-warning': {}, |
60 |
- } |
61 |
- fetcher = Connector(connector_output, None, "Gentoo Keys") |
62 |
- self.logger.debug( |
63 |
- _unicode("ACTIONS: verify; fetching %s signed file ") % filepath) |
64 |
- self.logger.debug( |
65 |
- _unicode("ACTIONS: verify; timestamp path: %s") % timestamp_path) |
66 |
- success, signedfile, timestamp = fetcher.fetch_file( |
67 |
- url, filepath, timestamp_path, climit=climit) |
68 |
- if not success: |
69 |
- messages.append(_unicode("File %s cannot be retrieved.") % filepath) |
70 |
- elif '.' + url.rsplit('.', 1)[1] not in EXTENSIONS: |
71 |
- if not signature: |
72 |
- success_fetch = False |
73 |
- for ext in EXTENSIONS: |
74 |
- sig_path = filepath + ext |
75 |
- if isurl: |
76 |
- signature = url + ext |
77 |
- self.logger.debug( |
78 |
- _unicode("ACTIONS: verify; fetching %s signature ") |
79 |
- % signature) |
80 |
- success_fetch, sig, timestamp = fetcher.fetch_file(signature, sig_path) |
81 |
- if success_fetch: |
82 |
- break |
83 |
- else: |
84 |
- signature = None |
85 |
+ fetcher = Fetch(self.logger) |
86 |
+ success, msgs = fetcher.fetch_url(url, filepath, signature, timestamp_path=timestamp_path, |
87 |
+ climit=climit) |
88 |
+ messages.extend(msgs) |
89 |
elif signature is not None and os.path.exists(signature): |
90 |
sig_path = signature |
91 |
else: |
92 |
@@ -864,9 +828,8 @@ class Actions(ActionBase): |
93 |
self.logger.debug( |
94 |
_unicode("ACTIONS: verify; local file %s") % filepath) |
95 |
success = os.path.isfile(filepath) |
96 |
- if (not signature |
97 |
+ if (success and not signature |
98 |
and '.' + filepath.rsplit('.', 1)[-1] not in EXTENSIONS): |
99 |
- success_fetch = False |
100 |
for ext in EXTENSIONS: |
101 |
sig_path = filepath + ext |
102 |
sig_path = os.path.abspath(sig_path) |
103 |
|
104 |
diff --git a/gkeys/gkeys/base.py b/gkeys/gkeys/base.py |
105 |
index a67b330..2de1493 100644 |
106 |
--- a/gkeys/gkeys/base.py |
107 |
+++ b/gkeys/gkeys/base.py |
108 |
@@ -130,6 +130,12 @@ class CliBase(object): |
109 |
action='store_true', default=False, |
110 |
help='Use CASE matching in searches') |
111 |
|
112 |
+ @staticmethod |
113 |
+ def _option_fetcthonly(parser=None): |
114 |
+ parser.add_argument('--fetchonly', |
115 |
+ dest='fetchonly', default=False, |
116 |
+ help="Only fetch the seed file if there is an update or doesn't exist locally") |
117 |
+ |
118 |
@staticmethod |
119 |
def _option_file(parser=None): |
120 |
parser.add_argument('-F', '--file', dest='filename', default=None, |
121 |
|
122 |
diff --git a/gkeys/gkeys/seedhandler.py b/gkeys/gkeys/seedhandler.py |
123 |
index afe5971..cb082ef 100644 |
124 |
--- a/gkeys/gkeys/seedhandler.py |
125 |
+++ b/gkeys/gkeys/seedhandler.py |
126 |
@@ -17,12 +17,15 @@ import re |
127 |
from snakeoil.demandload import demandload |
128 |
|
129 |
from gkeys.gkey import GKEY |
130 |
+from gkeys.lock import LockDir |
131 |
from gkeys.seed import Seeds, decoder |
132 |
|
133 |
demandload( |
134 |
"json:load", |
135 |
"gkeys.exception:UpdateDbError", |
136 |
"gkeys.fileops:ensure_dirs", |
137 |
+ "gkeys.fetch:Fetch", |
138 |
+ "sslfetch.connections:get_timestamp", |
139 |
) |
140 |
|
141 |
|
142 |
@@ -34,6 +37,8 @@ class SeedHandler(object): |
143 |
self.fingerprint_re = re.compile('[0-9A-Fa-f]{40}') |
144 |
self.finerprint_re2 = re.compile('[0-9A-Fa-f]{4}( [0-9A-Fa-f]{4}){9}') |
145 |
self.seeds = None |
146 |
+ self.seedsdir_lock = None |
147 |
+ self.update_lock = None |
148 |
|
149 |
|
150 |
def new(self, args, checkgkey=False): |
151 |
@@ -70,10 +75,10 @@ class SeedHandler(object): |
152 |
if attr in GKEY._fields: |
153 |
keyinfo[attr] = None |
154 |
return keyinfo |
155 |
- |
156 |
+ |
157 |
def compare_seeds(self, seeds1, seeds2) : |
158 |
'''Compares two seed lists and returns the differences |
159 |
- |
160 |
+ |
161 |
@param seeds1: set of seeds to be compared |
162 |
@param seeds2: set of seeds to be compared |
163 |
@return added_gkeys: list of keys that are included in seed2 but not seed1 |
164 |
@@ -98,7 +103,7 @@ class SeedHandler(object): |
165 |
if old_gkey not in new_gkeys and old_gkey not in old_changed_gkeys: |
166 |
removed_gkeys.append(old_gkey) |
167 |
else: |
168 |
- added_gkeys = new_gkeys |
169 |
+ added_gkeys = new_gkeys |
170 |
return(added_gkeys, changed_gkeys, removed_gkeys) |
171 |
|
172 |
def compare_seeds(self, seeds1, seeds2) : |
173 |
@@ -181,7 +186,8 @@ class SeedHandler(object): |
174 |
with open(gkey_path, 'r') as fileseed: |
175 |
seed = load(fileseed) |
176 |
except IOError as error: |
177 |
- self.logger.debug("SeedHandler: load_category; IOError loading seed file %s." % gkey_path) |
178 |
+ self.logger.debug("SeedHandler: load_category; IOError loading seed file %s." |
179 |
+ % gkey_path) |
180 |
self.logger.debug("Error was: %s" % str(error)) |
181 |
if seed: |
182 |
for nick in sorted(seed): |
183 |
@@ -206,6 +212,7 @@ class SeedHandler(object): |
184 |
'''Fetch new seed files |
185 |
|
186 |
@param seeds: list of seed nicks to download |
187 |
+ @param args: argparse namespace instance |
188 |
@param verified_dl: Function pointer to the Actions.verify() |
189 |
instance needed to do the download and verification |
190 |
''' |
191 |
@@ -224,20 +231,45 @@ class SeedHandler(object): |
192 |
except KeyError: |
193 |
pass |
194 |
succeeded = [] |
195 |
- seedsdir = self.config.get_key('seedsdir') |
196 |
+ seedsdir = os.path.join(self.config.get_key('seedsdir')) |
197 |
+ updatedir = os.path.join(seedsdir, "__updates__") |
198 |
mode = int(self.config.get_key('permissions', 'directories'),0) |
199 |
- ensure_dirs(seedsdir, mode=mode) |
200 |
+ ensure_dirs(updatedir, mode=mode) |
201 |
+ self.update_lock = LockDir(updatedir) |
202 |
+ self.update_lock.write_lock() |
203 |
+ fetcher = Fetch(self.logger) |
204 |
for (seed, url, filepath) in urls: |
205 |
- verify_info = self.config.get_key('verify-seeds', seed).split() |
206 |
- args.category = verify_info[0] |
207 |
- args.nick = verify_info[1] |
208 |
- args.filename = url |
209 |
- args.signature = None |
210 |
- args.timestamp = True |
211 |
- args.destination = filepath |
212 |
- verified, messages_ = verified_dl(args) |
213 |
- succeeded.append(verified) |
214 |
- messages.append(messages_) |
215 |
+ tmppath = os.path.join(updatedir, os.path.split(filepath)[-1]) |
216 |
+ # use the real timestamp file for the dl timestamp |
217 |
+ tpath = filepath + ".timestamp" |
218 |
+ # verify the re-fetch cycle timer |
219 |
+ if fetcher.verify_cycle(tpath, climit=60): |
220 |
+ timestamp = get_timestamp(filepath + ".timestamp") |
221 |
+ success, msgs = fetcher.fetch_url(url, tmppath, timestamp=timestamp) |
222 |
+ messages.extend(msgs) |
223 |
+ if success: |
224 |
+ verify_info = self.config.get_key('verify-seeds', seed).split() |
225 |
+ args.category = verify_info[0] |
226 |
+ args.nick = verify_info[1] |
227 |
+ args.filename = url |
228 |
+ args.signature = tmppath + ".timestamp" |
229 |
+ #args.timestamp = True |
230 |
+ args.destination = tmppath |
231 |
+ verified, messages_ = verified_dl(args) |
232 |
+ messages.append(messages_) |
233 |
+ if verified and not args.fetchonly: |
234 |
+ self.seedsdir_lock = LockDir(seedsdir) |
235 |
+ if updateseeds(tmppath, filepath) and updateseeds(args.signature, tpath): |
236 |
+ self.logger.info("Updated seed file...: %s ... OK" % (filepath)) |
237 |
+ succeeded.append(verified) |
238 |
+ else: |
239 |
+ self.logger.info("Updating seed file...: %s ... Failed" % (filepath)) |
240 |
+ succeeded.append(False) |
241 |
+ self.seedsdir_lock.unlock() |
242 |
+ else: |
243 |
+ # sha512sum the 2 files |
244 |
+ pass |
245 |
+ self.update_lock.unlock() |
246 |
return (succeeded, messages) |
247 |
|
248 |
def check_gkey(self, args): |