Gentoo Archives: gentoo-commits

From: Brian Dolbec <dolsen@g.o>
To: gentoo-commits@l.g.o
Subject: [gentoo-commits] proj/gentoo-keys:master commit in: gkeys/gkeys/
Date: Sat, 07 Jul 2018 05:23:58
Message-Id: 1530940932.eed71e755fca18ebc271b3a3b72e70c0822f55d2.dolsen@gentoo
1 commit: eed71e755fca18ebc271b3a3b72e70c0822f55d2
2 Author: Brian Dolbec <dolsen <AT> gentoo <DOT> org>
3 AuthorDate: Mon Jul 2 21:57:23 2018 +0000
4 Commit: Brian Dolbec <dolsen <AT> gentoo <DOT> org>
5 CommitDate: Sat Jul 7 05:22:12 2018 +0000
6 URL: https://gitweb.gentoo.org/proj/gentoo-keys.git/commit/?id=eed71e75
7
8 gkeys: Implement use of the new Fetch class
9
10 Signed-off-by: Brian Dolbec <dolsen <AT> gentoo.org>
11
12 Signed-off-by: Brian Dolbec <dolsen <AT> gentoo.org>
13
14 gkeys/gkeys/actions.py | 51 +++++-------------------------------
15 gkeys/gkeys/base.py | 6 +++++
16 gkeys/gkeys/seedhandler.py | 64 ++++++++++++++++++++++++++++++++++------------
17 3 files changed, 61 insertions(+), 60 deletions(-)
18
19 diff --git a/gkeys/gkeys/actions.py b/gkeys/gkeys/actions.py
20 index 64d6123..0f03880 100644
21 --- a/gkeys/gkeys/actions.py
22 +++ b/gkeys/gkeys/actions.py
23 @@ -29,6 +29,7 @@ from snakeoil.demandload import demandload
24
25 demandload(
26 "gkeys.base:Args",
27 + "gkeys.fetch:Fetch",
28 "json:load",
29 )
30
31 @@ -718,9 +719,7 @@ class Actions(ActionBase):
32
33 def installed(self, args):
34 '''Lists the installed key directories'''
35 - if args.category:
36 - keyring = self.config.get_key('keyring')
37 - else:
38 + if not args.category:
39 return (False, ["Please specify a category."])
40 catdir = self._set_category(args.category)
41 self.logger.debug("ACTIONS: installed; catdir = %s" % catdir)
42 @@ -818,45 +817,10 @@ class Actions(ActionBase):
43 climit = 0
44 sig_path = None
45 if isurl:
46 - from sslfetch.connections import Connector
47 - connector_output = {
48 - 'info': self.logger.info,
49 - 'debug': self.logger.debug,
50 - 'error': self.logger.error,
51 - 'exception': self.logger.exception,
52 - # we want any warnings to be printed to the terminal
53 - # so assign it to logging.error
54 - 'warning': self.logger.error,
55 - 'kwargs-info': {},
56 - 'kwargs-debug': {},
57 - 'kwargs-error': {},
58 - 'kwargs-exception': {},
59 - 'kwargs-warning': {},
60 - }
61 - fetcher = Connector(connector_output, None, "Gentoo Keys")
62 - self.logger.debug(
63 - _unicode("ACTIONS: verify; fetching %s signed file ") % filepath)
64 - self.logger.debug(
65 - _unicode("ACTIONS: verify; timestamp path: %s") % timestamp_path)
66 - success, signedfile, timestamp = fetcher.fetch_file(
67 - url, filepath, timestamp_path, climit=climit)
68 - if not success:
69 - messages.append(_unicode("File %s cannot be retrieved.") % filepath)
70 - elif '.' + url.rsplit('.', 1)[1] not in EXTENSIONS:
71 - if not signature:
72 - success_fetch = False
73 - for ext in EXTENSIONS:
74 - sig_path = filepath + ext
75 - if isurl:
76 - signature = url + ext
77 - self.logger.debug(
78 - _unicode("ACTIONS: verify; fetching %s signature ")
79 - % signature)
80 - success_fetch, sig, timestamp = fetcher.fetch_file(signature, sig_path)
81 - if success_fetch:
82 - break
83 - else:
84 - signature = None
85 + fetcher = Fetch(self.logger)
86 + success, msgs = fetcher.fetch_url(url, filepath, signature, timestamp_path=timestamp_path,
87 + climit=climit)
88 + messages.extend(msgs)
89 elif signature is not None and os.path.exists(signature):
90 sig_path = signature
91 else:
92 @@ -864,9 +828,8 @@ class Actions(ActionBase):
93 self.logger.debug(
94 _unicode("ACTIONS: verify; local file %s") % filepath)
95 success = os.path.isfile(filepath)
96 - if (not signature
97 + if (success and not signature
98 and '.' + filepath.rsplit('.', 1)[-1] not in EXTENSIONS):
99 - success_fetch = False
100 for ext in EXTENSIONS:
101 sig_path = filepath + ext
102 sig_path = os.path.abspath(sig_path)
103
104 diff --git a/gkeys/gkeys/base.py b/gkeys/gkeys/base.py
105 index a67b330..2de1493 100644
106 --- a/gkeys/gkeys/base.py
107 +++ b/gkeys/gkeys/base.py
108 @@ -130,6 +130,12 @@ class CliBase(object):
109 action='store_true', default=False,
110 help='Use CASE matching in searches')
111
112 + @staticmethod
113 + def _option_fetcthonly(parser=None):
114 + parser.add_argument('--fetchonly',
115 + dest='fetchonly', default=False,
116 + help="Only fetch the seed file if there is an update or doesn't exist locally")
117 +
118 @staticmethod
119 def _option_file(parser=None):
120 parser.add_argument('-F', '--file', dest='filename', default=None,
121
122 diff --git a/gkeys/gkeys/seedhandler.py b/gkeys/gkeys/seedhandler.py
123 index afe5971..cb082ef 100644
124 --- a/gkeys/gkeys/seedhandler.py
125 +++ b/gkeys/gkeys/seedhandler.py
126 @@ -17,12 +17,15 @@ import re
127 from snakeoil.demandload import demandload
128
129 from gkeys.gkey import GKEY
130 +from gkeys.lock import LockDir
131 from gkeys.seed import Seeds, decoder
132
133 demandload(
134 "json:load",
135 "gkeys.exception:UpdateDbError",
136 "gkeys.fileops:ensure_dirs",
137 + "gkeys.fetch:Fetch",
138 + "sslfetch.connections:get_timestamp",
139 )
140
141
142 @@ -34,6 +37,8 @@ class SeedHandler(object):
143 self.fingerprint_re = re.compile('[0-9A-Fa-f]{40}')
144 self.finerprint_re2 = re.compile('[0-9A-Fa-f]{4}( [0-9A-Fa-f]{4}){9}')
145 self.seeds = None
146 + self.seedsdir_lock = None
147 + self.update_lock = None
148
149
150 def new(self, args, checkgkey=False):
151 @@ -70,10 +75,10 @@ class SeedHandler(object):
152 if attr in GKEY._fields:
153 keyinfo[attr] = None
154 return keyinfo
155 -
156 +
157 def compare_seeds(self, seeds1, seeds2) :
158 '''Compares two seed lists and returns the differences
159 -
160 +
161 @param seeds1: set of seeds to be compared
162 @param seeds2: set of seeds to be compared
163 @return added_gkeys: list of keys that are included in seed2 but not seed1
164 @@ -98,7 +103,7 @@ class SeedHandler(object):
165 if old_gkey not in new_gkeys and old_gkey not in old_changed_gkeys:
166 removed_gkeys.append(old_gkey)
167 else:
168 - added_gkeys = new_gkeys
169 + added_gkeys = new_gkeys
170 return(added_gkeys, changed_gkeys, removed_gkeys)
171
172 def compare_seeds(self, seeds1, seeds2) :
173 @@ -181,7 +186,8 @@ class SeedHandler(object):
174 with open(gkey_path, 'r') as fileseed:
175 seed = load(fileseed)
176 except IOError as error:
177 - self.logger.debug("SeedHandler: load_category; IOError loading seed file %s." % gkey_path)
178 + self.logger.debug("SeedHandler: load_category; IOError loading seed file %s."
179 + % gkey_path)
180 self.logger.debug("Error was: %s" % str(error))
181 if seed:
182 for nick in sorted(seed):
183 @@ -206,6 +212,7 @@ class SeedHandler(object):
184 '''Fetch new seed files
185
186 @param seeds: list of seed nicks to download
187 + @param args: argparse namespace instance
188 @param verified_dl: Function pointer to the Actions.verify()
189 instance needed to do the download and verification
190 '''
191 @@ -224,20 +231,45 @@ class SeedHandler(object):
192 except KeyError:
193 pass
194 succeeded = []
195 - seedsdir = self.config.get_key('seedsdir')
196 + seedsdir = os.path.join(self.config.get_key('seedsdir'))
197 + updatedir = os.path.join(seedsdir, "__updates__")
198 mode = int(self.config.get_key('permissions', 'directories'),0)
199 - ensure_dirs(seedsdir, mode=mode)
200 + ensure_dirs(updatedir, mode=mode)
201 + self.update_lock = LockDir(updatedir)
202 + self.update_lock.write_lock()
203 + fetcher = Fetch(self.logger)
204 for (seed, url, filepath) in urls:
205 - verify_info = self.config.get_key('verify-seeds', seed).split()
206 - args.category = verify_info[0]
207 - args.nick = verify_info[1]
208 - args.filename = url
209 - args.signature = None
210 - args.timestamp = True
211 - args.destination = filepath
212 - verified, messages_ = verified_dl(args)
213 - succeeded.append(verified)
214 - messages.append(messages_)
215 + tmppath = os.path.join(updatedir, os.path.split(filepath)[-1])
216 + # use the real timestamp file for the dl timestamp
217 + tpath = filepath + ".timestamp"
218 + # verify the re-fetch cycle timer
219 + if fetcher.verify_cycle(tpath, climit=60):
220 + timestamp = get_timestamp(filepath + ".timestamp")
221 + success, msgs = fetcher.fetch_url(url, tmppath, timestamp=timestamp)
222 + messages.extend(msgs)
223 + if success:
224 + verify_info = self.config.get_key('verify-seeds', seed).split()
225 + args.category = verify_info[0]
226 + args.nick = verify_info[1]
227 + args.filename = url
228 + args.signature = tmppath + ".timestamp"
229 + #args.timestamp = True
230 + args.destination = tmppath
231 + verified, messages_ = verified_dl(args)
232 + messages.append(messages_)
233 + if verified and not args.fetchonly:
234 + self.seedsdir_lock = LockDir(seedsdir)
235 + if updateseeds(tmppath, filepath) and updateseeds(args.signature, tpath):
236 + self.logger.info("Updated seed file...: %s ... OK" % (filepath))
237 + succeeded.append(verified)
238 + else:
239 + self.logger.info("Updating seed file...: %s ... Failed" % (filepath))
240 + succeeded.append(False)
241 + self.seedsdir_lock.unlock()
242 + else:
243 + # sha512sum the 2 files
244 + pass
245 + self.update_lock.unlock()
246 return (succeeded, messages)
247
248 def check_gkey(self, args):