[gentoo-commits] proj/gentoo-keys:master commit in: gkeys/gkeys/

[Brian Dolbec]

commit:     eed71e755fca18ebc271b3a3b72e70c0822f55d2
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Mon Jul  2 21:57:23 2018 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Sat Jul  7 05:22:12 2018 +0000
URL:        https://gitweb.gentoo.org/proj/gentoo-keys.git/commit/?id=eed71e75

gkeys: Implement use of the new Fetch class

Signed-off-by: Brian Dolbec <dolsen <AT> gentoo.org>

Signed-off-by: Brian Dolbec <dolsen <AT> gentoo.org>

 gkeys/gkeys/actions.py     | 51 +++++-------------------------------
 gkeys/gkeys/base.py        |  6 +++++
 gkeys/gkeys/seedhandler.py | 64 ++++++++++++++++++++++++++++++++++------------
 3 files changed, 61 insertions(+), 60 deletions(-)

diff --git a/gkeys/gkeys/actions.py b/gkeys/gkeys/actions.py
index 64d6123..0f03880 100644
--- a/gkeys/gkeys/actions.py
+++ b/gkeys/gkeys/actions.py
@@ -29,6 +29,7 @@ from snakeoil.demandload import demandload
 
 demandload(
     "gkeys.base:Args",
+    "gkeys.fetch:Fetch",
     "json:load",
 )
 
@@ -718,9 +719,7 @@ class Actions(ActionBase):
 
     def installed(self, args):
         '''Lists the installed key directories'''
-        if args.category:
-            keyring = self.config.get_key('keyring')
-        else:
+        if not args.category:
             return (False, ["Please specify a category."])
         catdir = self._set_category(args.category)
         self.logger.debug("ACTIONS: installed; catdir = %s" % catdir)
@@ -818,45 +817,10 @@ class Actions(ActionBase):
             climit = 0
         sig_path = None
         if isurl:
-            from sslfetch.connections import Connector
-            connector_output = {
-                 'info': self.logger.info,
-                 'debug': self.logger.debug,
-                 'error': self.logger.error,
-                 'exception': self.logger.exception,
-                 # we want any warnings to be printed to the terminal
-                 # so assign it to logging.error
-                 'warning': self.logger.error,
-                 'kwargs-info': {},
-                 'kwargs-debug': {},
-                 'kwargs-error': {},
-                 'kwargs-exception': {},
-                 'kwargs-warning': {},
-            }
-            fetcher = Connector(connector_output, None, "Gentoo Keys")
-            self.logger.debug(
-                _unicode("ACTIONS: verify; fetching %s signed file ") % filepath)
-            self.logger.debug(
-                _unicode("ACTIONS: verify; timestamp path: %s") % timestamp_path)
-            success, signedfile, timestamp = fetcher.fetch_file(
-                url, filepath, timestamp_path, climit=climit)
-            if not success:
-                messages.append(_unicode("File %s cannot be retrieved.") % filepath)
-            elif '.' + url.rsplit('.', 1)[1] not in EXTENSIONS:
-                if not signature:
-                    success_fetch = False
-                    for ext in EXTENSIONS:
-                        sig_path = filepath + ext
-                        if isurl:
-                            signature = url + ext
-                            self.logger.debug(
-                                _unicode("ACTIONS: verify; fetching %s signature ")
-                                % signature)
-                            success_fetch, sig, timestamp = fetcher.fetch_file(signature, sig_path)
-                        if success_fetch:
-                            break
-                        else:
-                            signature = None
+            fetcher = Fetch(self.logger)
+            success, msgs = fetcher.fetch_url(url, filepath, signature, timestamp_path=timestamp_path,
+                                                  climit=climit)
+            messages.extend(msgs)
         elif signature is not None and os.path.exists(signature):
             sig_path = signature
         else:
@@ -864,9 +828,8 @@ class Actions(ActionBase):
             self.logger.debug(
                 _unicode("ACTIONS: verify; local file %s") % filepath)
             success = os.path.isfile(filepath)
-            if (not signature
+            if (success and not signature
                 and '.' + filepath.rsplit('.', 1)[-1] not in EXTENSIONS):
-                success_fetch = False
                 for ext in EXTENSIONS:
                     sig_path = filepath + ext
                     sig_path = os.path.abspath(sig_path)

diff --git a/gkeys/gkeys/base.py b/gkeys/gkeys/base.py
index a67b330..2de1493 100644
--- a/gkeys/gkeys/base.py
+++ b/gkeys/gkeys/base.py
@@ -130,6 +130,12 @@ class CliBase(object):
             action='store_true', default=False,
             help='Use CASE matching in searches')
 
+    @staticmethod
+    def _option_fetcthonly(parser=None):
+        parser.add_argument('--fetchonly',
+            dest='fetchonly', default=False,
+            help="Only fetch the seed file if there is an update or doesn't exist locally")
+
     @staticmethod
     def _option_file(parser=None):
         parser.add_argument('-F', '--file', dest='filename', default=None,

diff --git a/gkeys/gkeys/seedhandler.py b/gkeys/gkeys/seedhandler.py
index afe5971..cb082ef 100644
--- a/gkeys/gkeys/seedhandler.py
+++ b/gkeys/gkeys/seedhandler.py
@@ -17,12 +17,15 @@ import re
 from snakeoil.demandload import demandload
 
 from gkeys.gkey import GKEY
+from gkeys.lock import LockDir
 from gkeys.seed import Seeds, decoder
 
 demandload(
     "json:load",
     "gkeys.exception:UpdateDbError",
     "gkeys.fileops:ensure_dirs",
+    "gkeys.fetch:Fetch",
+    "sslfetch.connections:get_timestamp",
 )
 
 
@@ -34,6 +37,8 @@ class SeedHandler(object):
         self.fingerprint_re = re.compile('[0-9A-Fa-f]{40}')
         self.finerprint_re2 = re.compile('[0-9A-Fa-f]{4}( [0-9A-Fa-f]{4}){9}')
         self.seeds = None
+        self.seedsdir_lock = None
+        self.update_lock = None
 
 
     def new(self, args, checkgkey=False):
@@ -70,10 +75,10 @@ class SeedHandler(object):
                 if attr in GKEY._fields:
                     keyinfo[attr] = None
         return keyinfo
-    
+
     def compare_seeds(self, seeds1, seeds2) :
         '''Compares two seed lists and returns the differences
-        
+
         @param seeds1: set of seeds to be compared
         @param seeds2: set of seeds to be compared
         @return added_gkeys: list of keys that are included in seed2 but not seed1
@@ -98,7 +103,7 @@ class SeedHandler(object):
                 if old_gkey not in new_gkeys and old_gkey not in old_changed_gkeys:
                     removed_gkeys.append(old_gkey)
         else:
-            added_gkeys = new_gkeys 
+            added_gkeys = new_gkeys
         return(added_gkeys, changed_gkeys, removed_gkeys)
 
     def compare_seeds(self, seeds1, seeds2) :
@@ -181,7 +186,8 @@ class SeedHandler(object):
                     with open(gkey_path, 'r') as fileseed:
                         seed = load(fileseed)
                 except IOError as error:
-                    self.logger.debug("SeedHandler: load_category; IOError loading seed file %s." % gkey_path)
+                    self.logger.debug("SeedHandler: load_category; IOError loading seed file %s."
+                                      % gkey_path)
                     self.logger.debug("Error was: %s" % str(error))
                 if seed:
                     for nick in sorted(seed):
@@ -206,6 +212,7 @@ class SeedHandler(object):
         '''Fetch new seed files
 
         @param seeds: list of seed nicks to download
+        @param args: argparse namespace instance
         @param verified_dl: Function pointer to the Actions.verify()
                 instance needed to do the download and verification
         '''
@@ -224,20 +231,45 @@ class SeedHandler(object):
         except KeyError:
             pass
         succeeded = []
-        seedsdir = self.config.get_key('seedsdir')
+        seedsdir = os.path.join(self.config.get_key('seedsdir'))
+        updatedir = os.path.join(seedsdir, "__updates__")
         mode = int(self.config.get_key('permissions', 'directories'),0)
-        ensure_dirs(seedsdir, mode=mode)
+        ensure_dirs(updatedir, mode=mode)
+        self.update_lock = LockDir(updatedir)
+        self.update_lock.write_lock()
+        fetcher = Fetch(self.logger)
         for (seed, url, filepath) in urls:
-            verify_info = self.config.get_key('verify-seeds', seed).split()
-            args.category = verify_info[0]
-            args.nick = verify_info[1]
-            args.filename = url
-            args.signature = None
-            args.timestamp = True
-            args.destination = filepath
-            verified, messages_ = verified_dl(args)
-            succeeded.append(verified)
-            messages.append(messages_)
+            tmppath = os.path.join(updatedir, os.path.split(filepath)[-1])
+            # use the real timestamp file for the dl timestamp
+            tpath = filepath + ".timestamp"
+            # verify the re-fetch cycle timer
+            if fetcher.verify_cycle(tpath, climit=60):
+                timestamp = get_timestamp(filepath + ".timestamp")
+                success, msgs = fetcher.fetch_url(url, tmppath, timestamp=timestamp)
+                messages.extend(msgs)
+                if success:
+                    verify_info = self.config.get_key('verify-seeds', seed).split()
+                    args.category = verify_info[0]
+                    args.nick = verify_info[1]
+                    args.filename = url
+                    args.signature = tmppath + ".timestamp"
+                    #args.timestamp = True
+                    args.destination = tmppath
+                    verified, messages_ = verified_dl(args)
+                    messages.append(messages_)
+                if verified and not args.fetchonly:
+                    self.seedsdir_lock = LockDir(seedsdir)
+                    if updateseeds(tmppath, filepath) and updateseeds(args.signature, tpath):
+                        self.logger.info("Updated seed file...: %s ... OK" % (filepath))
+                        succeeded.append(verified)
+                    else:
+                        self.logger.info("Updating seed file...: %s ... Failed" % (filepath))
+                        succeeded.append(False)
+                    self.seedsdir_lock.unlock()
+                else:
+                    # sha512sum the 2 files
+                    pass
+        self.update_lock.unlock()
         return (succeeded, messages)
 
     def check_gkey(self, args):