[gentoo-commits] proj/R_overlay:master commit in: roverlay/util/, roverlay/remote/

[André Erdmann]

commit:     ca1020e87ff4421499df80c1d1af9e6c33f9afe2
Author:     André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Sun Feb 16 16:44:15 2014 +0000
Commit:     André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Sun Feb 16 16:44:15 2014 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=ca1020e8

roverlay/remote/websync: show download status

---
 roverlay/remote/websync.py   |  39 ++++++--
 roverlay/util/progressbar.py | 210 +++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 242 insertions(+), 7 deletions(-)

diff --git a/roverlay/remote/websync.py b/roverlay/remote/websync.py
index 24f5cd5..06689ad 100644
--- a/roverlay/remote/websync.py
+++ b/roverlay/remote/websync.py
@@ -4,6 +4,7 @@
 # Distributed under the terms of the GNU General Public License;
 # either version 2 of the License, or (at your option) any later version.
 
+from __future__ import division
 from __future__ import print_function
 
 """websync, sync packages via http"""
@@ -31,6 +32,7 @@ HTTPError = _urllib_error.HTTPError
 
 from roverlay                  import config, digest, util
 from roverlay.remote.basicrepo import BasicRepo
+from roverlay.util.progressbar import DownloadProgressBar, NullProgressBar
 
 # number of sync retries
 #  changed 2014-02-15: does no longer include the first run
@@ -50,6 +52,15 @@ class WebsyncBase ( BasicRepo ):
    HTTP_ERROR_RETRY_CODES = frozenset ({ 404, 410, 500, 503 })
    URL_ERROR_RETRY_CODES  = frozenset ({ errno.ETIMEDOUT, })
    RETRY_ON_TIMEOUT       = True
+   PROGRESS_BAR_CLS       = None
+
+   def __new__ ( cls, *args, **kwargs ):
+      if cls.PROGRESS_BAR_CLS is None:
+         cls.PROGRESS_BAR_CLS = (
+            DownloadProgressBar if VERBOSE else NullProgressBar
+         )
+      return super ( WebsyncBase, cls ).__new__ ( cls )
+   # --- end of __new__ (...) ---
 
    def __init__ ( self,
       name,
@@ -170,27 +181,41 @@ class WebsyncBase ( BasicRepo ):
             bytes_fetched = 0
             assert blocksize
 
-            # FIXME: debug print (?)
-            if VERBOSE:
-               print (
-                  "Fetching {f} from {u} ...".format ( f=package_file, u=src_uri )
-               )
-
             # unlink the existing file first (if it exists)
             #  this is necessary for keeping hardlinks intact (-> package mirror)
             util.try_unlink ( distfile )
 
-            with open ( distfile, mode='wb' ) as fh:
+            with \
+               open ( distfile, mode='wb' ) as fh, \
+               self.PROGRESS_BAR_CLS (
+                  package_file.ljust(50), expected_filesize
+            ) as progress_bar:
+
+               progress_bar.update ( 0 )
                block = webh.read ( blocksize )
+
                while block:
                   # write block to file
                   fh.write ( block )
                   # ? bytelen
                   bytes_fetched += len ( block )
 
+                  # update progress bar on every 4th block
+                  #  blocks_fetched := math.ceil ( bytes_fetched / blocksize )
+                  #
+                  #  Usually, only the last block's size is <= blocksize,
+                  #  so floordiv is sufficient here
+                  #  (the progress bar gets updated for the last block anyway)
+                  #
+                  if 0 == ( bytes_fetched // blocksize ) % 4:
+                     progress_bar.update ( bytes_fetched )
+
                   # get the next block
                   block = webh.read ( blocksize )
                # -- end while
+
+               # final progress bar update (before closing the file)
+               progress_bar.update ( bytes_fetched )
             # -- with
 
             if bytes_fetched == expected_filesize:

diff --git a/roverlay/util/progressbar.py b/roverlay/util/progressbar.py
new file mode 100644
index 0000000..24c934e
--- /dev/null
+++ b/roverlay/util/progressbar.py
@@ -0,0 +1,210 @@
+# R overlay -- util, progressbar
+# -*- coding: utf-8 -*-
+# Copyright (C) 2014 André Erdmann <dywi@×××××××.de>
+# Distributed under the terms of the GNU General Public License;
+# either version 2 of the License, or (at your option) any later version.
+from __future__ import division
+
+import abc
+import sys
+
+import roverlay.util.objects
+
+
+class AbstractProgressBarBase ( roverlay.util.objects.AbstractObject ):
+   """Abstract base class for progress bars."""
+
+   @abc.abstractmethod
+   def setup ( self, *args, **kwargs ):
+      """Initialization code for __init__() and reset().
+
+      Returns: None
+
+      arguments:
+      * *args, **kwargs -- progress bar data
+      """
+      pass
+   # --- end of setup (...) ---
+
+   def reset ( self, *args, **kwargs ):
+      """Finalizes the current progress bar and resets it afterwards.
+
+      Returns: None
+
+      arguments:
+      * *args, **kwargs -- passed to setup()
+      """
+      self.print_newline()
+      self.setup ( *args, **kwargs )
+   # --- end of reset (...) ---
+
+   def __init__ ( self, *args, **kwargs ):
+      """Initializes a progress bar instance by calling its setup() method.
+
+      arguments:
+      * *args, **kwargs -- passed to setup()
+      """
+      super ( AbstractProgressBarBase, self ).__init__()
+      self.setup ( *args, **kwargs )
+   # --- end of __init__ (...) ---
+
+   @abc.abstractmethod
+   def write ( self, message ):
+      """(Over-)writes the progress bar, using the given message.
+
+      Note: message should not contain newline chars.
+
+      Returns: None
+
+      arguments:
+      * message --
+      """
+      raise NotImplementedError()
+   # --- end of write (...) ---
+
+   @abc.abstractmethod
+   def print_newline ( self ):
+      """
+      Finalizes the current progress bar, usually by printing a newline char.
+
+      Returns: None
+      """
+      raise NotImplementedError()
+   # --- end of print_newline (...) ---
+
+   @abc.abstractmethod
+   def update ( self, *args, **kwargs ):
+      """Updates the progress bar using the given data.
+
+      Returns: None
+
+      arguments:
+      * *args, **kwargs -- not specified by this class
+      """
+      raise NotImplementedError()
+   # --- end of update (...) ---
+
+   def __enter__ ( self ):
+      # "with"-statement, setup code
+      return self
+
+   def __exit__ ( self, _type, value, traceback ):
+      # "with"-statement, teardown code
+      self.print_newline()
+
+# --- end of AbstractProgressBarBase ---
+
+
+class AbstractProgressBar ( AbstractProgressBarBase ):
+   """
+   Abstract base class for progress bars that write to a stream, e.g. stdout.
+   """
+
+   CARRIAGE_RET_CHR = chr(13)
+   #BACKSPACE_CHR    = chr(8)
+
+   def setup ( self, stream=None ):
+      self.stream = ( sys.stdout if stream is None else stream )
+   # --- end of __init__ (...) ---
+
+   def write ( self, message ):
+      self.stream.write ( self.CARRIAGE_RET_CHR + message )
+      self.stream.flush()
+   # --- end of write (...) ---
+
+   def print_newline ( self ):
+      self.stream.write ( "\n" )
+      self.stream.flush()
+   # --- end of print_newline (...) ---
+
+# --- end of AbstractProgressBar ---
+
+
+class AbstractPercentageProgressBar ( AbstractProgressBar ):
+   """Base class for displaying progress as percentage 0.00%..100.00%."""
+   # not a real progress bar, just a progress indicator
+
+   # str for formatting the percentage
+   #  by default, reserve space for 7 chars ("ddd.dd%")
+   #  might be set by derived classes and/or instances
+   PERCENTAGE_FMT = "{:>7.2%}"
+
+   def setup ( self, message_header=None, stream=None ):
+      super ( AbstractPercentageProgressBar, self ).setup ( stream=stream )
+      self.message_header = message_header
+   # --- end of setup (...) ---
+
+   @abc.abstractmethod
+   def get_percentage ( self, *args, **kwargs ):
+      """Returns a float or int expressing a percentage.
+
+      Any value < 0 is interpreted as "UNKNOWN".
+
+      arguments:
+      * *args, **kwargs -- progress information (from update())
+      """
+      raise NotImplementedError()
+   # --- end of get_percentage (...) ---
+
+   def _update ( self, percentage ):
+      if self.message_header:
+         message = str(self.message_header) + " "
+      else:
+         message = ""
+
+      if percentage < 0:
+         message += "UNKNOWN"
+      else:
+         message += self.PERCENTAGE_FMT.format ( percentage )
+
+      self.write ( message )
+   # --- end of _update (...) ---
+
+   def update ( self, *args, **kwargs ):
+      self._update ( self.get_percentage ( *args, **kwargs ) )
+   # --- end of update (...) ---
+
+# --- end of AbstractPercentageProgressBar ---
+
+
+class NullProgressBar ( AbstractProgressBarBase ):
+   """A progress bar that discards any information."""
+
+   def setup ( self, *args, **kwargs ):
+      pass
+
+   def write ( self, *args, **kwargs ):
+      pass
+
+   def print_newline ( self, *args, **kwargs ):
+      pass
+
+   def update ( self, *args, **kwargs ):
+      pass
+
+# --- end of NullProgressBar ---
+
+
+class DownloadProgressBar ( AbstractPercentageProgressBar ):
+   """A progress bar for file transfers,
+   expressing a percentage "bytes transferred / total size".
+
+   Note:
+    update() shouldn't be called too often as writing to console is rather slow
+   """
+
+   def setup ( self, filename=None, filesize=None, stream=None ):
+      super ( DownloadProgressBar, self ).setup (
+         message_header = (
+            ( "Fetching " + str(filename) ) if filename else None
+         ),
+         stream = stream
+      )
+      self.filesize = filesize
+   # --- end of setup (...) ---
+
+   def get_percentage ( self, current_filesize ):
+      return ( current_filesize / self.filesize ) if self.filesize else -1.0
+   # --- end of get_percentage (...) ---
+
+# --- end of DownloadProgressBar ---