[gentoo-commits] proj/R_overlay:master commit in: roverlay/, roverlay/util/

[André Erdmann]

commit:     aa77ac2daaac2b4666fdecf574db00f2d5e1bba2
Author:     André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Tue Jul 30 16:03:37 2013 +0000
Commit:     André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Tue Jul 30 16:03:37 2013 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=aa77ac2d

roverlay/util/fileio: detect filetype when reading

---
 roverlay/strutil.py     |  2 +-
 roverlay/util/common.py |  2 +-
 roverlay/util/fileio.py | 75 ++++++++++++++++++++++++++++++++++++++++---------
 3 files changed, 63 insertions(+), 16 deletions(-)

diff --git a/roverlay/strutil.py b/roverlay/strutil.py
index 755a191..a8e4361 100644
--- a/roverlay/strutil.py
+++ b/roverlay/strutil.py
@@ -145,7 +145,7 @@ def bytes_try_decode (
    * charwise_only -- do charwise conversion only
    * force_decode  -- decode byte_str even if it's already a str
    """
-   if not isinstance ( byte_str, str ):
+   if not isinstance ( byte_str, str ) or force_decode:
       if not charwise_only and encodings:
          ret = None
          if not isinstance ( encodings, str ):

diff --git a/roverlay/util/common.py b/roverlay/util/common.py
index 6053015..8449b2c 100644
--- a/roverlay/util/common.py
+++ b/roverlay/util/common.py
@@ -219,7 +219,7 @@ def dodir ( directory, mkdir_p=False, **makedirs_kw ):
 # --- end of dodir (...) ---
 
 def dodir_for_file ( filepath, mkdir_p=True, **kw ):
-   return dodir ( os.path.basename ( filepath ), mkdir_p=mkdir_p, **kw )
+   return dodir ( os.path.dirname ( filepath ), mkdir_p=mkdir_p, **kw )
 # --- end of dodir_for_file (...) ---
 
 def getsize ( filepath ):

diff --git a/roverlay/util/fileio.py b/roverlay/util/fileio.py
index 5d996f8..c6e33e5 100644
--- a/roverlay/util/fileio.py
+++ b/roverlay/util/fileio.py
@@ -10,14 +10,8 @@ import mimetypes
 import sys
 
 import roverlay.util.common
-
-if sys.hexversion >= 0x3000000:
-   def iter_decode ( lv ):
-      for l in lv:
-         yield l.decode()
-else:
-   def iter_decode ( lv ):
-      return lv
+import roverlay.strutil
+from roverlay.strutil import bytes_try_decode
 
 
 _MIME = mimetypes.MimeTypes()
@@ -37,19 +31,72 @@ SUPPORTED_COMPRESSION = {
 }
 
 
-def read_text_file ( filepath, preparse=None ):
+def read_compressed_file_handle ( CH, preparse=None ):
+   if preparse is None:
+      for line in CH.readlines():
+         yield bytes_try_decode ( line )
+   else:
+      yield preparse ( bytes_try_decode ( line ) )
+# --- end of read_compressed_file_handle (...) ---
+
+def read_text_file ( filepath, preparse=None, try_harder=True ):
+   """Generator that reads a compressed/uncompressed file and yields text
+   lines. Optionally preparses the rext lines.
+
+   arguments:
+   * filepath   -- file to read
+   * preparse   -- function for (pre-)parsing lines
+   * try_harder -- try known compression formats if file extension cannot
+                   be detected (defaults to True)
+   """
+
    ftype         = guess_filetype ( filepath )
    compress_open = SUPPORTED_COMPRESSION.get ( ftype[1], None )
 
    if compress_open is not None:
       with compress_open ( filepath, mode='r' ) as CH:
-         for line in iter_decode ( CH.readlines() ):
-            yield line if preparse is None else preparse ( line )
+         for line in read_compressed_file_handle ( CH, preparse ):
+            yield line
+
+   elif try_harder:
+      # guess_filetype detects file extensions only
+      #
+      #  try known compression formats
+      #
+      for comp in ( COMP_BZIP2, COMP_GZIP ):
+         CH = None
+         try:
+            CH = SUPPORTED_COMPRESSION [comp] ( filepath, mode='r' )
+            for line in read_compressed_file_handle ( CH, preparse ):
+               yield line
+            CH.close()
+         except IOError as ioerr:
+            if CH:
+               CH.close()
+            if ioerr.errno is not None:
+               raise
+         else:
+            break
+      else:
+         with open ( filepath, 'rt' ) as FH:
+            if preparse is None:
+               for line in FH.readlines():
+                  yield line
+            else:
+               for line in FH.readlines():
+                  yield preparse ( line )
+      # -- end for <comp>
    else:
       with open ( filepath, 'rt' ) as FH:
-         for line in FH.readlines():
-            yield line if preparse is None else preparse ( line )
+         if preparse is None:
+            for line in FH.readlines():
+               yield line
+         else:
+            for line in FH.readlines():
+               yield preparse ( line )
+   # -- end if <compress_open?, try_harder?>
 
+# --- end of read_text_file (...) ---
 
 def write_text_file (
    filepath, lines, compression=None, mode='wt',
@@ -64,7 +111,7 @@ def write_text_file (
       roverlay.util.common.dodir_for_file ( filepath )
 
    if compress_open:
-      NL = '\n'.encode()
+      NL = newline.encode()
       with compress_open ( filepath, mode.rstrip ( 'tu' ) ) as CH:
          for line in lines:
             CH.write ( str ( line ).encode() )