| 1 |
commit: cbebf76d8e5666aad4984f87c2be83d474fe5a7e |
| 2 |
Author: W-Mark Kubacki <wmark <AT> hurrikane <DOT> de> |
| 3 |
AuthorDate: Wed Aug 1 18:36:31 2012 +0000 |
| 4 |
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org> |
| 5 |
CommitDate: Thu Aug 2 00:54:15 2012 +0000 |
| 6 |
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=cbebf76d |
| 7 |
|
| 8 |
Add support for HTTP compression (bzip2, gzip and deflate). |
| 9 |
|
| 10 |
--- |
| 11 |
pym/portage/util/_urlopen.py | 32 +++++++++++++++++++++++++++++++- |
| 12 |
1 files changed, 31 insertions(+), 1 deletions(-) |
| 13 |
|
| 14 |
diff --git a/pym/portage/util/_urlopen.py b/pym/portage/util/_urlopen.py |
| 15 |
index 4296188..bcd8f7c 100644 |
| 16 |
--- a/pym/portage/util/_urlopen.py |
| 17 |
+++ b/pym/portage/util/_urlopen.py |
| 18 |
@@ -1,6 +1,7 @@ |
| 19 |
# Copyright 2012 Gentoo Foundation |
| 20 |
# Distributed under the terms of the GNU General Public License v2 |
| 21 |
|
| 22 |
+import io |
| 23 |
import sys |
| 24 |
from datetime import datetime |
| 25 |
from time import mktime |
| 26 |
@@ -33,7 +34,7 @@ def urlopen(url, if_modified_since=None): |
| 27 |
request.add_header('User-Agent', 'Gentoo Portage') |
| 28 |
if if_modified_since: |
| 29 |
request.add_header('If-Modified-Since', _timestamp_to_http(if_modified_since)) |
| 30 |
- opener = urllib_request.build_opener() |
| 31 |
+ opener = urllib_request.build_opener(CompressedResponseProcessor) |
| 32 |
hdl = opener.open(request) |
| 33 |
if hdl.headers.get('last-modified', ''): |
| 34 |
try: |
| 35 |
@@ -77,3 +78,32 @@ def _http_to_timestamp(http_datetime_string): |
| 36 |
tuple = parsedate(http_datetime_string) |
| 37 |
timestamp = mktime(tuple) |
| 38 |
return str(long(timestamp)) |
| 39 |
+ |
| 40 |
+class CompressedResponseProcessor(urllib_request.BaseHandler): |
| 41 |
+ # Handler for compressed responses. |
| 42 |
+ |
| 43 |
+ def http_request(self, req): |
| 44 |
+ req.add_header('Accept-Encoding', 'bzip2,gzip,deflate') |
| 45 |
+ return req |
| 46 |
+ https_request = http_request |
| 47 |
+ |
| 48 |
+ def http_response(self, req, response): |
| 49 |
+ decompressed = None |
| 50 |
+ if response.headers.get('content-encoding') == 'bzip2': |
| 51 |
+ import bz2 |
| 52 |
+ decompressed = io.BytesIO(bz2.decompress(response.read())) |
| 53 |
+ elif response.headers.get('content-encoding') == 'gzip': |
| 54 |
+ from gzip import GzipFile |
| 55 |
+ decompressed = GzipFile(fileobj=io.BytesIO(response.read()), mode='r') |
| 56 |
+ elif response.headers.get('content-encoding') == 'deflate': |
| 57 |
+ import zlib |
| 58 |
+ try: |
| 59 |
+ decompressed = io.BytesIO(zlib.decompress(response.read())) |
| 60 |
+ except zlib.error: # they ignored RFC1950 |
| 61 |
+ decompressed = io.BytesIO(zlib.decompress(response.read(), -zlib.MAX_WBITS)) |
| 62 |
+ if decompressed: |
| 63 |
+ old_response = response |
| 64 |
+ response = urllib_request.addinfourl(decompressed, old_response.headers, old_response.url, old_response.code) |
| 65 |
+ response.msg = old_response.msg |
| 66 |
+ return response |
| 67 |
+ https_response = http_response |
Archives