1 |
commit: 51f3cf18de49ca9dffc34053da53e62930be9fab |
2 |
Author: Paul Varner <fuzzyray <AT> gentoo <DOT> org> |
3 |
AuthorDate: Thu Jun 30 23:09:52 2016 +0000 |
4 |
Commit: Paul Varner <fuzzyray <AT> gentoo <DOT> org> |
5 |
CommitDate: Thu Jun 30 23:30:54 2016 +0000 |
6 |
URL: https://gitweb.gentoo.org/proj/gentoolkit.git/commit/?id=51f3cf18 |
7 |
|
8 |
Change open function to use UTF-8 encoding for content. |
9 |
|
10 |
Uses io.open for Python 2 and built-in open for Python 3 |
11 |
All data from files is treated as Unicode and this should fix |
12 |
most UnicodeDecodeErrors. |
13 |
|
14 |
pym/gentoolkit/deprecated/helpers.py | 6 +++++- |
15 |
pym/gentoolkit/eclean/exclude.py | 4 +++- |
16 |
pym/gentoolkit/enalyze/rebuild.py | 5 ++++- |
17 |
pym/gentoolkit/equery/uses.py | 11 +++++++---- |
18 |
pym/gentoolkit/equery/which.py | 5 ++++- |
19 |
pym/gentoolkit/eshowkw/keywords_header.py | 9 +++++++-- |
20 |
pym/gentoolkit/glsa/__init__.py | 5 ++++- |
21 |
pym/gentoolkit/helpers.py | 3 ++- |
22 |
pym/gentoolkit/revdep_rebuild/analyse.py | 6 +++++- |
23 |
pym/gentoolkit/revdep_rebuild/cache.py | 13 +++++++++---- |
24 |
pym/gentoolkit/revdep_rebuild/collect.py | 8 ++++++-- |
25 |
pym/gentoolkit/revdep_rebuild/settings.py | 4 +++- |
26 |
pym/gentoolkit/test/eclean/creator.py | 6 +++++- |
27 |
13 files changed, 64 insertions(+), 21 deletions(-) |
28 |
|
29 |
diff --git a/pym/gentoolkit/deprecated/helpers.py b/pym/gentoolkit/deprecated/helpers.py |
30 |
index 81fa45c..bb0fb7e 100644 |
31 |
--- a/pym/gentoolkit/deprecated/helpers.py |
32 |
+++ b/pym/gentoolkit/deprecated/helpers.py |
33 |
@@ -11,6 +11,10 @@ from __future__ import print_function |
34 |
|
35 |
import warnings |
36 |
|
37 |
+import sys |
38 |
+if sys.hexversion < 0x3000000: |
39 |
+ from io import open |
40 |
+ |
41 |
import portage |
42 |
from portage import _encodings, _unicode_decode, _unicode_encode |
43 |
from gentoolkit import * |
44 |
@@ -101,7 +105,7 @@ def find_world_packages(prefilter=None): |
45 |
"""Returns a tuple of lists, first list is resolved world packages, |
46 |
seond is unresolved package names.""" |
47 |
f = open(_unicode_encode(portage.root+portage.WORLD_FILE, |
48 |
- encoding=_encodings['fs'])) |
49 |
+ encoding=_encodings['fs']), encoding=_encodings['content']) |
50 |
pkglist = f.readlines() |
51 |
resolved = [] |
52 |
unresolved = [] |
53 |
|
54 |
diff --git a/pym/gentoolkit/eclean/exclude.py b/pym/gentoolkit/eclean/exclude.py |
55 |
index d19c1d1..513346d 100644 |
56 |
--- a/pym/gentoolkit/eclean/exclude.py |
57 |
+++ b/pym/gentoolkit/eclean/exclude.py |
58 |
@@ -9,6 +9,8 @@ from __future__ import print_function |
59 |
|
60 |
import os |
61 |
import sys |
62 |
+if sys.hexversion < 0x3000000: |
63 |
+ from io import open |
64 |
import re |
65 |
import portage |
66 |
from portage import _encodings, _unicode_decode, _unicode_encode |
67 |
@@ -83,7 +85,7 @@ def parseExcludeFile(filepath, output): |
68 |
output("Parsing Exclude file: " + filepath) |
69 |
try: |
70 |
file_ = open(_unicode_encode(filepath, |
71 |
- encoding=_encodings['fs']), mode="r") |
72 |
+ encoding=_encodings['fs']), mode="r", encoding=_encodings['content']) |
73 |
except IOError: |
74 |
raise ParseExcludeFileException("Could not open exclusion file: " + |
75 |
filepath) |
76 |
|
77 |
diff --git a/pym/gentoolkit/enalyze/rebuild.py b/pym/gentoolkit/enalyze/rebuild.py |
78 |
index 11feb31..53fded4 100644 |
79 |
--- a/pym/gentoolkit/enalyze/rebuild.py |
80 |
+++ b/pym/gentoolkit/enalyze/rebuild.py |
81 |
@@ -15,6 +15,8 @@ from __future__ import print_function |
82 |
|
83 |
import os |
84 |
import sys |
85 |
+if sys.hexversion < 0x3000000: |
86 |
+ from io import open |
87 |
|
88 |
import gentoolkit |
89 |
from gentoolkit.module_base import ModuleBase |
90 |
@@ -352,7 +354,8 @@ class Rebuild(ModuleBase): |
91 |
""" |
92 |
if not self.options["quiet"]: |
93 |
print(' - Saving file: %s' %filepath) |
94 |
- with open(_unicode_encode(filepath, encoding=_encodings['fs']), mode="w") as output: |
95 |
+ with open(_unicode_encode(filepath, encoding=_encodings['fs']), mode="w", |
96 |
+ encoding=_encodings['content']) as output: |
97 |
output.write('\n'.join(data)) |
98 |
print(" - Done") |
99 |
|
100 |
|
101 |
diff --git a/pym/gentoolkit/equery/uses.py b/pym/gentoolkit/equery/uses.py |
102 |
index 79f1118..a8f13dc 100644 |
103 |
--- a/pym/gentoolkit/equery/uses.py |
104 |
+++ b/pym/gentoolkit/equery/uses.py |
105 |
@@ -16,6 +16,9 @@ __docformat__ = 'epytext' |
106 |
|
107 |
import os |
108 |
import sys |
109 |
+if sys.hexversion < 0x3000000: |
110 |
+ from io import open |
111 |
+ |
112 |
from functools import partial |
113 |
from getopt import gnu_getopt, GetoptError |
114 |
from glob import glob |
115 |
@@ -136,9 +139,9 @@ def get_global_useflags(): |
116 |
# Get global USE flag descriptions |
117 |
try: |
118 |
path = os.path.join(settings["PORTDIR"], 'profiles', 'use.desc') |
119 |
- with open(_unicode_encode(path, encoding=_encodings['fs'])) as open_file: |
120 |
+ with open(_unicode_encode(path, encoding=_encodings['fs']), |
121 |
+ encoding=_encodings['content']) as open_file: |
122 |
for line in open_file: |
123 |
- line = _unicode_decode(line) |
124 |
if line.startswith('#'): |
125 |
continue |
126 |
# Ex. of fields: ['syslog', 'Enables support for syslog\n'] |
127 |
@@ -157,9 +160,9 @@ def get_global_useflags(): |
128 |
for path in glob(os.path.join(settings["PORTDIR"], |
129 |
'profiles', 'desc', '*.desc')): |
130 |
try: |
131 |
- with open(_unicode_encode(path, encoding=_encodings['fs'])) as open_file: |
132 |
+ with open(_unicode_encode(path, encoding=_encodings['fs']), |
133 |
+ encoding=_encodings['content']) as open_file: |
134 |
for line in open_file: |
135 |
- line = _unicode_decode(line) |
136 |
if line.startswith('#'): |
137 |
continue |
138 |
fields = [field.strip() for field in line.split(" - ", 1)] |
139 |
|
140 |
diff --git a/pym/gentoolkit/equery/which.py b/pym/gentoolkit/equery/which.py |
141 |
index 0d30a8d..137e52a 100644 |
142 |
--- a/pym/gentoolkit/equery/which.py |
143 |
+++ b/pym/gentoolkit/equery/which.py |
144 |
@@ -18,6 +18,8 @@ __docformat__ = 'epytext' |
145 |
|
146 |
import os |
147 |
import sys |
148 |
+if sys.hexversion < 0x3000000: |
149 |
+ from io import open |
150 |
from getopt import gnu_getopt, GetoptError |
151 |
|
152 |
|
153 |
@@ -62,7 +64,8 @@ def print_help(with_description=True): |
154 |
|
155 |
def print_ebuild(ebuild_path): |
156 |
"""Output the ebuild to std_out""" |
157 |
- with open(_unicode_encode(ebuild_path, encoding=_encodings['fs'])) as f: |
158 |
+ with open(_unicode_encode(ebuild_path, encoding=_encodings['fs']), |
159 |
+ encoding=_encodings['content']) as f: |
160 |
lines = f.readlines() |
161 |
print("\n\n") |
162 |
print("".join(lines)) |
163 |
|
164 |
diff --git a/pym/gentoolkit/eshowkw/keywords_header.py b/pym/gentoolkit/eshowkw/keywords_header.py |
165 |
index bc5c0c5..1b9af0d 100644 |
166 |
--- a/pym/gentoolkit/eshowkw/keywords_header.py |
167 |
+++ b/pym/gentoolkit/eshowkw/keywords_header.py |
168 |
@@ -6,6 +6,9 @@ __all__ = ['keywords_header'] |
169 |
|
170 |
import portage |
171 |
import os |
172 |
+import sys |
173 |
+if sys.hexversion < 0x3000000: |
174 |
+ from io import open |
175 |
from portage import _encodings, _unicode_decode, _unicode_encode |
176 |
from portage import settings as ports |
177 |
from portage.output import colorize |
178 |
@@ -31,7 +34,8 @@ def load_profile_data(portdir=None, repo='gentoo'): |
179 |
|
180 |
try: |
181 |
arch_list = os.path.join(portdir, 'profiles', 'arch.list') |
182 |
- with open(_unicode_encode(arch_list, encoding=_encodings['fs'])) as f: |
183 |
+ with open(_unicode_encode(arch_list, encoding=_encodings['fs']), |
184 |
+ encoding=_encodings['content']) as f: |
185 |
for line in f: |
186 |
line = line.split('#', 1)[0].strip() |
187 |
if line: |
188 |
@@ -47,7 +51,8 @@ def load_profile_data(portdir=None, repo='gentoo'): |
189 |
None: 3, |
190 |
} |
191 |
profiles_list = os.path.join(portdir, 'profiles', 'profiles.desc') |
192 |
- with open(_unicode_encode(profiles_list, encoding=_encodings['fs'])) as f: |
193 |
+ with open(_unicode_encode(profiles_list, encoding=_encodings['fs']), |
194 |
+ encoding=_encodings['content']) as f: |
195 |
for line in f: |
196 |
line = line.split('#', 1)[0].split() |
197 |
if line: |
198 |
|
199 |
diff --git a/pym/gentoolkit/glsa/__init__.py b/pym/gentoolkit/glsa/__init__.py |
200 |
index 30a5ae2..ba1eed7 100644 |
201 |
--- a/pym/gentoolkit/glsa/__init__.py |
202 |
+++ b/pym/gentoolkit/glsa/__init__.py |
203 |
@@ -17,6 +17,8 @@ __author__ = "Marius Mauch <genone@g.o>" |
204 |
|
205 |
|
206 |
import sys |
207 |
+if sys.hexversion < 0x3000000: |
208 |
+ from io import open |
209 |
import os |
210 |
try: |
211 |
from urllib import urlopen |
212 |
@@ -705,7 +707,8 @@ class Glsa: |
213 |
""" |
214 |
if not self.isInjected(): |
215 |
checkfile = open(_unicode_encode(self.config["CHECKFILE"], |
216 |
- encoding=_encodings['fs']), mode="a+") |
217 |
+ encoding=_encodings['fs']), mode="a+", |
218 |
+ encoding=_encodings['content']) |
219 |
checkfile.write(self.nr+"\n") |
220 |
checkfile.close() |
221 |
return None |
222 |
|
223 |
diff --git a/pym/gentoolkit/helpers.py b/pym/gentoolkit/helpers.py |
224 |
index b7314b9..0d985b6 100644 |
225 |
--- a/pym/gentoolkit/helpers.py |
226 |
+++ b/pym/gentoolkit/helpers.py |
227 |
@@ -195,7 +195,8 @@ class ChangeLog(object): |
228 |
result = [] |
229 |
partial_entries = [] |
230 |
with open(_unicode_encode(self.changelog_path, |
231 |
- encoding=_encodings['fs'], errors="replace")) as log: |
232 |
+ encoding=_encodings['fs'], errors="replace"), |
233 |
+ encoding=_encodings['content']) as log: |
234 |
for line in log: |
235 |
if line.startswith('#'): |
236 |
continue |
237 |
|
238 |
diff --git a/pym/gentoolkit/revdep_rebuild/analyse.py b/pym/gentoolkit/revdep_rebuild/analyse.py |
239 |
index 59240b4..9f018b5 100644 |
240 |
--- a/pym/gentoolkit/revdep_rebuild/analyse.py |
241 |
+++ b/pym/gentoolkit/revdep_rebuild/analyse.py |
242 |
@@ -7,6 +7,9 @@ from __future__ import print_function |
243 |
import os |
244 |
import re |
245 |
import time |
246 |
+import sys |
247 |
+if sys.hexversion < 0x3000000: |
248 |
+ from io import open |
249 |
|
250 |
from portage import _encodings, _unicode_decode, _unicode_encode |
251 |
from portage.output import bold, blue, yellow, green |
252 |
@@ -83,7 +86,8 @@ def extract_dependencies_from_la(la, libraries, to_check, logger): |
253 |
if not os.path.exists(_file): |
254 |
continue |
255 |
|
256 |
- for line in open(_unicode_encode(_file, encoding=_encodings['fs']), mode='r').readlines(): |
257 |
+ for line in open(_unicode_encode(_file, encoding=_encodings['fs']), mode='r', |
258 |
+ encoding=_encodings['content']).readlines(): |
259 |
line = line.strip() |
260 |
if line.startswith('dependency_libs='): |
261 |
match = re.match("dependency_libs='([^']+)'", line) |
262 |
|
263 |
diff --git a/pym/gentoolkit/revdep_rebuild/cache.py b/pym/gentoolkit/revdep_rebuild/cache.py |
264 |
index 6d1a1a3..7359d05 100644 |
265 |
--- a/pym/gentoolkit/revdep_rebuild/cache.py |
266 |
+++ b/pym/gentoolkit/revdep_rebuild/cache.py |
267 |
@@ -7,6 +7,9 @@ from __future__ import print_function |
268 |
|
269 |
import os |
270 |
import time |
271 |
+import sys |
272 |
+if sys.hexversion < 0x3000000: |
273 |
+ from io import open |
274 |
|
275 |
from portage import _encodings, _unicode_decode, _unicode_encode |
276 |
from portage.output import red |
277 |
@@ -31,7 +34,7 @@ def read_cache(temp_path=DEFAULTS['DEFAULT_TMP_DIR']): |
278 |
try: |
279 |
for key,val in ret.items(): |
280 |
_file = open(_unicode_encode(os.path.join(temp_path, key), |
281 |
- encoding=_encodings['fs'])) |
282 |
+ encoding=_encodings['fs']), encoding=_encodings['content']) |
283 |
for line in _file.readlines(): |
284 |
val.add(line.strip()) |
285 |
#libraries.remove('\n') |
286 |
@@ -55,13 +58,14 @@ def save_cache(logger, to_save={}, temp_path=DEFAULTS['DEFAULT_TMP_DIR']): |
287 |
|
288 |
try: |
289 |
_file = open(_unicode_encode(os.path.join(temp_path, 'timestamp'), |
290 |
- encoding=_encodings['fs']), mode='w') |
291 |
+ encoding=_encodings['fs']), mode='w', encoding=_encodings['content']) |
292 |
_file.write(str(int(time.time()))) |
293 |
_file.close() |
294 |
|
295 |
for key,val in to_save.items(): |
296 |
_file = open(_unicode_encode(os.path.join(temp_path, key), |
297 |
- encoding=_encodings['fs']), mode='w') |
298 |
+ encoding=_encodings['fs']), mode='w', |
299 |
+ encoding=_encodings['content']) |
300 |
for line in val: |
301 |
_file.write(line + '\n') |
302 |
_file.close() |
303 |
@@ -89,7 +93,8 @@ def check_temp_files(temp_path=DEFAULTS['DEFAULT_TMP_DIR'], max_delay=3600, |
304 |
return False |
305 |
|
306 |
try: |
307 |
- _file = open(_unicode_encode(timestamp_path, encoding=_encodings['fs'])) |
308 |
+ _file = open(_unicode_encode(timestamp_path, encoding=_encodings['fs']), |
309 |
+ encoding=_encodings['content']) |
310 |
timestamp = int(_file.readline()) |
311 |
_file .close() |
312 |
except Exception as ex: |
313 |
|
314 |
diff --git a/pym/gentoolkit/revdep_rebuild/collect.py b/pym/gentoolkit/revdep_rebuild/collect.py |
315 |
index 1f34f1c..ab3ef97 100644 |
316 |
--- a/pym/gentoolkit/revdep_rebuild/collect.py |
317 |
+++ b/pym/gentoolkit/revdep_rebuild/collect.py |
318 |
@@ -9,6 +9,8 @@ import os |
319 |
import glob |
320 |
import stat |
321 |
import sys |
322 |
+if sys.hexversion < 0x3000000: |
323 |
+ from io import open |
324 |
|
325 |
import portage |
326 |
from portage import _encodings, _unicode_decode, _unicode_encode |
327 |
@@ -35,7 +37,8 @@ def parse_conf(conf_file, visited=None, logger=None): |
328 |
|
329 |
for conf in conf_file: |
330 |
try: |
331 |
- with open(_unicode_encode(conf, encoding=_encodings['fs'])) as _file: |
332 |
+ with open(_unicode_encode(conf, encoding=_encodings['fs']), |
333 |
+ encoding=_encodings['content']) as _file: |
334 |
for line in _file.readlines(): |
335 |
line = line.strip() |
336 |
if line.startswith('#'): |
337 |
@@ -77,7 +80,8 @@ def prepare_search_dirs(logger, settings): |
338 |
#try: |
339 |
with open(_unicode_encode(os.path.join( |
340 |
portage.root, settings['DEFAULT_ENV_FILE']), |
341 |
- encoding=_encodings['fs']), mode='r') as _file: |
342 |
+ encoding=_encodings['fs']), mode='r', |
343 |
+ encoding=_encodings['content']) as _file: |
344 |
for line in _file.readlines(): |
345 |
line = line.strip() |
346 |
match = re.match("^export (ROOT)?PATH='([^']+)'", line) |
347 |
|
348 |
diff --git a/pym/gentoolkit/revdep_rebuild/settings.py b/pym/gentoolkit/revdep_rebuild/settings.py |
349 |
index 14f5bc8..589ea29 100644 |
350 |
--- a/pym/gentoolkit/revdep_rebuild/settings.py |
351 |
+++ b/pym/gentoolkit/revdep_rebuild/settings.py |
352 |
@@ -7,6 +7,8 @@ from __future__ import print_function |
353 |
import argparse |
354 |
import os |
355 |
import sys |
356 |
+if sys.hexversion < 0x3000000: |
357 |
+ from io import open |
358 |
import re |
359 |
import glob |
360 |
|
361 |
@@ -138,7 +140,7 @@ def parse_revdep_config(revdep_confdir): |
362 |
|
363 |
for _file in os.listdir(revdep_confdir): |
364 |
for line in open(_unicode_encode(os.path.join(revdep_confdir, _file), |
365 |
- encoding=_encodings['fs'])): |
366 |
+ encoding=_encodings['fs']), encoding=_encodings['content']): |
367 |
line = line.strip() |
368 |
#first check for comment, we do not want to regex all lines |
369 |
if not line.startswith('#'): |
370 |
|
371 |
diff --git a/pym/gentoolkit/test/eclean/creator.py b/pym/gentoolkit/test/eclean/creator.py |
372 |
index db0eba4..63bffd1 100644 |
373 |
--- a/pym/gentoolkit/test/eclean/creator.py |
374 |
+++ b/pym/gentoolkit/test/eclean/creator.py |
375 |
@@ -11,10 +11,13 @@ from __future__ import print_function |
376 |
|
377 |
import os |
378 |
import sys |
379 |
+if sys.hexversion < 0x3000000: |
380 |
+ from io import open |
381 |
import shutil |
382 |
import random |
383 |
|
384 |
import gentoolkit.pprinter as pp |
385 |
+from portage import _encodings, _unicode_decode, _unicode_encode |
386 |
|
387 |
__version__= "0.0.1" |
388 |
__author__ = "Brian Dolbec" |
389 |
@@ -54,7 +57,8 @@ def make_dist(path, files, clean_dict=None): |
390 |
size = random.randint(1000,5000) |
391 |
data = "0" * size |
392 |
filepath = os.path.join(path, file_) |
393 |
- with open(filepath, 'w', file_mode) as new_file: |
394 |
+ with open(_unicode_encode(filepath, encoding=_encodings['fs']), 'w', file_mode, |
395 |
+ encoding=_encodings['content']) as new_file: |
396 |
new_file.write(data) |
397 |
if file_ not in clean_dict: |
398 |
# it is included in a multifile target |