[gentoo-commits] proj/autodep:master commit in: portage_with_autodep/pym/_emerge/, /, portage_with_autodep/pym/portage/

[Александр Берсенев]

commit:     745976728dea89598b910f8308c3b948b9c5750c
Author:     Alexander Bersenev <bay <AT> hackerdom <DOT> ru>
AuthorDate: Sat Sep 24 01:49:14 2011 +0000
Commit:     Александр Берсенев <bay <AT> hackerdom <DOT> ru>
CommitDate: Sat Sep 24 01:49:14 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/autodep.git;a=commit;h=74597672

unicode fixes

---
 NOTES                                              |    6 +
 portage_with_autodep/pym/_emerge/EventsAnalyser.py |   40 ++++++--
 portage_with_autodep/pym/_emerge/EventsLogger.py   |  102 +++++++++++---------
 portage_with_autodep/pym/portage/const.py          |    1 +
 4 files changed, 92 insertions(+), 57 deletions(-)

diff --git a/NOTES b/NOTES
index 99a214d..fa18c55 100644
--- a/NOTES
+++ b/NOTES
@@ -31,3 +31,9 @@ This is few notes mainly for myself.
   }
  }
 
+5. Names of files is always raw strings(non-unicode)
+
+6. Commands to launch custom emerge:
+export PATH="/home/bay/autodep/portage_with_autodep/bin:$PATH"
+export PYTHONPATH="/home/bay/autodep/portage_with_autodep/pym:$PYTHONPATH"
+

diff --git a/portage_with_autodep/pym/_emerge/EventsAnalyser.py b/portage_with_autodep/pym/_emerge/EventsAnalyser.py
index 2dc2d35..e6e21be 100644
--- a/portage_with_autodep/pym/_emerge/EventsAnalyser.py
+++ b/portage_with_autodep/pym/_emerge/EventsAnalyser.py
@@ -209,17 +209,30 @@ class GentoolkitUtils:
 				stdin=subprocess.PIPE, stdout=subprocess.PIPE,stderr=subprocess.PIPE, 
 				bufsize=4096)
 						
-			out,err=proc.communicate("\n".join(listtocheck).encode("utf8"))
+			out,err=proc.communicate(b"\n".join(listtocheck))
 			
-			lines=out.decode("utf8").split("\n")
+			lines=out.split(b"\n")
 			#print lines
 			line_re=re.compile(r"^([^ ]+)\s+\(([^)]+)\)$")
 			for line in lines:
+				try:
+				  line=line.decode("utf-8")
+				except UnicodeDecodeError:
+				  portage.util.writemsg("Util qfile returned non-utf8 string: %s\n" % line)
+				
+				#import pdb; pdb.set_trace()
+				
 				if len(line)==0:
 					continue
 				match=line_re.match(line)
 				if match:
-					ret[match.group(2)]=match.group(1)
+					try:
+						ret[match.group(2).encode("utf-8")]=match.group(1)
+					except UnicodeEncodeError:
+					  portage.util.writemsg(
+						"Util qfile failed to encode string %s to unicode\n" % 
+						match.group(2))
+						
 				else:
 					portage.util.writemsg("Util qfile returned unparsable string: %s\n" % line)
 
@@ -243,7 +256,7 @@ class GentoolkitUtils:
 			
 			out,err=proc.communicate()
 			
-			ret=out.decode("utf8").split("\n")
+			ret=out.split(b"\n")
 			if ret==['']:
 				ret=[]
 		except OSError as e:
@@ -265,7 +278,7 @@ class GentoolkitUtils:
 			  
 			out,err=proc.communicate()
 			
-			ret=out.decode("utf8").split("\n")
+			ret=out.split(b"\n")
 		except OSError as e:
 			portage.util.writemsg("Error while launching qfile: %s\n" % e)
 
@@ -390,7 +403,8 @@ class EventsAnalyser:
 		stagesorder={"clean":1,"setup":2,"unpack":3,"prepare":4,"configure":5,"compile":6,"test":7,
 			 "install":8,"preinst":9,"postinst":10,"prerm":11,"postrm":12,"unknown":13}
 		packagesinfo=self.packagesinfo
-		# print information grouped by package	  
+		# print information grouped by package
+		print(packagesinfo.keys())
 		for package in sorted(packagesinfo):
 			# not showing special directory package
 			if package=="directory":
@@ -474,7 +488,12 @@ class EventsAnalyser:
 			
 			for filename in filenames:
 				event_info=tuple(filenames[filename])
-				portage.util.writemsg("  %-56s %-21s\n" % (filename,action[event_info]))
+				try:
+					portage.util.writemsg(
+						"  %-56s %-21s\n" % (filename.decode('utf-8'),action[event_info]))
+				except UnicodeDecodeError:
+					portage.util.writemsg(
+						"  %-56s %-21s\n" % ('NON-UTF8-FILENAME',action[event_info]))
 				filescounter+=1
 				if filescounter>10:
 					portage.util.writemsg("  ... and %d more ...\n" % (len(filenames)-10))
@@ -493,7 +512,7 @@ class EventsAnalyser:
 		""" some basic heuristics here to cut part of packages """
 
 		excluded_paths=set(
-			['/etc/sandbox.d/']
+			[b'/etc/sandbox.d/']
 		)
 
 		excluded_packages=set(
@@ -524,8 +543,9 @@ class EventsAnalyser:
 			  continue
 			
 			# test 1: package is not useful if all files are *.desktop or *.xml or *.m4		
-			if not (f.endswith(".desktop") or f.endswith(".xml") or f.endswith(".m4") or f.endswith(".pc")):
-			  break
+			if not (f.endswith(b".desktop") or f.endswith(b".xml") or 
+				f.endswith(b".m4") or f.endswith(b".pc")):
+				break
 		else:
 			return False # we get here if cycle ends not with break
 		  

diff --git a/portage_with_autodep/pym/_emerge/EventsLogger.py b/portage_with_autodep/pym/_emerge/EventsLogger.py
index 68b3c67..1ade9fd 100644
--- a/portage_with_autodep/pym/_emerge/EventsLogger.py
+++ b/portage_with_autodep/pym/_emerge/EventsLogger.py
@@ -100,62 +100,69 @@ class EventsLogger(threading.Thread):
 							continue
 					
 						#import pdb; pdb.set_trace()
-						try:
-							message=record.decode("utf8").split("\0")
-						except UnicodeDecodeError:
-  							print("Bad message %s" % record)
-  							continue
+						#try:
+						message=record.split(b"\0")
+						#except UnicodeDecodeError:
+  						#	print("Bad message %s" % record)
+  						#	continue
 
 						#  continue
 
 						#print(message)
 					
 						try:
-							if message[4]=="ASKING":
-								if self.filter_proc(message[1],message[2],message[3]):
-									s.sendall(b"ALLOW\0")
-								else:
-									# TODO: log through portage infrastructure
-									#print("Blocking an access to %s" % message[2])
-									s.sendall(b"DENY\0")
+							eventname,filename,stage,result=message[1:5]
+							eventname=eventname.decode("utf-8")
+							stage=stage.decode("utf-8")
+							result=result.decode("utf-8")
+						except IndexError:
+							print("IndexError while parsing %s" % record)
+						except ValueError:
+							print("ValueError while parsing %s" % record)
+						except UnicodeDecodeError:
+							print("UnicodeDecodeError while parsing %s" % record)  
+							
+						if result=="ASKING":
+							if self.filter_proc(eventname,filename,stage):
+								s.sendall(b"ALLOW\0")
 							else:
-								eventname,filename,stage,result=message[1:5]
-
-								if not stage in self.events:
-									self.events[stage]=[{},{}]
+								# TODO: log through portage infrastructure
+								#print("Blocking an access to %s" % message[2])
+								s.sendall(b"DENY\0")
+						else:
+							if not stage in self.events:
+								self.events[stage]=[{},{}]
+					
+							hashofsucesses=self.events[stage][0]
+							hashoffailures=self.events[stage][1]
+
+							if result=="DENIED":
+								print("Blocking an access to %s" % filename)
+
+							if result=="OK":
+								if not filename in hashofsucesses:
+									hashofsucesses[filename]=[False,False]
+					  
+								readed_or_writed=hashofsucesses[filename]
+					  
+								if eventname=="read":
+									readed_or_writed[0]=True
+								elif eventname=="write":
+									readed_or_writed[1]=True
 						
-								hashofsucesses=self.events[stage][0]
-								hashoffailures=self.events[stage][1]
-
-								if result=="DENIED":
-									print("Blocking an access to %s" % filename)
-
-								if result=="OK":
-									if not filename in hashofsucesses:
-										hashofsucesses[filename]=[False,False]
-						  
-									readed_or_writed=hashofsucesses[filename]
-						  
-									if eventname=="read":
-										readed_or_writed[0]=True
-									elif eventname=="write":
-										readed_or_writed[1]=True
-							
-								elif result[0:3]=="ERR" or result=="DENIED":
-									if not filename in hashoffailures:
-										hashoffailures[filename]=[False,False]
-									notfound_or_blocked=hashoffailures[filename]
-								  
-									if result=="ERR/2":
-										notfound_or_blocked[0]=True
-									elif result=="DENIED":
-										notfound_or_blocked[1]=True
-
-								else:
-									print("Error in logger module<->analyser protocol")
+							elif result[0:3]=="ERR" or result=="DENIED":
+								if not filename in hashoffailures:
+									hashoffailures[filename]=[False,False]
+								notfound_or_blocked=hashoffailures[filename]
+							  
+								if result=="ERR/2":
+									notfound_or_blocked[0]=True
+								elif result=="DENIED":
+									notfound_or_blocked[1]=True
+
+							else:
+								print("Error in logger module<->analyser protocol")
 						
-						except IndexError:
-								print("IndexError while parsing %s" % record)
 			except IOError as e:
 				if e.errno!=4: # handling "Interrupted system call" errors
 					raise
@@ -177,4 +184,5 @@ class EventsLogger(threading.Thread):
 		
 		# We assume portage clears tmp folder, so no deleting a socket file
 		# We assume that no new socket data will arrive after this moment
+		#print(self.events)
 		return self.events

diff --git a/portage_with_autodep/pym/portage/const.py b/portage_with_autodep/pym/portage/const.py
index f34398d..2a391db 100644
--- a/portage_with_autodep/pym/portage/const.py
+++ b/portage_with_autodep/pym/portage/const.py
@@ -68,6 +68,7 @@ BASH_BINARY              = "/bin/bash"
 MOVE_BINARY              = "/bin/mv"
 PRELINK_BINARY           = "/usr/sbin/prelink"
 AUTODEP_LIBRARY			 = "/usr/lib/file_hook.so"
+#AUTODEP_LIBRARY			 = "/home/bay/autodep/src/hook_lib/file_hook.so"
 
 
 INVALID_ENV_FILE         = "/etc/spork/is/not/valid/profile.env"