Re: [gentoo-portage-dev] [PATCH] xattr: centralize the various shims in one place

[Arfrever Frehtes Taifersar Arahesis]

2013-10-16 23:03 Mike Frysinger napisał(a):
> Rather than each module implementing its own shim around the various
> methods for accessing extended attributes, start a dedicated module
> that exports a consistent API.

Some things are incompatible with Python 3.
See other comments below.

> ---
>  bin/xattr-helper.py          |  11 +--
>  pym/portage/util/_xattr.py   | 189 +++++++++++++++++++++++++++++++++++++++++++
>  pym/portage/util/movefile.py |  99 ++++++-----------------
>  3 files changed, 213 insertions(+), 86 deletions(-)
>  create mode 100644 pym/portage/util/_xattr.py
>
> diff --git a/bin/xattr-helper.py b/bin/xattr-helper.py
> index 6d99521..69b83f7 100755
> --- a/bin/xattr-helper.py
> +++ b/bin/xattr-helper.py
> @@ -17,16 +17,7 @@ import re
>  import sys
>
>  from portage.util._argparse import ArgumentParser
> -
> -if hasattr(os, "getxattr"):
> -
> -	class xattr(object):
> -		get = os.getxattr
> -		set = os.setxattr
> -		list = os.listxattr
> -
> -else:
> -	import xattr
> +from portage.util._xattr import xattr
>
>
>  _UNQUOTE_RE = re.compile(br'\\[0-7]{3}')
> diff --git a/pym/portage/util/_xattr.py b/pym/portage/util/_xattr.py
> new file mode 100644
> index 0000000..0e594f9
> --- /dev/null
> +++ b/pym/portage/util/_xattr.py
> @@ -0,0 +1,189 @@
> +# Copyright 2010-2013 Gentoo Foundation
> +# Distributed under the terms of the GNU General Public License v2
> +
> +"""Portability shim for xattr support
> +
> +Exported API is the xattr object with get/get_all/set/remove/list operations.
> +
> +See the standard xattr module for more documentation.
> +"""
> +
> +import contextlib
> +import os
> +import subprocess
> +
> +from portage.exception import OperationNotSupported
> +
> +
> +class _XattrGetAll(object):
> +	"""Implement get_all() using list()/get() if there is no easy bulk method"""
> +
> +	@classmethod
> +	def get_all(cls, item, nofollow=False, namespace=None):
> +		return [(name, cls.get(item, name, nofollow=nofollow, namespace=namespace))
> +		        for name in cls.list(item, nofollow=nofollow, namespace=namespace)]
> +
> +
> +class _XattrSystemCommands(_XattrGetAll):
> +	"""Implement things with getfattr/setfattr"""
> +
> +	@staticmethod
> +	def _parse_output(output):
> +		for line in proc.stdout.readlines():

NameError: global name 'proc' is not defined

> +			if line.startswith('#'):
> +				continue
> +			line = line.rstrip()
> +			if not line:
> +				continue
> +			# The line will have the format:
> +			# user.name0="value0"
> +			yield line.split('=', 1)
> +
> +	@classmethod
> +	def get(item, name, nofollow=False, namespace=None):
> +		if namespace:
> +			name = '%s.%s' % (namespace, name)
> +		cmd = ['getfattr', '--absolute-names', '-n', name, item]
> +		if nofollow:
> +			cmd += ['-h']
> +		proc = subprocess.call(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
> +		proc.wait()

AttributeError: 'int' object has no attribute 'wait'

> +
> +		value = None
> +		for _, value in cls._parse_output(proc.stdout):
> +			break
> +
> +		proc.stdout.close()
> +		return value
> +
> +	@staticmethod
> +	def set(item, name, value, flags=0, namespace=None):
> +		if namespace:
> +			name = '%s.%s' % (namespace, name)
> +		cmd = ['setfattr', '-n', name, '-v', value, item]

Calling setfattr once per attribute is slower than once per file (as is now).

> +		subprocess.call(cmd)
> +
> +	@staticmethod
> +	def remove(item, name, nofollow=False, namespace=None):
> +		if namespace:
> +			name = '%s.%s' % (namespace, name)
> +		cmd = ['setfattr', '-x', name, item]
> +		subprocess.call(cmd)
> +
> +	@classmethod
> +	def list(cls, item, nofollow=False, namespace=None, _names_only=True):
> +		cmd = ['getfattr', '-d', '--absolute-names', item]
> +		if nofollow:
> +			cmd += ['-h']
> +		cmd += ['-m', ('^%s[.]' % namespace) if namespace else '']
> +		proc = subprocess.call(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
> +		proc.wait()

AttributeError: 'int' object has no attribute 'wait'

> +
> +		ret = []
> +		if namespace:
> +			namespace = '%s.' % namespace
> +		for name, value in cls._parse_output(proc.stdout):
> +			if namespace:
> +				if name.startswith(namespace):
> +					name = name[len(namespace):]
> +				else:
> +					continue
> +			if _names_only:
> +				ret.append(name)
> +			else:
> +				ret.append((name, value))
> +
> +		proc.stdout.close()
> +		return ret
> +
> +	@classmethod
> +	def get_all(cls, item, nofollow=False, namespace=None):
> +		cls.list(item, nofollow=nofollow, namespace=namespace, _names_only=False)
> +
> +
> +class _XattrStub(_XattrGetAll):
> +	"""Fake object since system doesn't support xattrs"""
> +
> +	@staticmethod
> +	def _raise():
> +		e = OSError('stub')
> +		e.errno = OperationNotSupported.errno
> +		raise e
> +
> +	@staticmethod
> +	def get(item, name, nofollow=False, namespace=None):
> +		raise OperationNotSupported('stub')
> +
> +	@staticmethod
> +	def set(item, name, value, flags=0, namespace=None):
> +		raise OperationNotSupported('stub')
> +
> +	@staticmethod
> +	def remove(item, name, nofollow=False, namespace=None):
> +		raise OperationNotSupported('stub')
> +
> +	@staticmethod
> +	def list(item, nofollow=False, namespace=None):
> +		raise OperationNotSupported('stub')
> +
> +
> +if hasattr(os, 'getxattr'):
> +	# Easy as pie -- active python supports it.
> +	class xattr(_XattrGetAll):
> +		"""Python >=3.3 and GNU/Linux"""
> +		get = os.getxattr
> +		set = os.setxattr
> +		remove = os.removexattr
> +		list = os.listxattr
> +
> +else:
> +	try:
> +		# Maybe we have the xattr module.
> +		import xattr
> +
> +	except ImportError:
> +		try:
> +			# Maybe we have the attr package.
> +			with open(os.devnull, 'wb') as f:
> +				subprocess.call(['getfattr', '--version'], stdout=f)
> +				subprocess.call(['setfattr', '--version'], stdout=f)
> +			xattr = _XattrSystemCommands
> +
> +		except OSError:
> +			# Stub it out completely.
> +			xattr = _XattrStub
> +
> +
> +@××××××××××.contextmanager
> +def preserve_xattrs(path, nofollow=False, namespace=None):
> +	"""Context manager to save/restore extended attributes on |path|
> +
> +	If you want to rewrite a file (possibly replacing it with a new one), but
> +	want to preserve the extended attributes, this will do the trick.
> +
> +	# First read all the extended attributes.
> +	with save_xattrs('/some/file'):
> +		... rewrite the file ...
> +	# Now the extended attributes are restored as needed.
> +	"""
> +	kwargs = {
> +		'nofollow': nofollow,
> +		'namespace': namespace,
> +	}
> +	old_attrs = dict(xattr.get_all(path, **kwargs))
> +	try:
> +		yield
> +	finally:
> +		new_attrs = dict(xattrs.get_all(path, **kwargs))
> +		for name, value in new_attrs.iteritems():
> +			if name not in old_attrs:
> +				# Clear out new ones.
> +				xattr.remove(path, name, **kwargs)
> +			elif new_attrs[name] != old:
> +				# Update changed ones.
> +				xattr.set(path, name, value, **kwargs)
> +
> +		for name, value in old_attrs.iteritems():
> +			if name not in new_attr:
> +				# Re-add missing ones.
> +				xattr.set(path, name, value, **kwargs)
> diff --git a/pym/portage/util/movefile.py b/pym/portage/util/movefile.py
> index 4f158cd..553374a 100644
> --- a/pym/portage/util/movefile.py
> +++ b/pym/portage/util/movefile.py
> @@ -23,6 +23,7 @@ from portage.exception import OperationNotSupported
>  from portage.localization import _
>  from portage.process import spawn
>  from portage.util import writemsg
> +from portage.util._xattr import xattr
>
>  def _apply_stat(src_stat, dest):
>  	_os.chown(dest, src_stat.st_uid, src_stat.st_gid)
> @@ -68,86 +69,32 @@ class _xattr_excluder(object):
>
>  		return False
>
> -if hasattr(_os, "getxattr"):
> -	# Python >=3.3 and GNU/Linux
> -	def _copyxattr(src, dest, exclude=None):
> -
> -		try:
> -			attrs = _os.listxattr(src)
> -		except OSError as e:
> -			if e.errno != OperationNotSupported.errno:
> -				raise
> -			attrs = ()
> -		if attrs:
> -			if exclude is not None and isinstance(attrs[0], bytes):
> -				exclude = exclude.encode(_encodings['fs'])
> -			exclude = _get_xattr_excluder(exclude)
> -
> -		for attr in attrs:
> -			if exclude(attr):
> -				continue
> -			try:
> -				_os.setxattr(dest, attr, _os.getxattr(src, attr))
> -				raise_exception = False
> -			except OSError:
> -				raise_exception = True
> -			if raise_exception:
> -				raise OperationNotSupported(_("Filesystem containing file '%s' "
> -					"does not support extended attribute '%s'") %
> -					(_unicode_decode(dest), _unicode_decode(attr)))
> -else:
> +def _copyxattr(src, dest, exclude=None):
> +	"""Copy the extended attributes from |src| to |dest|"""
>  	try:
> -		import xattr
> -	except ImportError:
> -		xattr = None
> -	if xattr is not None:
> -		def _copyxattr(src, dest, exclude=None):
> -
> -			try:
> -				attrs = xattr.list(src)
> -			except IOError as e:
> -				if e.errno != OperationNotSupported.errno:
> -					raise
> -				attrs = ()
> +		attrs = xattr.list(src)
> +	except (OSError, IOError) as e:
> +		if e.errno != OperationNotSupported.errno:
> +			raise
> +		attrs = ()
>
> -			if attrs:
> -				if exclude is not None and isinstance(attrs[0], bytes):
> -					exclude = exclude.encode(_encodings['fs'])
> -				exclude = _get_xattr_excluder(exclude)
> +	if attrs:
> +		if exclude is not None and isinstance(attrs[0], bytes):
> +			exclude = exclude.encode(_encodings['fs'])
> +		exclude = _get_xattr_excluder(exclude)
>
> -			for attr in attrs:
> -				if exclude(attr):
> -					continue
> -				try:
> -					xattr.set(dest, attr, xattr.get(src, attr))
> -					raise_exception = False
> -				except IOError:
> -					raise_exception = True
> -				if raise_exception:
> -					raise OperationNotSupported(_("Filesystem containing file '%s' "
> -						"does not support extended attribute '%s'") %
> -						(_unicode_decode(dest), _unicode_decode(attr)))
> -	else:
> +	for attr in attrs:
> +		if exclude(attr):
> +			continue
>  		try:
> -			with open(os.devnull, 'wb') as f:
> -				subprocess.call(["getfattr", "--version"], stdout=f)
> -				subprocess.call(["setfattr", "--version"], stdout=f)
> -		except OSError:
> -			def _copyxattr(src, dest, exclude=None):
> -				# TODO: implement exclude
> -				getfattr_process = subprocess.Popen(["getfattr", "-d", "--absolute-names", src], stdout=subprocess.PIPE)
> -				getfattr_process.wait()
> -				extended_attributes = getfattr_process.stdout.readlines()
> -				getfattr_process.stdout.close()
> -				if extended_attributes:
> -					extended_attributes[0] = b"# file: " + _unicode_encode(dest) + b"\n"
> -					setfattr_process = subprocess.Popen(["setfattr", "--restore=-"], stdin=subprocess.PIPE, stderr=subprocess.PIPE)
> -					setfattr_process.communicate(input=b"".join(extended_attributes))
> -					if setfattr_process.returncode != 0:
> -						raise OperationNotSupported("Filesystem containing file '%s' does not support extended attributes" % dest)
> -		else:
> -			def _copyxattr(src, dest, exclude=None):
> -				pass
> +			xattr.set(dest, attr, xattr.get(src, attr))
> +			raise_exception = False
> +		except (OSError, IOError) as e:

Unused variable e

> +			raise_exception = True
> +		if raise_exception:
> +			raise OperationNotSupported(_("Filesystem containing file '%s' "
> +				"does not support extended attribute '%s'") %
> +				(_unicode_decode(dest), _unicode_decode(attr)))
>
>  def movefile(src, dest, newmtime=None, sstat=None, mysettings=None,
>  		hardlink_candidates=None, encoding=_encodings['fs']):
>

--
Arfrever Frehtes Taifersar Arahesis