[gentoo-commits] proj/portage:master commit in: pym/portage/emaint/modules/merges/, pym/portage/emaint/

[Pavel Kazakov]

commit:     d705ea3a502d1e92e124a81101501906f8737110
Author:     Pavel Kazakov <nullishzero <AT> gentoo <DOT> org>
AuthorDate: Sun Aug  3 21:45:14 2014 +0000
Commit:     Pavel Kazakov <nullishzero <AT> gentoo <DOT> org>
CommitDate: Sun Aug  3 21:45:14 2014 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=d705ea3a

New emaint module: Merges

This emaint module scans for failed package merges and will display or
fix any failed packages found. This module also saves failed merges
found using a tracking file. Subsequent runs of the module will re-load
that info for re-display, re-emerge of those packages.  Fix typo:
-collision-detect ==> -collision-protect <Brian Dolbec>

---
 pym/portage/emaint/main.py                    |   6 +-
 pym/portage/emaint/modules/merges/__init__.py |  30 +++
 pym/portage/emaint/modules/merges/merges.py   | 290 ++++++++++++++++++++++++++
 3 files changed, 324 insertions(+), 2 deletions(-)

diff --git a/pym/portage/emaint/main.py b/pym/portage/emaint/main.py
index 6a17027..646883d 100644
--- a/pym/portage/emaint/main.py
+++ b/pym/portage/emaint/main.py
@@ -98,10 +98,11 @@ def module_opts(module_controller, module):
 class TaskHandler(object):
 	"""Handles the running of the tasks it is given"""
 
-	def __init__(self, show_progress_bar=True, verbose=True, callback=None):
+	def __init__(self, show_progress_bar=True, verbose=True, callback=None, module_output=None):
 		self.show_progress_bar = show_progress_bar
 		self.verbose = verbose
 		self.callback = callback
+		self.module_output = module_output
 		self.isatty = os.environ.get('TERM') != 'dumb' and sys.stdout.isatty()
 		self.progress_bar = ProgressBar(self.isatty, title="Emaint", max_desc_length=27)
 
@@ -124,6 +125,7 @@ class TaskHandler(object):
 				onProgress = None
 			kwargs = {
 				'onProgress': onProgress,
+				'module_output': self.module_output,
 				# pass in a copy of the options so a module can not pollute or change
 				# them for other tasks if there is more to do.
 				'options': options.copy()
@@ -219,5 +221,5 @@ def emaint_main(myargv):
 	# need to pass the parser options dict to the modules
 	# so they are available if needed.
 	task_opts = options.__dict__
-	taskmaster = TaskHandler(callback=print_results)
+	taskmaster = TaskHandler(callback=print_results, module_output=sys.stdout)
 	taskmaster.run_tasks(tasks, func, status, options=task_opts)

diff --git a/pym/portage/emaint/modules/merges/__init__.py b/pym/portage/emaint/modules/merges/__init__.py
new file mode 100644
index 0000000..96ee71b
--- /dev/null
+++ b/pym/portage/emaint/modules/merges/__init__.py
@@ -0,0 +1,30 @@
+# Copyright 2005-2014 Gentoo Foundation
+# Distributed under the terms of the GNU General Public License v2
+
+"""Scan for failed merges and fix them."""
+
+
+module_spec = {
+	'name': 'merges',
+	'description': __doc__,
+	'provides': {
+		'merges': {
+			'name': "merges",
+			'class': "MergesHandler",
+			'description': __doc__,
+			'functions': ['check', 'fix', 'purge'],
+			'func_desc': {
+				'purge': {
+					'short': '-P', 'long': '--purge-tracker',
+					'help': 'Removes the list of previously failed merges.' +
+							' WARNING: Only use this option if you plan on' +
+							' manually fixing them or do not want them'
+							' re-installed.',
+					'status': "Removing %s",
+					'action': 'store_true',
+					'func': 'purge'
+				}
+			}
+		}
+	}
+}

diff --git a/pym/portage/emaint/modules/merges/merges.py b/pym/portage/emaint/modules/merges/merges.py
new file mode 100644
index 0000000..085fe2a
--- /dev/null
+++ b/pym/portage/emaint/modules/merges/merges.py
@@ -0,0 +1,290 @@
+# Copyright 2005-2014 Gentoo Foundation
+# Distributed under the terms of the GNU General Public License v2
+
+from _emerge.actions import load_emerge_config
+
+import portage
+from portage import os, _unicode_encode
+from portage.const import MERGING_IDENTIFIER, PORTAGE_BIN_PATH, PRIVATE_PATH, \
+	VDB_PATH
+from portage.dep import isvalidatom
+
+import shutil
+import subprocess
+import sys
+import time
+
+class TrackingFile(object):
+	"""File for keeping track of failed merges."""
+
+
+	def __init__(self, tracking_path):
+		"""
+		Create a TrackingFile object.
+
+		@param tracking_path: file path used to keep track of failed merges
+		@type tracking_path: String
+		"""
+		self._tracking_path = _unicode_encode(tracking_path)
+
+
+	def save(self, failed_pkgs):
+		"""
+		Save the specified packages that failed to merge.
+
+		@param failed_pkgs: dictionary of failed packages
+		@type failed_pkgs: dict
+		"""
+		tracking_path = self._tracking_path
+		lines = ['%s %s' % (pkg, mtime) for pkg, mtime in failed_pkgs.items()]
+		portage.util.write_atomic(tracking_path, '\n'.join(lines))
+
+
+	def load(self):
+		"""
+		Load previously failed merges.
+
+		@rtype: dict
+		@return: dictionary of packages that failed to merge
+		"""
+		tracking_path = self._tracking_path
+		if not self.exists():
+			return {}
+		failed_pkgs = {}
+		with open(tracking_path, 'r') as tracking_file:
+			for failed_merge in tracking_file:
+				pkg, mtime = failed_merge.strip().split()
+				failed_pkgs[pkg] = mtime
+		return failed_pkgs
+
+
+	def exists(self):
+		"""
+		Check if tracking file exists.
+
+		@rtype: bool
+		@return: true if tracking file exists, false otherwise
+		"""
+		return os.path.exists(self._tracking_path)
+
+
+	def purge(self):
+		"""Delete previously saved tracking file if one exists."""
+		if self.exists():
+			os.remove(self._tracking_path)
+
+
+	def __iter__(self):
+		"""
+		Provide an interator over failed merges.
+
+		@return: iterator of packages that failed to merge
+		"""
+		return self.load().items().__iter__()
+
+
+class MergesHandler(object):
+	"""Handle failed package merges."""
+
+	short_desc = "Remove failed merges"
+
+	@staticmethod
+	def name():
+		return "merges"
+
+
+	def __init__(self):
+		"""Create MergesHandler object."""
+		eroot = portage.settings['EROOT']
+		tracking_path = os.path.join(eroot, PRIVATE_PATH, 'failed-merges');
+		self._tracking_file = TrackingFile(tracking_path)
+		self._vardb_path = os.path.join(eroot, VDB_PATH)
+
+
+	def can_progressbar(self, func):
+		return func == 'check'
+
+
+	def _scan(self, onProgress=None):
+		"""
+		Scan the file system for failed merges and return any found.
+
+		@param onProgress: function to call for updating progress
+		@type onProgress: Function
+		@rtype: dict
+		@return: dictionary of packages that failed to merges
+		"""
+		failed_pkgs = {}
+		for cat in os.listdir(self._vardb_path):
+			pkgs_path = os.path.join(self._vardb_path, cat)
+			if not os.path.isdir(pkgs_path):
+				continue
+			pkgs = os.listdir(pkgs_path)
+			maxval = len(pkgs)
+			for i, pkg in enumerate(pkgs):
+				if onProgress:
+					onProgress(maxval, i+1)
+				if MERGING_IDENTIFIER in pkg:
+					mtime = int(os.stat(os.path.join(pkgs_path, pkg)).st_mtime)
+					pkg = os.path.join(cat, pkg)
+					failed_pkgs[pkg] = mtime
+		return failed_pkgs
+
+
+	def _failed_pkgs(self, onProgress=None):
+		"""
+		Return failed packages from both the file system and tracking file.
+
+		@rtype: dict
+		@return: dictionary of packages that failed to merges
+		"""
+		failed_pkgs = self._scan(onProgress)
+		for pkg, mtime in self._tracking_file:
+			if pkg not in failed_pkgs:
+				failed_pkgs[pkg] = mtime
+		return failed_pkgs
+
+
+	def _remove_failed_dirs(self, failed_pkgs):
+		"""
+		Remove the directories of packages that failed to merge.
+
+		@param failed_pkgs: failed packages whose directories to remove
+		@type failed_pkg: dict
+		"""
+		for failed_pkg in failed_pkgs:
+			pkg_path = os.path.join(self._vardb_path, failed_pkg)
+			# delete failed merge directory if it exists (it might not exist
+			# if loaded from tracking file)
+			if os.path.exists(pkg_path):
+				shutil.rmtree(pkg_path)
+			# TODO: try removing package CONTENTS to prevent orphaned
+			# files
+
+
+	def _get_pkg_atoms(self, failed_pkgs, pkg_atoms, pkg_invalid_entries):
+		"""
+		Get the package atoms for the specified failed packages.
+
+		@param failed_pkgs: failed packages to iterate
+		@type failed_pkgs: dict
+		@param pkg_atoms: append package atoms to this set
+		@type pkg_atoms: set
+		@param pkg_invalid_entries: append any packages that are invalid to this set
+		@type pkg_invalid_entries: set
+		"""
+
+		emerge_config = load_emerge_config()
+		portdb = emerge_config.target_config.trees['porttree'].dbapi
+		for failed_pkg in failed_pkgs:
+			# validate pkg name
+			pkg_name = '%s' % failed_pkg.replace(MERGING_IDENTIFIER, '')
+			pkg_atom = '=%s' % pkg_name
+
+			if not isvalidatom(pkg_atom):
+				pkg_invalid_entries.append("'%s' is an invalid package atom."
+					% pkg_atom)
+			if not portdb.cpv_exists(pkg_name):
+				pkg_invalid_entries.append(
+					"'%s' does not exist in the portage tree." % pkg_name)
+			pkg_atoms.add(pkg_atom)
+
+
+	def _emerge_pkg_atoms(self, module_output, pkg_atoms):
+		"""
+		Emerge the specified packages atoms.
+
+		@param module_output: output will be written to
+		@type module_output: Class
+		@param pkg_atoms: packages atoms to emerge
+		@type pkg_atoms: set
+		@rtype: list
+		@return: List of results
+		"""
+		# TODO: rewrite code to use portage's APIs instead of a subprocess
+		env = {
+			"FEATURES" : "-collision-protect -protect-owned",
+			"PATH" : os.environ["PATH"]
+		}
+		emerge_cmd = (
+			portage._python_interpreter,
+			'-b',
+			os.path.join(PORTAGE_BIN_PATH, 'emerge'),
+			'--quiet',
+			'--oneshot',
+			'--complete-graph=y'
+		)
+		results = []
+		msg = 'Re-Emerging packages that failed to merge...\n'
+		if module_output:
+			module_output.write(msg)
+		else:
+			module_output = subprocess.PIPE
+			results.append(msg)
+		proc = subprocess.Popen(emerge_cmd + tuple(pkg_atoms), env=env,
+			stdout=module_output, stderr=sys.stderr)
+		output = proc.communicate()[0]
+		if output:
+			results.append(output)
+		if proc.returncode != os.EX_OK:
+			 emerge_status = "Failed to emerge '%s'" % (' '.join(pkg_atoms))
+		else:
+			 emerge_status = "Successfully emerged '%s'" % (' '.join(pkg_atoms))
+		results.append(emerge_status)
+		return results
+
+
+	def check(self, **kwargs):
+		"""Check for failed merges."""
+		onProgress = kwargs.get('onProgress', None)
+		failed_pkgs = self._failed_pkgs(onProgress)
+		errors = []
+		for pkg, mtime in failed_pkgs.items():
+			mtime_str = time.ctime(int(mtime))
+			errors.append("'%s' failed to merge on '%s'" % (pkg, mtime_str))
+		return errors
+
+
+	def fix(self, **kwargs):
+		"""Attempt to fix any failed merges."""
+		module_output = kwargs.get('module_output', None)
+		failed_pkgs = self._failed_pkgs()
+		if not failed_pkgs:
+			return ['No failed merges found.']
+
+		pkg_invalid_entries = set()
+		pkg_atoms = set()
+		self._get_pkg_atoms(failed_pkgs, pkg_atoms, pkg_invalid_entries)
+		if pkg_invalid_entries:
+			return pkg_invalid_entries
+
+		try:
+			self._tracking_file.save(failed_pkgs)
+		except IOError as ex:
+			errors = ['Unable to save failed merges to tracking file: %s\n'
+				% str(ex)]
+			errors.append(', '.join(sorted(failed_pkgs)))
+			return errors
+		self._remove_failed_dirs(failed_pkgs)
+		results = self._emerge_pkg_atoms(module_output, pkg_atoms)
+		# list any new failed merges
+		for pkg in sorted(self._scan()):
+			results.append("'%s' still found as a failed merge." % pkg)
+		# reload config and remove successful packages from tracking file
+		emerge_config = load_emerge_config()
+		vardb = emerge_config.target_config.trees['vartree'].dbapi
+		still_failed_pkgs = {}
+		for pkg, mtime in failed_pkgs.items():
+			pkg_name = '%s' % pkg.replace(MERGING_IDENTIFIER, '')
+			if not vardb.cpv_exists(pkg_name):
+				still_failed_pkgs[pkg] = mtime
+		self._tracking_file.save(still_failed_pkgs)
+		return results
+
+
+	def purge(self, **kwargs):
+		"""Attempt to remove previously saved tracking file."""
+		if not self._tracking_file.exists():
+			return ['Tracking file not found.']
+		self._tracking_file.purge()
+		return ['Removed tracking file.']