Author: zmedico
Date: 2008-06-29 14:51:05 +0000 (Sun, 29 Jun 2008)
New Revision: 10849
Modified:
main/trunk/pym/_emerge/__init__.py
main/trunk/pym/portage/__init__.py
Log:
* Add "fd_pipes" and "returnpid" parameters to doebuild() and pass
these into spawn calls, enabling ebuild processes to execute
asynchronously.
* Add a EbuildPhase class that's derived from the pty logging code
inside portage.spawn().
* Integrate post-phase code from spawnebuild() into EbuildBuild.execute()
so that it still gets called even though doebuild() calls execute
asynchronously.
Modified: main/trunk/pym/_emerge/__init__.py
===================================================================
--- main/trunk/pym/_emerge/__init__.py 2008-06-29 09:38:16 UTC (rev 10848)
+++ main/trunk/pym/_emerge/__init__.py 2008-06-29 14:51:05 UTC (rev 10849)
@@ -1541,22 +1541,184 @@
root_config = self.pkg.root_config
portdb = root_config.trees["porttree"].dbapi
ebuild_path = portdb.findname(self.pkg.cpv)
- debug = self.settings.get("PORTAGE_DEBUG") == "1"
+ settings = self.settings
+ debug = settings.get("PORTAGE_DEBUG") == "1"
+ cleanup = 1
retval = portage.doebuild(ebuild_path, "clean",
- root_config.root, self.settings, debug, cleanup=1,
+ root_config.root, settings, debug, cleanup=cleanup,
mydbapi=portdb, tree="porttree")
if retval != os.EX_OK:
return retval
+ # This initializes PORTAGE_LOG_FILE.
+ portage.prepare_build_dirs(root_config.root, settings, cleanup)
+
+ fd_pipes = {
+ 0 : sys.stdin.fileno(),
+ 1 : sys.stdout.fileno(),
+ 2 : sys.stderr.fileno(),
+ }
+
for mydo in self._phases:
- retval = portage.doebuild(ebuild_path, mydo,
- root_config.root, self.settings, debug,
- mydbapi=portdb, tree="porttree")
+ ebuild_phase = EbuildPhase(fd_pipes=fd_pipes,
+ pkg=self.pkg, phase=mydo, settings=settings)
+ ebuild_phase.start()
+ ebuild_phase._output_handler()
+ retval = ebuild_phase.wait()
+
+ portage._post_phase_userpriv_perms(settings)
+ if mydo == "install":
+ portage._check_build_log(settings)
+ if retval == os.EX_OK:
+ retval = portage._post_src_install_checks(settings)
+
if retval != os.EX_OK:
return retval
+
return os.EX_OK
+class EbuildPhase(SlotObject):
+
+ __slots__ = ("fd_pipes", "phase", "pkg", "settings",
+ "pid", "returncode", "files")
+
+ _file_names = ("log", "stdout", "ebuild")
+ _files_dict = slot_dict_class(_file_names)
+ _bufsize = 4096
+
+ def start(self):
+ root_config = self.pkg.root_config
+ portdb = root_config.trees["porttree"].dbapi
+ ebuild_path = portdb.findname(self.pkg.cpv)
+ settings = self.settings
+ debug = settings.get("PORTAGE_DEBUG") == "1"
+ logfile = settings.get("PORTAGE_LOG_FILE")
+ master_fd = None
+ slave_fd = None
+ fd_pipes = self.fd_pipes.copy()
+
+ # flush any pending output
+ for fd in fd_pipes.itervalues():
+ if fd == sys.stdout.fileno():
+ sys.stdout.flush()
+ if fd == sys.stderr.fileno():
+ sys.stderr.flush()
+
+ fd_pipes_orig = None
+ self.files = self._files_dict()
+ files = self.files
+ got_pty = False
+
+ portage._doebuild_exit_status_unlink(
+ settings.get("EBUILD_EXIT_STATUS_FILE"))
+
+ if logfile:
+ if portage._disable_openpty:
+ master_fd, slave_fd = os.pipe()
+ else:
+ from pty import openpty
+ try:
+ master_fd, slave_fd = openpty()
+ got_pty = True
+ except EnvironmentError, e:
+ portage._disable_openpty = True
+ portage.writemsg("openpty failed: '%s'\n" % str(e),
+ noiselevel=-1)
+ del e
+ master_fd, slave_fd = os.pipe()
+
+ if got_pty:
+ # Disable post-processing of output since otherwise weird
+ # things like \n -> \r\n transformations may occur.
+ import termios
+ mode = termios.tcgetattr(slave_fd)
+ mode[1] &= ~termios.OPOST
+ termios.tcsetattr(slave_fd, termios.TCSANOW, mode)
+
+ import fcntl
+ fcntl.fcntl(master_fd, fcntl.F_SETFL,
+ fcntl.fcntl(master_fd, fcntl.F_GETFL) | os.O_NONBLOCK)
+
+ fd_pipes.setdefault(0, sys.stdin.fileno())
+ fd_pipes_orig = fd_pipes.copy()
+ if got_pty and os.isatty(fd_pipes_orig[1]):
+ from portage.output import get_term_size, set_term_size
+ rows, columns = get_term_size()
+ set_term_size(rows, columns, slave_fd)
+ fd_pipes[0] = fd_pipes_orig[0]
+ fd_pipes[1] = slave_fd
+ fd_pipes[2] = slave_fd
+
+ retval = portage.doebuild(ebuild_path, self.phase,
+ root_config.root, settings, debug,
+ mydbapi=portdb, tree="porttree",
+ fd_pipes=fd_pipes, returnpid=True)
+
+ self.pid = retval[0]
+
+ if logfile:
+ os.close(slave_fd)
+ files["log"] = open(logfile, 'a')
+ files["stdout"] = os.fdopen(os.dup(fd_pipes_orig[1]), 'w')
+ files["ebuild"] = os.fdopen(master_fd, 'r')
+
+ def _output_handler(self):
+ log_file = self.files.get("log")
+ if log_file is None:
+ return
+ ebuild_file = self.files["ebuild"]
+ stdout_file = self.files["stdout"]
+ iwtd = [ebuild_file]
+ owtd = []
+ ewtd = []
+ import array, select
+ buffsize = self._bufsize
+ eof = False
+ while not eof:
+ events = select.select(iwtd, owtd, ewtd)
+ for f in events[0]:
+ # Use non-blocking mode to prevent read
+ # calls from blocking indefinitely.
+ buf = array.array('B')
+ try:
+ buf.fromfile(f, buffsize)
+ except EOFError:
+ pass
+ if not buf:
+ eof = True
+ break
+ if f is ebuild_file:
+ buf.tofile(stdout_file)
+ stdout_file.flush()
+ buf.tofile(log_file)
+ log_file.flush()
+ log_file.close()
+ stdout_file.close()
+ ebuild_file.close()
+
+ def wait(self):
+ pid = self.pid
+ retval = os.waitpid(pid, 0)[1]
+ portage.process.spawned_pids.remove(pid)
+ if retval != os.EX_OK:
+ if retval & 0xff:
+ retval = (retval & 0xff) << 8
+ else:
+ retval = retval >> 8
+
+ msg = portage._doebuild_exit_status_check(
+ self.phase, self.settings)
+ if msg:
+ retval = 1
+ from textwrap import wrap
+ from portage.elog.messages import eerror
+ for l in wrap(msg, 72):
+ eerror(l, phase=self.phase, key=self.pkg.cpv)
+
+ self.returncode = retval
+ return self.returncode
+
class EbuildBinpkg(Task):
"""
This assumes that src_install() has successfully completed.
Modified: main/trunk/pym/portage/__init__.py
===================================================================
--- main/trunk/pym/portage/__init__.py 2008-06-29 09:38:16 UTC (rev 10848)
+++ main/trunk/pym/portage/__init__.py 2008-06-29 14:51:05 UTC (rev 10849)
@@ -2940,6 +2940,10 @@
env=mysettings.environ()
keywords["opt_name"]="[%s]" % mysettings["PF"]
+ if "EMERGE_FROM" in mysettings:
+ # emerge handles logging externally
+ keywords.pop("logfile", None)
+
fd_pipes = keywords.get("fd_pipes")
if fd_pipes is None:
fd_pipes = {
@@ -4160,12 +4164,15 @@
return 1
# parse actionmap to spawn ebuild with the appropriate args
-def spawnebuild(mydo,actionmap,mysettings,debug,alwaysdep=0,logfile=None):
+def spawnebuild(mydo, actionmap, mysettings, debug, alwaysdep=0,
+ logfile=None, fd_pipes=None, returnpid=False):
if "EMERGE_FROM" not in mysettings and \
(alwaysdep or "noauto" not in mysettings.features):
# process dependency first
if "dep" in actionmap[mydo]:
- retval=spawnebuild(actionmap[mydo]["dep"],actionmap,mysettings,debug,alwaysdep=alwaysdep,logfile=logfile)
+ retval = spawnebuild(actionmap[mydo]["dep"], actionmap,
+ mysettings, debug, alwaysdep=alwaysdep, logfile=logfile,
+ fd_pipes=fd_pipes, returnpid=returnpid)
if retval:
return retval
kwargs = actionmap[mydo]["args"]
@@ -4177,10 +4184,13 @@
mysettings._filter_calling_env = True
try:
phase_retval = spawn(actionmap[mydo]["cmd"] % mydo,
- mysettings, debug=debug, logfile=logfile, **kwargs)
+ mysettings, debug=debug, logfile=logfile,
+ fd_pipes=fd_pipes, returnpid=returnpid, **kwargs)
finally:
mysettings["EBUILD_PHASE"] = ""
mysettings._filter_calling_env = filter_calling_env_state
+ if returnpid:
+ return phase_retval
msg = _doebuild_exit_status_check(mydo, mysettings)
if msg:
phase_retval = 1
@@ -4189,28 +4199,30 @@
for l in wrap(msg, 72):
eerror(l, phase=mydo, key=mysettings.mycpv)
- if "userpriv" in mysettings.features and \
- not kwargs["droppriv"] and secpass >= 2:
+ _post_phase_userpriv_perms(mysettings)
+ if mydo == "install":
+ _check_build_log(mysettings)
+ if phase_retval == os.EX_OK:
+ phase_retval = _post_src_install_checks(mysettings)
+ return phase_retval
+
+def _post_phase_userpriv_perms(mysettings):
+ if "userpriv" in mysettings.features and secpass >= 2:
""" Privileged phases may have left files that need to be made
writable to a less privileged user."""
apply_recursive_permissions(mysettings["T"],
uid=portage_uid, gid=portage_gid, dirmode=070, dirmask=0,
filemode=060, filemask=0)
- if phase_retval == os.EX_OK:
- if mydo == "install" and logfile:
- _check_build_log(mysettings)
+def _post_src_install_checks(mysettings):
+ _post_src_install_uid_fix(mysettings)
+ retval = _spawn_misc_sh(mysettings, ["install_qa_check",
+ "install_symlink_html_docs"])
+ if retval != os.EX_OK:
+ writemsg("!!! install_qa_check failed; exiting.\n",
+ noiselevel=-1)
+ return retval
- if mydo == "install":
- _post_src_install_uid_fix(mysettings)
- qa_retval = _spawn_misc_sh(mysettings, ["install_qa_check",
- "install_symlink_html_docs"], **kwargs)
- if qa_retval != os.EX_OK:
- writemsg("!!! install_qa_check failed; exiting.\n",
- noiselevel=-1)
- return qa_retval
- return phase_retval
-
def _check_build_log(mysettings):
"""
Search the content of $PORTAGE_LOG_FILE if it exists
@@ -4220,7 +4232,9 @@
* command not found
* Unrecognized configure options
"""
- logfile = mysettings.get("PORTAGE_LOG_FILE", None)
+ logfile = mysettings.get("PORTAGE_LOG_FILE")
+ if logfile is None:
+ return
try:
f = open(logfile, 'rb')
except EnvironmentError:
@@ -4777,8 +4791,9 @@
def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0,
fetchonly=0, cleanup=0, dbkey=None, use_cache=1, fetchall=0, tree=None,
- mydbapi=None, vartree=None, prev_mtimes=None):
-
+ mydbapi=None, vartree=None, prev_mtimes=None,
+ fd_pipes=None, returnpid=False):
+
"""
Wrapper function that invokes specific ebuild phases through the spawning
of ebuild.sh
@@ -5190,7 +5205,10 @@
elif mydo == "setup":
retval = spawn(
_shell_quote(ebuild_sh_binary) + " " + mydo, mysettings,
- debug=debug, free=1, logfile=logfile)
+ debug=debug, free=1, logfile=logfile, fd_pipes=fd_pipes,
+ returnpid=returnpid)
+ if returnpid:
+ return retval
retval = exit_status_check(retval)
if secpass >= 2:
""" Privileged phases may have left files that need to be made
@@ -5418,7 +5436,8 @@
raise portage.exception.PermissionDenied(
"access('%s', os.W_OK)" % parent_dir)
retval = spawnebuild(mydo,
- actionmap, mysettings, debug, logfile=logfile)
+ actionmap, mysettings, debug, logfile=logfile,
+ fd_pipes=fd_pipes, returnpid=returnpid)
elif mydo=="qmerge":
# check to ensure install was run. this *only* pops up when users
# forget it and are using ebuild
@@ -5438,7 +5457,8 @@
mydbapi=mydbapi, vartree=vartree, prev_mtimes=prev_mtimes)
elif mydo=="merge":
retval = spawnebuild("install", actionmap, mysettings, debug,
- alwaysdep=1, logfile=logfile)
+ alwaysdep=1, logfile=logfile, fd_pipes=fd_pipes,
+ returnpid=returnpid)
retval = exit_status_check(retval)
if retval != os.EX_OK:
# The merge phase handles this already. Callers don't know how
--
gentoo-commits@g.o mailing list
|
