Author: grobian
Date: 2008-07-30 13:55:41 +0000 (Wed, 30 Jul 2008)
New Revision: 11286
Modified:
main/branches/prefix/doc/package/ebuild/eapi/2.docbook
main/branches/prefix/man/emerge.1
main/branches/prefix/pym/_emerge/__init__.py
main/branches/prefix/pym/_emerge/help.py
main/branches/prefix/pym/portage/dep.py
main/branches/prefix/pym/portage/sets/dbapi.py
main/branches/prefix/pym/portage/util.py
Log:
Merged from trunk 11248:11267
| 11249 | Bug #233165 - When waiting for jobs and merges to finish in |
| zmedico | Scheduler._main_loop(), keep scheduling the merge queue |
| | since it doesn't autoschedule, and skip the poll loop if |
| | there no event handlers due to synchronous merge tasks being |
| | the only things left to do. |
| 11250 | Fix broken reference to "categories" in |
| zmedico | CategorySet.singleBuilder(). Thanks to Thargor for |
| | reporting. |
| 11251 | Make use of the new config.iteritems() method. |
| zmedico | |
| 11252 | Document the doman language code path translation extension |
| zmedico | from bug #222439. |
| 11253 | Tweak the conditional USE deps examples. |
| zmedico | |
| 11254 | Fix alignment. |
| zmedico | |
| 11255 | Add syntax examples for unconditional USE deps. |
| zmedico | |
| 11256 | Redirect the FEATURES=buildsyspkg message to the log when in |
| zmedico | background mode. |
| 11257 | Fix _use_dep.__str__() to work correctly in the case when |
| zmedico | conditional USE deps have evaluated to nothing (empty string |
| | rather than []). Thanks to ABCD for reporting. |
| 11258 | Implement _use_dep.__nonzero__(). |
| zmedico | |
| 11259 | Tweak table alignment. |
| zmedico | |
| 11260 | In apply_recursive_permissions(), ignore InvalidLocation |
| zmedico | exceptions such as FileNotFound and DirectoryNotFound since |
| | sometimes things disappear, like when adjusting permissions |
| | on DISTCC_DIR. |
| 11261 | Add support for the --jobs option to be specified without an |
| zmedico | argument, and also support -j as a short option. Since |
| | optparse doesn't natively support options with non-required |
| | args, create an insert_optional_args() function that inserts |
| | the required argument into the args so that optparse is |
| | happy. The function inserts the string True as a substitute |
| | for the argument that is required. This string is later |
| | converted to the True constant when stored in the emerge |
| | opts dict (similar to how normal boolean options are |
| | stored). The PollScheduler and SequentialTaskQueue classes |
| | recognize the meaning of the True constant to mean unlimited |
| | concurrent jobs. |
| 11262 | Fix slightly broken loop logic in insert_optional_args() by |
| zmedico | converting it to pop args off of a stack. |
| 11263 | During the first minute of entering the main scheduler loop, |
| zmedico | if --load-average is enabled then limit the rate that new |
| | jobs are spawned, so that the load average measurement has |
| | time to respond to the new load introduced by the new jobs. |
| | The time between spawning new jobs is proportional to the |
| | number of currently running jobs. |
| 11264 | Enable Scheduler._job_delay() whenever --load-average is |
| zmedico | enabled, for whole time the scheduler is running. This |
| | protects against too many jobs being sheduled if the load |
| | average temporarily drops. |
| 11265 | Fix Scheduler._choose_pkg() so that it doesn't choose |
| zmedico | packages prematurely in some cases. |
| 11266 | Make sure Scheduler._choose_pkg() doesn't return a package |
| zmedico | too early when there's no digraph and the previous merge |
| | hasn't completed yet. |
| 11267 | Fix Scheduler._set_digraph() to correctly handle cases when |
| zmedico | max_jobs is True. |
Modified: main/branches/prefix/doc/package/ebuild/eapi/2.docbook
===================================================================
--- main/branches/prefix/doc/package/ebuild/eapi/2.docbook 2008-07-30 10:27:58 UTC (rev 11285)
+++ main/branches/prefix/doc/package/ebuild/eapi/2.docbook 2008-07-30 13:55:41 UTC (rev 11286)
@@ -1,7 +1,67 @@
<sect1 id='package-ebuild-eapi-2'>
<title>EAPI 2_pre0</title>
+ <sect2 id='package-ebuild-eapi-2-helpers'>
+ <title>Helpers</title>
+ <sect3 id='package-ebuild-eapi-2-helpers-doman'>
+ <title>doman</title>
+ <para>
+ Language codes in file names are now used for path translation.
+ </para>
+ <table><title>Man Page Path Translation</title>
+ <tgroup cols='2' align='left' >
+ <colspec colname='source'/>
+ <colspec colname='destination'/>
+ <thead>
+ <row>
+ <entry>Source</entry>
+ <entry>Destination</entry>
+ </row>
+ </thead>
+ <tbody>
+ <row>
+ <entry>foo.1</entry>
+ <entry>/usr/share/man/man1/foo.1</entry>
+ </row>
+ <row>
+ <entry>foo.lang.1</entry>
+ <entry>/usr/share/man/lang/man1/foo.1</entry>
+ </row>
+ </tbody>
+ </tgroup>
+ </table>
+ </sect3>
+ </sect2>
<sect2 id='package-ebuild-eapi-2-use-deps'>
<title>USE Dependencies</title>
+ <sect3 id='package-ebuild-eapi-2-use-deps-unconditional'>
+ <title>Unconditional USE Dependencies</title>
+ <table><title>Syntax Examples</title>
+ <tgroup cols='2' align='left' >
+ <colspec colname='example'/>
+ <colspec colname='meaning'/>
+ <thead>
+ <row>
+ <entry>Example</entry>
+ <entry>Meaning</entry>
+ </row>
+ </thead>
+ <tbody>
+ <row>
+ <entry>foo[bar]</entry>
+ <entry>foo must have bar enabled</entry>
+ </row>
+ <row>
+ <entry>foo[bar,baz]</entry>
+ <entry>foo must have both bar and baz enabled</entry>
+ </row>
+ <row>
+ <entry>foo[-bar,baz]</entry>
+ <entry>foo must have bar disabled and baz enabled</entry>
+ </row>
+ </tbody>
+ </tgroup>
+ </table>
+ </sect3>
<sect3 id='package-ebuild-eapi-2-use-deps-conditional'>
<title>Conditional USE Dependencies</title>
<table><title>Syntax Examples</title>
@@ -17,19 +77,19 @@
<tbody>
<row>
<entry>foo[bar?]</entry>
- <entry>foo bar? ( foo[bar] )</entry>
+ <entry>bar? ( foo[bar] ) !bar? ( foo )</entry>
</row>
<row>
<entry>foo[-bar?]</entry>
- <entry>foo !bar? ( foo[-bar] )</entry>
+ <entry>bar? ( foo ) !bar? ( foo[-bar] )</entry>
</row>
<row>
<entry>foo[bar=]</entry>
- <entry>foo bar? ( foo[bar] ) !bar? ( foo[-bar] )</entry>
+ <entry>bar? ( foo[bar] ) !bar? ( foo[-bar] )</entry>
</row>
<row>
<entry>foo[bar!=]</entry>
- <entry>foo bar? ( foo[-bar] ) !bar? ( foo[bar] )</entry>
+ <entry>bar? ( foo[-bar] ) !bar? ( foo[bar] )</entry>
</row>
</tbody>
</tgroup>
Modified: main/branches/prefix/man/emerge.1
===================================================================
--- main/branches/prefix/man/emerge.1 2008-07-30 10:27:58 UTC (rev 11285)
+++ main/branches/prefix/man/emerge.1 2008-07-30 13:55:41 UTC (rev 11286)
@@ -323,9 +323,10 @@
.BR "\-\-ignore-default-opts"
Causes \fIEMERGE_DEFAULT_OPTS\fR (see \fBmake.conf\fR(5)) to be ignored.
.TP
-.BR \-\-jobs=JOBS
-Specifies the number of packages to build simultaneously. Also see
-the related \fB\-\-load\-average\fR option.
+.BR "-j [JOBS], \-\-jobs[=JOBS]"
+Specifies the number of packages to build simultaneously. If this option is
+given without an argument, emerge will not limit the number of jobs that can
+run simultaneously. Also see the related \fB\-\-load\-average\fR option.
.TP
.BR "\-\-keep\-going"
Continue as much as possible after an error. When an error occurs,
Modified: main/branches/prefix/pym/_emerge/__init__.py
===================================================================
--- main/branches/prefix/pym/_emerge/__init__.py 2008-07-30 10:27:58 UTC (rev 11285)
+++ main/branches/prefix/pym/_emerge/__init__.py 2008-07-30 13:55:41 UTC (rev 11286)
@@ -2348,7 +2348,7 @@
__slots__ = ("args_set", "background", "find_blockers",
"ldpath_mtimes", "logger", "opts", "pkg", "pkg_count",
"prefetcher", "settings", "world_atom") + \
- ("_build_dir", "_buildpkg", "_ebuild_path", "_tree")
+ ("_build_dir", "_buildpkg", "_ebuild_path", "_issyspkg", "_tree")
def _start(self):
@@ -2460,16 +2460,13 @@
logger.log(msg, short_msg=short_msg)
#buildsyspkg: Check if we need to _force_ binary package creation
- issyspkg = "buildsyspkg" in features and \
+ self._issyspkg = "buildsyspkg" in features and \
system_set.findAtomForPackage(pkg) and \
not opts.buildpkg
- if opts.buildpkg or issyspkg:
+ if opts.buildpkg or self._issyspkg:
self._buildpkg = True
- if issyspkg:
- portage.writemsg_stdout(">>> This is a system package, " + \
- "let's pack a rescue tarball.\n", noiselevel=-1)
msg = " === (%s of %s) Compiling/Packaging (%s::%s)" % \
(pkg_count.curval, pkg_count.maxval, pkg.cpv, ebuild_path)
@@ -2506,6 +2503,21 @@
self.wait()
return
+ if self._issyspkg:
+ msg = ">>> This is a system package, " + \
+ "let's pack a rescue tarball.\n"
+
+ log_path = self.settings.get("PORTAGE_LOG_FILE")
+ if log_path is not None:
+ log_file = open(log_path, 'a')
+ try:
+ log_file.write(msg)
+ finally:
+ log_file.close()
+
+ if not self.background:
+ portage.writemsg_stdout(msg, noiselevel=-1)
+
packager = EbuildBinpkg(background=self.background, pkg=self.pkg,
scheduler=self.scheduler, settings=self.settings)
@@ -3286,7 +3298,7 @@
"FILE" : os.path.basename(pkg_path)
}
- fetch_env = dict((k, settings[k]) for k in settings)
+ fetch_env = dict(settings.iteritems())
fetch_args = [portage.util.varexpand(x, mydict=fcmd_vars) \
for x in shlex.split(fcmd)]
@@ -8182,7 +8194,8 @@
if task.poll() is not None:
state_changed = True
- while task_queue and (len(running_tasks) < max_jobs):
+ while task_queue and \
+ (max_jobs is True or len(running_tasks) < max_jobs):
task = task_queue.popleft()
cancelled = getattr(task, "cancelled", None)
if not cancelled:
@@ -8300,10 +8313,12 @@
max_jobs = self._max_jobs
max_load = self._max_load
- if self._running_job_count() >= self._max_jobs:
+ if self._max_jobs is not True and \
+ self._running_job_count() >= self._max_jobs:
return False
- if max_load is not None and max_jobs > 1 and \
+ if max_load is not None and \
+ (max_jobs is True or max_jobs > 1) and \
self._running_job_count() > 1:
try:
avg1, avg5, avg15 = os.getloadavg()
@@ -8886,6 +8901,12 @@
self._max_load = myopts.get("--load-average")
+ # The load average takes some time to respond when new
+ # jobs are added, so we need to limit the rate of adding
+ # new jobs.
+ self._job_delay_factor = 0.5
+ self._previous_job_start_time = None
+
self._set_digraph(digraph)
# This is used to memoize the _choose_pkg() result when
@@ -8940,7 +8961,8 @@
@rtype: bool
@returns: True if background mode is enabled, False otherwise.
"""
- background = (self._max_jobs > 1 or "--quiet" in self.myopts) and \
+ background = (self._max_jobs is True or \
+ self._max_jobs > 1 or "--quiet" in self.myopts) and \
not bool(self._opts_no_background.intersection(self.myopts))
self._status_display.quiet = \
@@ -8955,7 +8977,8 @@
return background
def _set_digraph(self, digraph):
- if self._max_jobs < 2:
+ if self._max_jobs is not True and \
+ self._max_jobs < 2:
# save some memory
self._digraph = None
return
@@ -9587,7 +9610,10 @@
if self._choose_pkg_return_early:
return None
- if self._max_jobs < 2 or self._jobs == 0:
+ if self._digraph is None:
+ if self._jobs or self._task_queues.merge:
+ self._choose_pkg_return_early = True
+ return None
return self._pkg_queue.pop(0)
self._prune_digraph()
@@ -9664,6 +9690,9 @@
self._poll_loop()
while self._jobs or merge_queue:
+ if merge_queue.schedule() and \
+ not self._poll_event_handlers:
+ continue
self._poll_loop()
def _schedule_tasks(self):
@@ -9679,6 +9708,22 @@
self._status_display.display()
return remaining
+ def _job_delay(self):
+ """
+ @rtype: bool
+ @returns: True if job scheduling should be delayed, False otherwise.
+ """
+
+ if self._jobs and self._max_load is not None:
+
+ current_time = time.time()
+
+ if current_time - self._previous_job_start_time < \
+ self._job_delay_factor * self._jobs:
+ return True
+
+ return False
+
def _schedule_tasks_imp(self):
"""
@rtype: bool
@@ -9693,7 +9738,8 @@
return (False, state_change)
if self._choose_pkg_return_early or \
- not self._can_add_job():
+ not self._can_add_job() or \
+ self._job_delay():
return (True, state_change)
pkg = self._choose_pkg()
@@ -9714,12 +9760,14 @@
elif pkg.built:
self._jobs += 1
+ self._previous_job_start_time = time.time()
self._status_display.running = self._jobs
task.addExitListener(self._extract_exit)
self._task_queues.jobs.add(task)
else:
self._jobs += 1
+ self._previous_job_start_time = time.time()
self._status_display.running = self._jobs
task.addExitListener(self._build_exit)
self._task_queues.jobs.add(task)
@@ -12798,6 +12846,63 @@
sys.stderr.write("!!! '%s' or '%s'\n\n" % (action1, action2))
sys.exit(1)
+def insert_optional_args(args):
+ """
+ Parse optional arguments and insert a value if one has
+ not been provided. This is done before feeding the args
+ to the optparse parser since that parser does not support
+ this feature natively.
+ """
+
+ new_args = []
+ jobs_opts = ("-j", "--jobs")
+ arg_stack = args[:]
+ arg_stack.reverse()
+ while arg_stack:
+ arg = arg_stack.pop()
+
+ short_job_opt = bool("j" in arg and arg[:1] == "-" and arg[:2] != "--")
+ if not (short_job_opt or arg in jobs_opts):
+ new_args.append(arg)
+ continue
+
+ # Insert an empty placeholder in order to
+ # satisfy the requirements of optparse.
+
+ new_args.append("--jobs")
+ job_count = None
+ saved_opts = None
+ if short_job_opt and len(arg) > 2:
+ if arg[:2] == "-j":
+ try:
+ job_count = int(arg[2:])
+ except ValueError:
+ saved_opts = arg[2:]
+ else:
+ job_count = "True"
+ saved_opts = arg[1:].replace("j", "")
+
+ if job_count is None and arg_stack:
+ try:
+ job_count = int(arg_stack[-1])
+ except ValueError:
+ pass
+ else:
+ # Discard the job count from the stack
+ # since we're consuming it here.
+ arg_stack.pop()
+
+ if job_count is None:
+ # unlimited number of jobs
+ new_args.append("True")
+ else:
+ new_args.append(str(job_count))
+
+ if saved_opts is not None:
+ new_args.append("-" + saved_opts)
+
+ return new_args
+
def parse_opts(tmpcmdline, silent=False):
myaction=None
myopts = {}
@@ -12868,15 +12973,22 @@
parser.add_option(myopt,
dest=myopt.lstrip("--").replace("-", "_"), **kwargs)
+ tmpcmdline = insert_optional_args(tmpcmdline)
+
myoptions, myargs = parser.parse_args(args=tmpcmdline)
if myoptions.jobs:
- try:
- jobs = int(myoptions.jobs)
- except ValueError:
- jobs = 0
+ jobs = None
+ if myoptions.jobs == "True":
+ jobs = True
+ else:
+ try:
+ jobs = int(myoptions.jobs)
+ except ValueError:
+ jobs = -1
- if jobs < 1:
+ if jobs is not True and \
+ jobs < 1:
jobs = None
if not silent:
writemsg("!!! Invalid --jobs parameter: '%s'\n" % \
Modified: main/branches/prefix/pym/_emerge/help.py
===================================================================
--- main/branches/prefix/pym/_emerge/help.py 2008-07-30 10:27:58 UTC (rev 11285)
+++ main/branches/prefix/pym/_emerge/help.py 2008-07-30 13:55:41 UTC (rev 11286)
@@ -14,7 +14,7 @@
print " "+turquoise("emerge")+" < "+turquoise("--sync")+" | "+turquoise("--metadata")+" | "+turquoise("--info")+" >"
print " "+turquoise("emerge")+" "+turquoise("--resume")+" [ "+green("--pretend")+" | "+green("--ask")+" | "+green("--skipfirst")+" ]"
print " "+turquoise("emerge")+" "+turquoise("--help")+" [ "+green("system")+" | "+green("world")+" | "+green("--sync")+" ] "
- print bold("Options:")+" "+green("-")+"["+green("abBcCdDefgGhkKlnNoOpqPsStuvV")+"]"
+ print bold("Options:")+" "+green("-")+"["+green("abBcCdDefgGhjkKlnNoOpqPsStuvV")+"]"
print " [ " + green("--color")+" < " + turquoise("y") + " | "+ turquoise("n")+" > ] [ "+green("--columns")+" ]"
print " [ "+green("--complete-graph")+" ] [ "+green("--deep")+" ]"
print " [ "+green("--jobs") + " " + turquoise("JOBS")+" ] [ "+green("--keep-going")+" ] [ " + green("--load-average")+" " + turquoise("LOAD") + " ]"
@@ -305,9 +305,12 @@
print " downloaded from the remote server without consulting packages"
print " existing in the packages directory."
print
- print " " + green("--jobs") + " " + turquoise("JOBS")
+ print " " + green("--jobs") + " " + turquoise("[JOBS]") + " ("+green("-j")+" short option)"
desc = "Specifies the number of packages " + \
- "to build simultaneously. Also see " + \
+ "to build simultaneously. If this option is " + \
+ "given without an argument, emerge will not " + \
+ "limit the number of jobs that " + \
+ "can run simultaneously. Also see " + \
"the related --load-average option."
for line in wrap(desc, desc_width):
print desc_indent + line
Modified: main/branches/prefix/pym/portage/dep.py
===================================================================
--- main/branches/prefix/pym/portage/dep.py 2008-07-30 10:27:58 UTC (rev 11285)
+++ main/branches/prefix/pym/portage/dep.py 2008-07-30 13:55:41 UTC (rev 11286)
@@ -395,7 +395,12 @@
self.conditional = conditional
break
+ def __nonzero__(self):
+ return bool(self.tokens)
+
def __str__(self):
+ if not self.tokens:
+ return ""
return "[%s]" % (",".join(self.tokens),)
def evaluate_conditionals(self, use):
@@ -413,17 +418,17 @@
x x= x
-x x= -x
- x x!= -x
- -x x!= x
+ x x!= -x
+ -x x!= x
Conditional syntax examples:
compact form equivalent expanded form
- foo[bar?] foo bar? ( foo[bar] )
- foo[-bar?] foo !bar? ( foo[-bar] )
- foo[bar=] foo bar? ( foo[bar] ) !bar? ( foo[-bar] )
- foo[bar!=] foo bar? ( foo[-bar] ) !bar? ( foo[bar] )
+ foo[bar?] bar? ( foo[bar] ) !bar? ( foo )
+ foo[-bar?] bar? ( foo ) !bar? ( foo[-bar] )
+ foo[bar=] bar? ( foo[bar] ) !bar? ( foo[-bar] )
+ foo[bar!=] bar? ( foo[-bar] ) !bar? ( foo[bar] )
"""
tokens = []
Modified: main/branches/prefix/pym/portage/sets/dbapi.py
===================================================================
--- main/branches/prefix/pym/portage/sets/dbapi.py 2008-07-30 10:27:58 UTC (rev 11285)
+++ main/branches/prefix/pym/portage/sets/dbapi.py 2008-07-30 13:55:41 UTC (rev 11286)
@@ -70,7 +70,7 @@
raise SetConfigError("no category given")
category = options["category"]
- if not category in categories:
+ if not category in settings.categories:
raise SetConfigError("invalid category name '%s'" % category)
repository = cls._builderGetRepository(options, trees.keys())
Modified: main/branches/prefix/pym/portage/util.py
===================================================================
--- main/branches/prefix/pym/portage/util.py 2008-07-30 10:27:58 UTC (rev 11285)
+++ main/branches/prefix/pym/portage/util.py 2008-07-30 13:55:41 UTC (rev 11286)
@@ -801,8 +801,12 @@
if not applied:
all_applied = False
except PortageException, e:
- all_applied = False
- onerror(e)
+ # Ignore InvalidLocation exceptions such as FileNotFound
+ # and DirectoryNotFound since sometimes things disappear,
+ # like when adjusting permissions on DISTCC_DIR.
+ if not isinstance(e, portage.exception.InvalidLocation):
+ all_applied = False
+ onerror(e)
return all_applied
def apply_secpass_permissions(filename, uid=-1, gid=-1, mode=-1, mask=-1,
|
