Author: grobian
Date: 2009-03-09 19:51:11 +0000 (Mon, 09 Mar 2009)
New Revision: 12807
Modified:
main/branches/prefix/bin/ebuild.sh
main/branches/prefix/bin/misc-functions.sh
main/branches/prefix/bin/repoman
main/branches/prefix/man/repoman.1
main/branches/prefix/pym/_emerge/__init__.py
main/branches/prefix/pym/portage/__init__.py
main/branches/prefix/pym/portage/cache/metadata.py
Log:
Merged from trunk -r12790:12806
| 12791 | Also support + character in eclass names. |
| zmedico | |
| 12792 | Fix typo from previous commit. |
| zmedico | |
| 12793 | for pointers cast to 32bit ints, we always want to abort on |
| SpankMan | 64bit systems, not just when FEATURES=stricter |
| 12794 | Generate a QA Notice when EXPORT_FUNCTIONS is called before |
| zmedico | inherit, since it's incompatible with <=portage-2.1.6.7. |
| 12795 | Replace things like [[ $PORTAGE_BUILDDIR/.prepared -nt |
| zmedico | $WORKDIR ]] with simply [[ -e $PORTAGE_BUILDDIR/.prepared |
| | ]] since the timestamp of $WORKDIR is practically always |
| | newer due to the other hidden files that are created there |
| | as each phase is executed. Thanks to Alfredo Tupone |
| | <tupone@g.o> for reporting. |
| 12796 | Make the --digest warning message show for FEATURES=digest. |
| zmedico | |
| 12797 | Bug #261675 - When doebuild() is called by emerge, don't |
| zmedico | call digestgen() for FEATURES=digest because it's redundant |
| | and it can cause interference with parallel-fetch and |
| | parallel builds. |
| 12798 | Bug #261675 - Also don't call digestgen() from |
| zmedico | parallel-fetch processes. |
| 12799 | Bug #261675 - Generate manifests in advance when |
| zmedico | FEATURES=digest is enabled. |
| 12800 | Never call digestgen() inside doebuild() which only 'fetch' |
| zmedico | has been requested since it's not needed in this case. |
| 12801 | Reorganize code for --digests and FEATURES=digest. |
| zmedico | |
| 12802 | Make manifest mode with --force option cause existing |
| zmedico | digests to be replaced for any files that exist in |
| | ${DISTDIR}. This provides an alternative to ebuild --force |
| | manifest for updating existing distfiles digests. Digests |
| | are assumed to be corect for files that do not exist in |
| | ${DISTDIR} since the user could simply remove the whole |
| | Manifest if they wanted to regenerate digests for all files |
| | (and it's safer to regenerate as few as possible because |
| | it's less probably that a valid digest will get replaced by |
| | an invalid one). |
| 12803 | Fix _setitem() to account for the change to |
| zmedico | serialize_eclasses = False. |
| 12804 | Inside dyn_unpack(), only call pre/post phase hooks when |
| zmedico | src_unpack is called (when .unpacked does not already |
| | exist). |
| 12805 | Use the ebuild_phase helper function to calls phase hooks |
| zmedico | and fix broken trap calls that try to unset multiple signal |
| | handlers at once without passing - as the first argument. |
| 12806 | When generating variable names to hold EXPORT_FUNCTIONS |
| zmedico | argument in, use $ECLASS_DEPTH as a unique id since it's a |
| | lot simpler then encoding the eclass name in the variable |
| | name. |
Modified: main/branches/prefix/bin/ebuild.sh
===================================================================
--- main/branches/prefix/bin/ebuild.sh 2009-03-09 10:32:13 UTC (rev 12806)
+++ main/branches/prefix/bin/ebuild.sh 2009-03-09 19:51:11 UTC (rev 12807)
@@ -649,7 +649,6 @@
}
dyn_unpack() {
- [ "$(type -t pre_src_unpack)" == "function" ] && qa_call pre_src_unpack
local newstuff="no"
if [ -e "${WORKDIR}" ]; then
local x
@@ -682,7 +681,6 @@
if [ -e "${WORKDIR}" ]; then
if [ "$newstuff" == "no" ]; then
vecho ">>> WORKDIR is up-to-date, keeping..."
- [ "$(type -t post_src_unpack)" == "function" ] && qa_call post_src_unpack
return 0
fi
fi
@@ -691,13 +689,12 @@
install -m${PORTAGE_WORKDIR_MODE:-0700} -d "${WORKDIR}" || die "Failed to create dir '${WORKDIR}'"
fi
cd "${WORKDIR}" || die "Directory change failed: \`cd '${WORKDIR}'\`"
+ ebuild_phase pre_src_unpack
vecho ">>> Unpacking source..."
ebuild_phase src_unpack
touch "${PORTAGE_BUILDDIR}/.unpacked" || die "IO Failure -- Failed 'touch .unpacked' in ${PORTAGE_BUILDDIR}"
vecho ">>> Source unpacked in ${WORKDIR}"
- cd "${PORTAGE_BUILDDIR}"
-
- [ "$(type -t post_src_unpack)" == "function" ] && qa_call post_src_unpack
+ ebuild_phase post_src_unpack
}
dyn_clean() {
@@ -824,7 +821,7 @@
echo
eval ${3}
#unset signal handler
- trap SIGINT SIGQUIT
+ trap - SIGINT SIGQUIT
}
abort_prepare() {
@@ -859,7 +856,7 @@
dyn_prepare() {
- if [[ $PORTAGE_BUILDDIR/.prepared -nt $WORKDIR ]] ; then
+ if [[ -e $PORTAGE_BUILDDIR/.prepared ]] ; then
vecho ">>> It appears that '$PF' is already prepared; skipping."
vecho ">>> Remove '$PORTAGE_BUILDDIR/.prepared' to force prepare."
return 0
@@ -882,12 +879,12 @@
vecho ">>> Source prepared."
ebuild_phase post_src_prepare
- trap SIGINT SIGQUIT
+ trap - SIGINT SIGQUIT
}
dyn_configure() {
- if [[ $PORTAGE_BUILDDIR/.configured -nt $WORKDIR ]] ; then
+ if [[ -e $PORTAGE_BUILDDIR/.configured ]] ; then
vecho ">>> It appears that '$PF' is already configured; skipping."
vecho ">>> Remove '$PORTAGE_BUILDDIR/.configured' to force configuration."
return 0
@@ -895,23 +892,21 @@
trap abort_configure SIGINT SIGQUIT
- [[ $(type -t pre_src_configure) = function ]] && \
- qa_call pre_src_configure
+ ebuild_phase pre_src_configure
vecho ">>> Configuring source in $srcdir ..."
ebuild_phase src_configure
touch "$PORTAGE_BUILDDIR"/.configured
vecho ">>> Source configured."
- [[ $(type -t post_src_configure) = function ]] && \
- qa_call post_src_configure
+ ebuild_phase post_src_configure
- trap SIGINT SIGQUIT
+ trap - SIGINT SIGQUIT
}
dyn_compile() {
- if [[ $PORTAGE_BUILDDIR/.compiled -nt $WORKDIR ]] ; then
+ if [[ -e $PORTAGE_BUILDDIR/.compiled ]] ; then
vecho ">>> It appears that '${PF}' is already compiled; skipping."
vecho ">>> Remove '$PORTAGE_BUILDDIR/.compiled' to force compilation."
return 0
@@ -919,17 +914,16 @@
trap abort_compile SIGINT SIGQUIT
- [[ $(type -t pre_src_compile) = function ]] && \
- qa_call pre_src_compile
+ ebuild_phase pre_src_compile
vecho ">>> Compiling source in ${srcdir} ..."
ebuild_phase src_compile
touch "$PORTAGE_BUILDDIR"/.compiled
vecho ">>> Source compiled."
- [ "$(type -t post_src_compile)" == "function" ] && qa_call post_src_compile
+ ebuild_phase post_src_compile
- trap SIGINT SIGQUIT
+ trap - SIGINT SIGQUIT
}
dyn_test() {
@@ -939,10 +933,9 @@
# like it's supposed to here.
! hasq test ${USE} && export USE="${USE} test"
fi
- [ "$(type -t pre_src_test)" == "function" ] && qa_call pre_src_test
- if [ "${PORTAGE_BUILDDIR}/.tested" -nt "${WORKDIR}" ]; then
+ ebuild_phase pre_src_test
+ if [[ -e $PORTAGE_BUILDDIR/.tested ]] ; then
vecho ">>> It appears that ${PN} has already been tested; skipping."
- [ "$(type -t post_src_test)" == "function" ] && qa_call post_src_test
return
fi
trap "abort_test" SIGINT SIGQUIT
@@ -962,23 +955,23 @@
SANDBOX_PREDICT="${SANDBOX_PREDICT%:/}"
fi
- cd "${PORTAGE_BUILDDIR}"
- touch .tested || die "Failed to 'touch .tested' in ${PORTAGE_BUILDDIR}"
- [ "$(type -t post_src_test)" == "function" ] && qa_call post_src_test
- trap SIGINT SIGQUIT
+ touch "$PORTAGE_BUILDDIR/.tested" || \
+ die "Failed to 'touch .tested' in $PORTAGE_BUILDDIR"
+ ebuild_phase post_src_test
+ trap - SIGINT SIGQUIT
}
dyn_install() {
[ -z "$PORTAGE_BUILDDIR" ] && die "${FUNCNAME}: PORTAGE_BUILDDIR is unset"
if hasq noauto $FEATURES ; then
rm -f "${PORTAGE_BUILDDIR}/.installed"
- elif [[ ${PORTAGE_BUILDDIR}/.installed -nt ${WORKDIR} ]] ; then
+ elif [[ -e $PORTAGE_BUILDDIR/.installed ]] ; then
vecho ">>> It appears that '${PF}' is already installed; skipping."
vecho ">>> Remove '${PORTAGE_BUILDDIR}/.installed' to force install."
return 0
fi
trap "abort_install" SIGINT SIGQUIT
- [ "$(type -t pre_src_install)" == "function" ] && qa_call pre_src_install
+ ebuild_phase pre_src_install
rm -rf "${PORTAGE_BUILDDIR}/image"
mkdir "${PORTAGE_BUILDDIR}/image"
if [ -d "${S}" ]; then
@@ -1007,8 +1000,7 @@
touch "${PORTAGE_BUILDDIR}/.installed"
vecho ">>> Completed installing ${PF} into ${ED}"
vecho
- cd ${PORTAGE_BUILDDIR}
- [ "$(type -t post_src_install)" == "function" ] && qa_call post_src_install
+ ebuild_phase post_src_install
cd "${PORTAGE_BUILDDIR}"/build-info
set -f
@@ -1041,7 +1033,7 @@
then
touch DEBUGBUILD
fi
- trap SIGINT SIGQUIT
+ trap - SIGINT SIGQUIT
}
dyn_preinst() {
@@ -1166,6 +1158,13 @@
debug-print "*** Multiple Inheritence (Level: ${ECLASS_DEPTH})"
fi
+ if [[ -n $ECLASS && -n ${!__export_funcs_var} ]] ; then
+ echo "QA Notice: EXPORT_FUNCTIONS is called before inherit in" \
+ "$ECLASS.eclass. For compatibility with <=portage-2.1.6.7," \
+ "only call EXPORT_FUNCTIONS after inherit(s)." \
+ | fmt -w 75 | while read ; do eqawarn "$REPLY" ; done
+ fi
+
local location
local olocation
local x
@@ -1183,11 +1182,7 @@
olocation=""
export ECLASS="$1"
- __export_funcs_var=__export_functions_$ECLASS
- while [[ $__export_funcs_var =~ [-.] ]] ; do
- __export_funcs_var=${__export_funcs_var/-/__dash__}
- __export_funcs_var=${__export_funcs_var/./__dot__}
- done
+ __export_funcs_var=__export_functions_$ECLASS_DEPTH
unset $__export_funcs_var
if [ "${EBUILD_PHASE}" != "depend" ] && \
Modified: main/branches/prefix/bin/misc-functions.sh
===================================================================
--- main/branches/prefix/bin/misc-functions.sh 2009-03-09 10:32:13 UTC (rev 12806)
+++ main/branches/prefix/bin/misc-functions.sh 2009-03-09 19:51:11 UTC (rev 12807)
@@ -536,14 +536,18 @@
fi
fi
- if [[ $abort = yes ]] && [[ $gentoo_bug != yes ]] ; then
- echo "Please do not file a Gentoo bug and instead" \
- "report the above QA issues directly to the upstream" \
- "developers of this software." | fmt -w 70 | \
- while read line ; do eqawarn "${line}" ; done
- eqawarn "Homepage: ${HOMEPAGE}"
+ if [[ ${abort} == "yes" ]] ; then
+ if [[ ${gentoo_bug} == "yes" ]] ; then
+ die "poor code kills airplanes"
+ else
+ echo "Please do not file a Gentoo bug and instead" \
+ "report the above QA issues directly to the upstream" \
+ "developers of this software." | fmt -w 70 | \
+ while read line ; do eqawarn "${line}" ; done
+ eqawarn "Homepage: ${HOMEPAGE}"
+ hasq stricter ${FEATURES} && die "poor code kills airplanes"
+ fi
fi
- [[ ${abort} == "yes" ]] && hasq stricter ${FEATURES} && die "poor code kills airplanes"
fi
# Compiled python objects do not belong in /usr/share (FHS violation)
Modified: main/branches/prefix/bin/repoman
===================================================================
--- main/branches/prefix/bin/repoman 2009-03-09 10:32:13 UTC (rev 12806)
+++ main/branches/prefix/bin/repoman 2009-03-09 19:51:11 UTC (rev 12807)
@@ -799,11 +799,50 @@
if options.mode == "manifest" or \
options.mode in ('commit', 'fix') and not options.pretend:
+ auto_assumed = set()
+ fetchlist_dict = portage.FetchlistDict(checkdir,
+ repoman_settings, portdb)
+ if options.mode == 'manifest' and options.force:
+ portage._doebuild_manifest_exempt_depend += 1
+ try:
+ distdir = repoman_settings['DISTDIR']
+ mf = portage.manifest.Manifest(checkdir, distdir,
+ fetchlist_dict=fetchlist_dict)
+ mf.create(requiredDistfiles=None,
+ assumeDistHashesAlways=True)
+ for distfiles in fetchlist_dict.itervalues():
+ for distfile in distfiles:
+ if os.path.isfile(os.path.join(distdir, distfile)):
+ mf.fhashdict['DIST'].pop(distfile, None)
+ else:
+ auto_assumed.add(distfile)
+ mf.write()
+ finally:
+ portage._doebuild_manifest_exempt_depend -= 1
+
repoman_settings["O"] = checkdir
if not portage.digestgen([], repoman_settings, myportdb=portdb):
print "Unable to generate manifest."
dofail = 1
if options.mode == "manifest":
+ if not dofail and options.force and auto_assumed and \
+ 'assume-digests' in repoman_settings.features:
+ # Show which digests were assumed despite the --force option
+ # being given. This output will already have been shown by
+ # digestgen() if assume-digests is not enabled, so only show
+ # it here if assume-digests is enabled.
+ pkgs = list(fetchlist_dict)
+ pkgs.sort()
+ portage.writemsg_stdout(" digest.assumed" + \
+ portage.output.colorize("WARN",
+ str(len(auto_assumed)).rjust(18)) + "\n")
+ for cpv in pkgs:
+ fetchmap = fetchlist_dict[cpv]
+ pf = portage.catsplit(cpv)[1]
+ for distfile in sorted(fetchmap):
+ if distfile in auto_assumed:
+ portage.writemsg_stdout(
+ " %s::%s\n" % (pf, distfile))
continue
elif dofail:
sys.exit(1)
Modified: main/branches/prefix/man/repoman.1
===================================================================
--- main/branches/prefix/man/repoman.1 2009-03-09 10:32:13 UTC (rev 12806)
+++ main/branches/prefix/man/repoman.1 2009-03-09 19:51:11 UTC (rev 12807)
@@ -18,6 +18,14 @@
causes the most time consuming QA checks to be skipped. The commit message will
include an indication that this option has been enabled, together with the
usual portage version stamp.
+
+When used together with \fBmanifest\fR mode, \fB--force\fR causes existing
+digests to be replaced for any files that exist in ${DISTDIR}.
+Existing digests are assumed to be correct for files that would otherwise
+have to be downloaded in order to recompute digests. \fBWARNING:\fR When
+replacing existing digests, it is the user's responsibility to ensure that
+files contained in ${DISTDIR} have the correct identities. Especially beware
+of partially downloaded files.
.TP
\fB-q\fR, \fB--quiet\fR
Be less verbose about extraneous info
@@ -69,7 +77,8 @@
Fix simple QA issues (stray digests, missing digests)
.TP
.B manifest
-Generate a Manifest (fetches files if necessary)
+Generate a Manifest (fetches distfiles if necessary). See the \fB\-\-force\fR
+option if you would like to replace existing distfiles digests.
.TP
.B commit
Scan directory tree for QA issues; if OK, commit via cvs
Modified: main/branches/prefix/pym/_emerge/__init__.py
===================================================================
--- main/branches/prefix/pym/_emerge/__init__.py 2009-03-09 10:32:13 UTC (rev 12806)
+++ main/branches/prefix/pym/_emerge/__init__.py 2009-03-09 19:51:11 UTC (rev 12807)
@@ -10470,6 +10470,51 @@
return ebuild_phase.returncode
+ def _generate_digests(self):
+ """
+ Generate digests if necessary for --digests or FEATURES=digest.
+ In order to avoid interference, this must done before parallel
+ tasks are started.
+ """
+
+ if '--fetchonly' in self.myopts:
+ return os.EX_OK
+
+ digest = '--digest' in self.myopts
+ if not digest:
+ for pkgsettings in self.pkgsettings.itervalues():
+ if 'digest' in pkgsettings.features:
+ digest = True
+ break
+
+ if not digest:
+ return os.EX_OK
+
+ for x in self._mergelist:
+ if not isinstance(x, Package) or \
+ x.type_name != 'ebuild' or \
+ x.operation != 'merge':
+ continue
+ pkgsettings = self.pkgsettings[x.root]
+ if '--digest' not in self.myopts and \
+ 'digest' not in pkgsettings.features:
+ continue
+ portdb = x.root_config.trees['porttree'].dbapi
+ ebuild_path = portdb.findname(x.cpv)
+ if not ebuild_path:
+ writemsg_level(
+ "!!! Could not locate ebuild for '%s'.\n" \
+ % x.cpv, level=logging.ERROR, noiselevel=-1)
+ return 1
+ pkgsettings['O'] = os.path.dirname(ebuild_path)
+ if not portage.digestgen([], pkgsettings, myportdb=portdb):
+ writemsg_level(
+ "!!! Unable to generate manifest for '%s'.\n" \
+ % x.cpv, level=logging.ERROR, noiselevel=-1)
+ return 1
+
+ return os.EX_OK
+
def _check_manifests(self):
# Verify all the manifests now so that the user is notified of failure
# as soon as possible.
@@ -10678,6 +10723,10 @@
self.pkgsettings[root] = portage.config(
clone=root_config.settings)
+ rval = self._generate_digests()
+ if rval != os.EX_OK:
+ return rval
+
rval = self._check_manifests()
if rval != os.EX_OK:
return rval
@@ -14350,8 +14399,13 @@
if pretend or fetchonly:
# make the mtimedb readonly
mtimedb.filename = None
- if "--digest" in myopts:
- msg = "The --digest option can prevent corruption from being" + \
+ if '--digest' in myopts or 'digest' in settings.features:
+ if '--digest' in myopts:
+ msg = "The --digest option"
+ else:
+ msg = "The FEATURES=digest setting"
+
+ msg += " can prevent corruption from being" + \
" noticed. The `repoman manifest` command is the preferred" + \
" way to generate manifests and it is capable of doing an" + \
" entire repository or category at once."
@@ -14657,20 +14711,6 @@
# Convert Atom instances to plain str.
mtimedb["resume"]["favorites"] = [str(x) for x in favorites]
- if ("--digest" in myopts) and not ("--fetchonly" in myopts or "--fetch-all-uri" in myopts):
- for pkgline in mydepgraph.altlist():
- if pkgline[0]=="ebuild" and pkgline[3]=="merge":
- y = trees[pkgline[1]]["porttree"].dbapi.findname(pkgline[2])
- tmpsettings = portage.config(clone=settings)
- edebug = 0
- if settings.get("PORTAGE_DEBUG", "") == "1":
- edebug = 1
- retval = portage.doebuild(
- y, "digest", settings["ROOT"], tmpsettings, edebug,
- ("--pretend" in myopts),
- mydbapi=trees[pkgline[1]]["porttree"].dbapi,
- tree="porttree")
-
pkglist = mydepgraph.altlist()
mydepgraph.saveNomergeFavorites()
mydepgraph.break_refs(pkglist)
Modified: main/branches/prefix/pym/portage/__init__.py
===================================================================
--- main/branches/prefix/pym/portage/__init__.py 2009-03-09 10:32:13 UTC (rev 12806)
+++ main/branches/prefix/pym/portage/__init__.py 2009-03-09 19:51:11 UTC (rev 12807)
@@ -5996,6 +5996,7 @@
mycpv = "/".join((mysettings["CATEGORY"], mysettings["PF"]))
emerge_skip_distfiles = returnpid
+ emerge_skip_digest = returnpid
# Only try and fetch the files if we are going to need them ...
# otherwise, if user has FEATURES=noauto and they run `ebuild clean
# unpack compile install`, we will try and fetch 4 times :/
@@ -6052,7 +6053,11 @@
elif mydo == "digest":
return not digestgen(aalist, mysettings, overwrite=1,
myportdb=mydbapi)
- elif "digest" in mysettings.features:
+ elif mydo != 'fetch' and not emerge_skip_digest and \
+ "digest" in mysettings.features:
+ # Don't do this when called by emerge or when called just
+ # for fetch (especially parallel-fetch) since it's not needed
+ # and it can interfere with parallel tasks.
digestgen(aalist, mysettings, overwrite=0, myportdb=mydbapi)
except portage.exception.PermissionDenied, e:
writemsg("!!! Permission Denied: %s\n" % (e,), noiselevel=-1)
Modified: main/branches/prefix/pym/portage/cache/metadata.py
===================================================================
--- main/branches/prefix/pym/portage/cache/metadata.py 2009-03-09 10:32:13 UTC (rev 12806)
+++ main/branches/prefix/pym/portage/cache/metadata.py 2009-03-09 19:51:11 UTC (rev 12807)
@@ -71,8 +71,7 @@
def _setitem(self, cpv, values):
if "_eclasses_" in values:
values = ProtectedDict(values)
- values["INHERITED"] = ' '.join(sorted(
- reconstruct_eclasses(cpv, values["_eclasses_"])))
+ values["INHERITED"] = ' '.join(sorted(values["_eclasses_"]))
s = cpv.rfind("/")
fp = os.path.join(self.location,cpv[:s],
|
