[gentoo-portage-dev] [PATCH v2] Support disabling docompress for binary package builds

[Michał Górny]

Add FEATURES=binpkg-docompress that can be used whether docompress
compression is performed before or after creating binary packages.  With
the feature enabled (the default), the current behavior of storing
compressed files in binpkg is preserved.  With it disabled, uncompressed
files are stored inside binary package and are compressed when
installing.

Storing uncompressed files in binary packages has two advantages:

1. Avoids the double-compression penalty, effectively improving binary
   package compression speed and compression ratio.

2. Allows the same packages to be reused on systems with different
   docompress configurations.

The option is roughly backwards compatible.  Old Portage versions will
install packages created with FEATURES=-binpkg-docompress correctly,
albeit without compression.  Portage with FEATURES=binpkg-docompress
should install old binpackages semi-correctly, potentially recompressing
them (and throwing already-compressed warnings on format mismatch).
The new behavior is left off by default to avoid those problems.

Signed-off-by: Michał Górny <mgorny@g.o>
---
 bin/misc-functions.sh                  | 43 +++++++++++++++++++++++---
 bin/phase-functions.sh                 |  2 +-
 cnf/make.globals                       |  2 +-
 lib/portage/const.py                   |  1 +
 lib/portage/dbapi/vartree.py           | 12 +++++++
 lib/portage/package/ebuild/doebuild.py |  4 ++-
 man/ebuild.1                           |  5 +++
 man/make.conf.5                        |  6 ++++
 8 files changed, 68 insertions(+), 7 deletions(-)

Changes in v2:
* added .instprepped file logic,
* added instprep -> install dependency,
* documented in ebuild.1.

I've tested this thoroughly via ebuild(1) and emerge(1).

diff --git a/bin/misc-functions.sh b/bin/misc-functions.sh
index db7aaed5a..3b3a5e09c 100755
--- a/bin/misc-functions.sh
+++ b/bin/misc-functions.sh
@@ -109,10 +109,13 @@ install_qa_check() {
 
 	[[ -d ${ED%/}/usr/share/info ]] && prepinfo
 
-	# Apply compression.
-	"${PORTAGE_BIN_PATH}"/ecompress --queue "${PORTAGE_DOCOMPRESS[@]}"
-	"${PORTAGE_BIN_PATH}"/ecompress --ignore "${PORTAGE_DOCOMPRESS_SKIP[@]}"
-	"${PORTAGE_BIN_PATH}"/ecompress --dequeue
+	# If binpkg-docompress is enabled, apply compression before creating
+	# the binary package.
+	if has binpkg-docompress ${FEATURES}; then
+		"${PORTAGE_BIN_PATH}"/ecompress --queue "${PORTAGE_DOCOMPRESS[@]}"
+		"${PORTAGE_BIN_PATH}"/ecompress --ignore "${PORTAGE_DOCOMPRESS_SKIP[@]}"
+		"${PORTAGE_BIN_PATH}"/ecompress --dequeue
+	fi
 
 	export STRIP_MASK
 	if ___eapi_has_dostrip; then
@@ -160,6 +163,38 @@ install_qa_check() {
 	rm -f "${ED%/}"/usr/share/info/dir{,.gz,.bz2} || die "rm failed!"
 }
 
+__dyn_instprep() {
+	if [[ -e ${PORTAGE_BUILDDIR}/.instprepped ]] ; then
+		__vecho ">>> It appears that '$PF' is already instprepped; skipping."
+		__vecho ">>> Remove '${PORTAGE_BUILDDIR}/.instprepped' to force instprep."
+		return 0
+	fi
+
+	if has chflags ${FEATURES}; then
+		# Save all the file flags for restoration afterwards.
+		mtree -c -p "${ED}" -k flags > "${T}/bsdflags.mtree"
+		# Remove all the file flags so that we can do anything necessary.
+		chflags -R noschg,nouchg,nosappnd,nouappnd "${ED}"
+		chflags -R nosunlnk,nouunlnk "${ED}" 2>/dev/null
+	fi
+
+	# If binpkg-docompress is disabled, we need to apply compression
+	# before installing.
+	if ! has binpkg-docompress ${FEATURES}; then
+		"${PORTAGE_BIN_PATH}"/ecompress --queue "${PORTAGE_DOCOMPRESS[@]}"
+		"${PORTAGE_BIN_PATH}"/ecompress --ignore "${PORTAGE_DOCOMPRESS_SKIP[@]}"
+		"${PORTAGE_BIN_PATH}"/ecompress --dequeue
+	fi
+
+	if has chflags ${FEATURES}; then
+		# Restore all the file flags that were saved earlier on.
+		mtree -U -e -p "${ED}" -k flags < "${T}/bsdflags.mtree" &> /dev/null
+	fi
+
+	>> "${PORTAGE_BUILDDIR}/.instprepped" || \
+		die "Failed to create ${PORTAGE_BUILDDIR}/.instprepped"
+}
+
 preinst_qa_check() {
 	postinst_qa_check preinst
 }
diff --git a/bin/phase-functions.sh b/bin/phase-functions.sh
index 51b480bfb..d8ebf3d3e 100644
--- a/bin/phase-functions.sh
+++ b/bin/phase-functions.sh
@@ -288,7 +288,7 @@ __dyn_clean() {
 
 	if [[ $EMERGE_FROM = binary ]] || ! has keepwork $FEATURES; then
 		rm -f "$PORTAGE_BUILDDIR"/.{ebuild_changed,logid,pretended,setuped,unpacked,prepared} \
-			"$PORTAGE_BUILDDIR"/.{configured,compiled,tested,packaged} \
+			"$PORTAGE_BUILDDIR"/.{configured,compiled,tested,packaged,instprepped} \
 			"$PORTAGE_BUILDDIR"/.die_hooks \
 			"$PORTAGE_BUILDDIR"/.ipc_{in,out,lock} \
 			"$PORTAGE_BUILDDIR"/.exit_status
diff --git a/cnf/make.globals b/cnf/make.globals
index 04a708af8..72b567e98 100644
--- a/cnf/make.globals
+++ b/cnf/make.globals
@@ -50,7 +50,7 @@ RESUMECOMMAND_SSH=${FETCHCOMMAND_SSH}
 FETCHCOMMAND_SFTP="bash -c \"x=\\\${2#sftp://} ; host=\\\${x%%/*} ; port=\\\${host##*:} ; host=\\\${host%:*} ; [[ \\\${host} = \\\${port} ]] && port= ; eval \\\"declare -a ssh_opts=(\\\${3})\\\" ; exec sftp \\\${port:+-P \\\${port}} \\\"\\\${ssh_opts[@]}\\\" \\\"\\\${host}:/\\\${x#*/}\\\" \\\"\\\$1\\\"\" sftp \"\${DISTDIR}/\${FILE}\" \"\${URI}\" \"\${PORTAGE_SSH_OPTS}\""
 
 # Default user options
-FEATURES="assume-digests binpkg-logs
+FEATURES="assume-digests binpkg-docompress binpkg-logs
           config-protect-if-modified distlocks ebuild-locks
           fixlafiles merge-sync multilib-strict news
           parallel-fetch preserve-libs protect-owned
diff --git a/lib/portage/const.py b/lib/portage/const.py
index 7f84bf0e9..a343fc040 100644
--- a/lib/portage/const.py
+++ b/lib/portage/const.py
@@ -122,6 +122,7 @@ EBUILD_PHASES            = (
 )
 SUPPORTED_FEATURES       = frozenset([
 	"assume-digests",
+	"binpkg-docompress",
 	"binpkg-logs",
 	"binpkg-multi-instance",
 	"buildpkg",
diff --git a/lib/portage/dbapi/vartree.py b/lib/portage/dbapi/vartree.py
index c16fdfe88..fd8aaeb8e 100644
--- a/lib/portage/dbapi/vartree.py
+++ b/lib/portage/dbapi/vartree.py
@@ -3719,6 +3719,7 @@ class dblink(object):
 
 		This function does the following:
 
+		calls doebuild(mydo=instprep)
 		calls get_ro_checker to retrieve a function for checking whether Portage
 		will write to a read-only filesystem, then runs it against the directory list
 		calls self._preserve_libs if FEATURES=preserve-libs
@@ -3768,6 +3769,17 @@ class dblink(object):
 				level=logging.ERROR, noiselevel=-1)
 			return 1
 
+		# run instprep internal phase
+		doebuild_environment(myebuild, "instprep",
+			settings=self.settings, db=mydbapi)
+		phase = EbuildPhase(background=False, phase="instprep",
+			scheduler=self._scheduler, settings=self.settings)
+		phase.start()
+		if phase.wait() != os.EX_OK:
+			showMessage(_("!!! instprep failed\n"),
+				level=logging.ERROR, noiselevel=-1)
+			return 1
+
 		is_binpkg = self.settings.get("EMERGE_FROM") == "binary"
 		slot = ''
 		for var_name in ('CHOST', 'SLOT'):
diff --git a/lib/portage/package/ebuild/doebuild.py b/lib/portage/package/ebuild/doebuild.py
index 9706de422..d0e96f34c 100644
--- a/lib/portage/package/ebuild/doebuild.py
+++ b/lib/portage/package/ebuild/doebuild.py
@@ -656,6 +656,7 @@ def doebuild(myebuild, mydo, _unused=DeprecationWarning, settings=None, debug=0,
 	"compile":["configure"],
 	"test":   ["compile"],
 	"install":["test"],
+	"instprep":["install"],
 	"rpm":    ["install"],
 	"package":["install"],
 	"merge"  :["install"],
@@ -674,7 +675,7 @@ def doebuild(myebuild, mydo, _unused=DeprecationWarning, settings=None, debug=0,
 	                "config", "info", "setup", "depend", "pretend",
 	                "fetch", "fetchall", "digest",
 	                "unpack", "prepare", "configure", "compile", "test",
-	                "install", "rpm", "qmerge", "merge",
+	                "install", "instprep", "rpm", "qmerge", "merge",
 	                "package", "unmerge", "manifest", "nofetch"]
 
 	if mydo not in validcommands:
@@ -1402,6 +1403,7 @@ def _spawn_actionmap(settings):
 "compile":  {"cmd":ebuild_sh, "args":{"droppriv":droppriv, "free":nosandbox, "sesandbox":sesandbox, "fakeroot":0}},
 "test":     {"cmd":ebuild_sh, "args":{"droppriv":droppriv, "free":nosandbox, "sesandbox":sesandbox, "fakeroot":0}},
 "install":  {"cmd":ebuild_sh, "args":{"droppriv":0,        "free":0,         "sesandbox":sesandbox, "fakeroot":fakeroot}},
+"instprep": {"cmd":misc_sh,   "args":{"droppriv":0,        "free":0,         "sesandbox":sesandbox, "fakeroot":fakeroot}},
 "rpm":      {"cmd":misc_sh,   "args":{"droppriv":0,        "free":0,         "sesandbox":0,         "fakeroot":fakeroot}},
 "package":  {"cmd":misc_sh,   "args":{"droppriv":0,        "free":0,         "sesandbox":0,         "fakeroot":fakeroot}},
 		}
diff --git a/man/ebuild.1 b/man/ebuild.1
index 1d48844da..5893c3df6 100644
--- a/man/ebuild.1
+++ b/man/ebuild.1
@@ -130,6 +130,11 @@ the \fIsrc_install()\fR function.  When completed, the
 will contain all the files that should either be merged to the local
 filesystem or included in a binary package.
 .TP
+.BR instprep
+Performs the additional post-install/pre-merge preparations inside
+the temporary \fIinstall directory\fR.  This is intended to be called
+\fBafter\fR building binary package but before executing \fBpreinst\fR.
+.TP
 .BR postinst
 Runs package-specific actions that need to be done after the package
 is installed into the live filesystem.  Usually helpful messages are
diff --git a/man/make.conf.5 b/man/make.conf.5
index a33929143..ec03c93ca 100644
--- a/man/make.conf.5
+++ b/man/make.conf.5
@@ -275,6 +275,12 @@ existing digest, the digest will be regenerated regardless of whether or
 not \fIassume\-digests\fR is enabled. The \fBebuild\fR(1) \fBdigest\fR command
 has a \fB\-\-force\fR option that can be used to force regeneration of digests.
 .TP
+.B binpkg\-docompress
+Perform \fBdocompress\fR (controllable file compression) before creating binary
+package.  When this option is enabled (the default), documentation files are
+already compressed inside binary packages.  When it is disabled, binary packages
+contain uncompressed documentation and Portage compresses it before installing.
+.TP
 .B binpkg\-logs
 Keep logs from successful binary package merges. This is relevant only when
 \fBPORT_LOGDIR\fR is set.
-- 
2.19.1