[gentoo-commits] repo/gentoo:master commit in: eclass/, eclass/tests/

[Michał Górny]

commit:     85820b0b54f298e1d96a4bc3ed9cd4952a46aea0
Author:     Michał Górny <mgorny <AT> gentoo <DOT> org>
AuthorDate: Thu Mar 31 21:09:30 2022 +0000
Commit:     Michał Górny <mgorny <AT> gentoo <DOT> org>
CommitDate: Sat Apr  2 16:29:46 2022 +0000
URL:        https://gitweb.gentoo.org/repo/gentoo.git/commit/?id=85820b0b

python-utils-r1.eclass: Make python_fix_shebang force full path

Change the behavior of python_fix_shebang to always output full path
to the Python interpreter (i.e. ${PYTHON}) instead of mangling
the original path.  Most importantly, this ensures that:

1. EPREFIX is included in the final path

2. /usr/bin/env is replaced by the absolute path to avoid issues
   when calling system executables from inside a venv

Note that this implies that a few unlikely corner cases may stop
working, notably:

a. "weird" shebangs such as "/usr/bin/foo python" will no longer work

b. the mangled scripts will escape temporary venv e.g. created
   in distutils-r1 PEP517 mode (python_fix_shebang is not used in such
   a way in ::gentoo)

Signed-off-by: Michał Górny <mgorny <AT> gentoo.org>

 eclass/python-utils-r1.eclass   | 134 +++++++++++++++-------------------------
 eclass/tests/python-utils-r1.sh |  86 ++++++++++++++++----------
 2 files changed, 106 insertions(+), 114 deletions(-)

diff --git a/eclass/python-utils-r1.eclass b/eclass/python-utils-r1.eclass
index 7b0c9aa280d8..98cb49c95fd7 100644
--- a/eclass/python-utils-r1.eclass
+++ b/eclass/python-utils-r1.eclass
@@ -1001,25 +1001,30 @@ _python_wrapper_setup() {
 # @FUNCTION: python_fix_shebang
 # @USAGE: [-f|--force] [-q|--quiet] <path>...
 # @DESCRIPTION:
-# Replace the shebang in Python scripts with the current Python
-# implementation (EPYTHON). If a directory is passed, works recursively
-# on all Python scripts.
+# Replace the shebang in Python scripts with the full path
+# to the current Python implementation (PYTHON, including EPREFIX).
+# If a directory is passed, works recursively on all Python scripts
+# found inside the directory tree.
 #
-# Only files having a 'python*' shebang will be modified. Files with
-# other shebang will either be skipped when working recursively
-# on a directory or treated as error when specified explicitly.
+# Only files having a Python shebang (a path to any known Python
+# interpreter, optionally preceded by env(1) invocation) will
+# be processed.  Files with any other shebang will either be skipped
+# silently when a directory was passed, or an error will be reported
+# for any files without Python shebangs specified explicitly.
 #
-# Shebangs matching explicitly current Python version will be left
-# unmodified. Shebangs requesting another Python version will be treated
-# as fatal error, unless --force is given.
+# Shebangs that are compatible with the current Python version will be
+# mangled unconditionally.  Incompatible shebangs will cause a fatal
+# error, unless --force is specified.
 #
-# --force causes the function to replace even shebangs that require
-# incompatible Python version. --quiet causes the function not to list
-# modified files verbosely.
+# --force causes the function to replace shebangs with incompatible
+# Python version (but not non-Python shebangs).  --quiet causes
+# the function not to list modified files verbosely.
 python_fix_shebang() {
 	debug-print-function ${FUNCNAME} "${@}"
 
 	[[ ${EPYTHON} ]] || die "${FUNCNAME}: EPYTHON unset (pkg_setup not called?)"
+	local PYTHON
+	_python_export "${EPYTHON}" PYTHON
 
 	local force quiet
 	while [[ ${@} ]]; do
@@ -1035,13 +1040,13 @@ python_fix_shebang() {
 
 	local path f
 	for path; do
-		local any_correct any_fixed is_recursive
+		local any_fixed is_recursive
 
 		[[ -d ${path} ]] && is_recursive=1
 
 		while IFS= read -r -d '' f; do
 			local shebang i
-			local error= from=
+			local error= match=
 
 			# note: we can't ||die here since read will fail if file
 			# has no newline characters
@@ -1050,64 +1055,36 @@ python_fix_shebang() {
 			# First, check if it's shebang at all...
 			if [[ ${shebang} == '#!'* ]]; then
 				local split_shebang=()
-				read -r -a split_shebang <<<${shebang} || die
-
-				# Match left-to-right in a loop, to avoid matching random
-				# repetitions like 'python2.7 python2'.
-				for i in "${split_shebang[@]}"; do
-					case "${i}" in
-						*"${EPYTHON}")
-							debug-print "${FUNCNAME}: in file ${f#${D%/}}"
-							debug-print "${FUNCNAME}: shebang matches EPYTHON: ${shebang}"
-
-							# Nothing to do, move along.
-							any_correct=1
-							from=${EPYTHON}
-							break
-							;;
-						*python|*python[23])
-							debug-print "${FUNCNAME}: in file ${f#${D%/}}"
-							debug-print "${FUNCNAME}: rewriting shebang: ${shebang}"
-
-							if [[ ${i} == *python2 ]]; then
-								from=python2
-								if [[ ! ${force} ]]; then
-									error=1
-								fi
-							elif [[ ${i} == *python3 ]]; then
-								from=python3
-							else
-								from=python
-							fi
-							break
-							;;
-						*python[23].[0-9]|*python3.[1-9][0-9]|*pypy|*pypy3|*jython[23].[0-9])
-							# Explicit mismatch.
-							if [[ ! ${force} ]]; then
-								error=1
-							else
-								case "${i}" in
-									*python[23].[0-9])
-										from="python[23].[0-9]";;
-									*python3.[1-9][0-9])
-										from="python3.[1-9][0-9]";;
-									*pypy)
-										from="pypy";;
-									*pypy3)
-										from="pypy3";;
-									*jython[23].[0-9])
-										from="jython[23].[0-9]";;
-									*)
-										die "${FUNCNAME}: internal error in 2nd pattern match";;
-								esac
-							fi
-							break
-							;;
-					esac
-				done
+				read -r -a split_shebang <<<${shebang#"#!"} || die
+
+				local in_path=${split_shebang[0]}
+				local from='^#! *[^ ]*'
+				# if the first component is env(1), skip it
+				if [[ ${in_path} == */env ]]; then
+					in_path=${split_shebang[1]}
+					from+=' *[^ ]*'
+				fi
+
+				case ${in_path##*/} in
+					"${EPYTHON}")
+						match=1
+						;;
+					python|python[23])
+						match=1
+						[[ ${in_path##*/} == python2 ]] && error=1
+						;;
+					python[23].[0-9]|python3.[1-9][0-9]|pypy|pypy3|jython[23].[0-9])
+						# Explicit mismatch.
+						match=1
+						error=1
+						;;
+				esac
 			fi
 
-			if [[ ! ${error} && ! ${from} ]]; then
+			# disregard mismatches in force mode
+			[[ ${force} ]] && error=
+
+			if [[ ! ${match} ]]; then
 				# Non-Python shebang. Allowed in recursive mode,
 				# disallowed when specifying file explicitly.
 				[[ ${is_recursive} ]] && continue
@@ -1119,13 +1096,9 @@ python_fix_shebang() {
 			fi
 
 			if [[ ! ${error} ]]; then
-				# We either want to match ${from} followed by space
-				# or at end-of-string.
-				if [[ ${shebang} == *${from}" "* ]]; then
-					sed -i -e "1s:${from} :${EPYTHON} :" "${f}" || die
-				else
-					sed -i -e "1s:${from}$:${EPYTHON}:" "${f}" || die
-				fi
+				debug-print "${FUNCNAME}: in file ${f#${D%/}}"
+				debug-print "${FUNCNAME}: rewriting shebang: ${shebang}"
+				sed -i -e "1s@${from}@#!${PYTHON}@" "${f}" || die
 				any_fixed=1
 			else
 				eerror "The file has incompatible shebang:"
@@ -1138,12 +1111,7 @@ python_fix_shebang() {
 
 		if [[ ! ${any_fixed} ]]; then
 			eerror "QA error: ${FUNCNAME}, ${path#${D%/}} did not match any fixable files."
-			if [[ ${any_correct} ]]; then
-				eerror "All files have ${EPYTHON} shebang already."
-			else
-				eerror "There are no Python files in specified directory."
-			fi
-
+			eerror "There are no Python files in specified directory."
 			die "${FUNCNAME} did not match any fixable files"
 		fi
 	done

diff --git a/eclass/tests/python-utils-r1.sh b/eclass/tests/python-utils-r1.sh
index 8c733b22294e..ef7687b8a9cf 100755
--- a/eclass/tests/python-utils-r1.sh
+++ b/eclass/tests/python-utils-r1.sh
@@ -41,7 +41,7 @@ test_fix_shebang() {
 	local expect=${3}
 	local args=( "${@:4}" )
 
-	tbegin "python_fix_shebang${args[@]+ ${args[*]}} from ${from} to ${to} (exp: ${expect})"
+	tbegin "python_fix_shebang${args[@]+ ${args[*]}} from ${from@Q} to ${to@Q} (exp: ${expect@Q})"
 
 	echo "${from}" > "${tmpfile}"
 	output=$( EPYTHON=${to} python_fix_shebang "${args[@]}" -q "${tmpfile}" 2>&1 )
@@ -156,36 +156,60 @@ fi
 test_var PYTHON_PKG_DEP pypy3 '*dev-python/pypy3*:0='
 test_var PYTHON_SCRIPTDIR pypy3 /usr/lib/python-exec/pypy3
 
-# generic shebangs
-test_fix_shebang '#!/usr/bin/python' python3.6 '#!/usr/bin/python3.6'
-test_fix_shebang '#!/usr/bin/python' pypy3 '#!/usr/bin/pypy3'
-
-# python2/python3 matching
-test_fix_shebang '#!/usr/bin/python3' python3.6 '#!/usr/bin/python3.6'
-test_fix_shebang '#!/usr/bin/python2' python3.6 FAIL
-test_fix_shebang '#!/usr/bin/python2' python3.6 '#!/usr/bin/python3.6' --force
-
-# pythonX.Y matching (those mostly test the patterns)
-test_fix_shebang '#!/usr/bin/python2.7' python3.2 FAIL
-test_fix_shebang '#!/usr/bin/python2.7' python3.2 '#!/usr/bin/python3.2' --force
-test_fix_shebang '#!/usr/bin/python3.2' python3.2 '#!/usr/bin/python3.2'
-
-# fancy path handling
-test_fix_shebang '#!/mnt/python2/usr/bin/python' python3.6 \
-	'#!/mnt/python2/usr/bin/python3.6'
-test_fix_shebang '#!/mnt/python2/usr/bin/python3' python3.8 \
-	'#!/mnt/python2/usr/bin/python3.8'
-test_fix_shebang '#!/mnt/python2/usr/bin/env python' python3.8 \
-	'#!/mnt/python2/usr/bin/env python3.8'
-test_fix_shebang '#!/mnt/python2/usr/bin/python3 python3' python3.8 \
-	'#!/mnt/python2/usr/bin/python3.8 python3'
-test_fix_shebang '#!/mnt/python2/usr/bin/python2 python3' python3.8 FAIL
-test_fix_shebang '#!/mnt/python2/usr/bin/python2 python3' python3.8 \
-	'#!/mnt/python2/usr/bin/python3.8 python3' --force
-test_fix_shebang '#!/usr/bin/foo' python3.8 FAIL
-
-# regression test for bug #522080
-test_fix_shebang '#!/usr/bin/python ' python3.8 '#!/usr/bin/python3.8 '
+for EPREFIX in '' /foo; do
+	einfo "with EPREFIX=${EPREFIX@Q}"
+	eindent
+	# generic shebangs
+	test_fix_shebang '#!/usr/bin/python' python3.6 \
+		"#!${EPREFIX}/usr/bin/python3.6"
+	test_fix_shebang '#!/usr/bin/python' pypy3 \
+		"#!${EPREFIX}/usr/bin/pypy3"
+
+	# python2/python3 matching
+	test_fix_shebang '#!/usr/bin/python3' python3.6 \
+		"#!${EPREFIX}/usr/bin/python3.6"
+	test_fix_shebang '#!/usr/bin/python2' python3.6 FAIL
+	test_fix_shebang '#!/usr/bin/python2' python3.6 \
+		"#!${EPREFIX}/usr/bin/python3.6" --force
+
+	# pythonX.Y matching (those mostly test the patterns)
+	test_fix_shebang '#!/usr/bin/python2.7' python3.2 FAIL
+	test_fix_shebang '#!/usr/bin/python2.7' python3.2 \
+		"#!${EPREFIX}/usr/bin/python3.2" --force
+	test_fix_shebang '#!/usr/bin/python3.2' python3.2 \
+		"#!${EPREFIX}/usr/bin/python3.2"
+
+	# fancy path handling
+	test_fix_shebang '#!/mnt/python2/usr/bin/python' python3.6 \
+		"#!${EPREFIX}/usr/bin/python3.6"
+	test_fix_shebang '#!/mnt/python2/usr/bin/python3' python3.8 \
+		"#!${EPREFIX}/usr/bin/python3.8"
+	test_fix_shebang '#!/mnt/python2/usr/bin/env python' python3.8 \
+		"#!${EPREFIX}/usr/bin/python3.8"
+	test_fix_shebang '#!/mnt/python2/usr/bin/python3 python3' python3.8 \
+		"#!${EPREFIX}/usr/bin/python3.8 python3"
+	test_fix_shebang '#!/mnt/python2/usr/bin/python2 python3' python3.8 FAIL
+	test_fix_shebang '#!/mnt/python2/usr/bin/python2 python3' python3.8 \
+		"#!${EPREFIX}/usr/bin/python3.8 python3" --force
+	test_fix_shebang '#!/usr/bin/foo' python3.8 FAIL
+
+	# regression test for bug #522080
+	test_fix_shebang '#!/usr/bin/python ' python3.8 \
+		"#!${EPREFIX}/usr/bin/python3.8 "
+
+	# test random whitespace in shebang
+	test_fix_shebang '#! /usr/bin/python' python3.8 \
+		"#!${EPREFIX}/usr/bin/python3.8"
+	test_fix_shebang '#!  /usr/bin/python' python3.8 \
+		"#!${EPREFIX}/usr/bin/python3.8"
+	test_fix_shebang '#! /usr/bin/env   python' python3.8 \
+		"#!${EPREFIX}/usr/bin/python3.8"
+
+	# test preserving options
+	test_fix_shebang '#! /usr/bin/python -b' python3.8 \
+		"#!${EPREFIX}/usr/bin/python3.8 -b"
+	eoutdent
+done
 
 # check _python_impl_matches behavior
 test_is "_python_impl_matches python3_6 -3" 0