[gentoo-dev] [PATCH 2/2] go-module.eclass: add support for EGO_SUM

[William Hubbs]

The EGO_SUM variable replaces EGO_VENDOR for go modules.

Signed-off-by: William Hubbs <williamh@g.o>
---
 eclass/go-module.eclass | 362 +++++++++++++++++++++++++++++++++++-----
 1 file changed, 322 insertions(+), 40 deletions(-)

diff --git a/eclass/go-module.eclass b/eclass/go-module.eclass
index 80ff2902b3a..68b95bbc510 100644
--- a/eclass/go-module.eclass
+++ b/eclass/go-module.eclass
@@ -4,28 +4,29 @@
 # @ECLASS: go-module.eclass
 # @MAINTAINER:
 # William Hubbs <williamh@g.o>
+# @AUTHOR:
+# William Hubbs <williamh@g.o>
+# Robin H. Johnson <robbat2@g.o>
 # @SUPPORTED_EAPIS: 7
 # @BLURB: basic eclass for building software written as go modules
 # @DESCRIPTION:
-# This eclass provides basic settings and functions
-# needed by all software written in the go programming language that uses
-# go modules.
-#
-# You will know the software you are packaging uses modules because
-# it will have files named go.sum and go.mod in its top-level source
-# directory. If it does not have these files, use the golang-* eclasses.
+# This eclass provides basic settings and functions needed by all software
+# written in the go programming language that uses modules.
 #
-# If it has these files and a directory named vendor in its top-level
-# source directory, you only need to inherit the eclass since upstream
-# is vendoring the dependencies.
+# If the software you are packaging  has a file named go.mod in its top
+# level directory, it uses modules and  your ebuild should inherit this
+# eclass. If it does not, your ebuild should use the golang-* eclasses.
 #
-# If it does not have a vendor directory, you should use the EGO_VENDOR
-# variable and the go-module_vendor_uris function as shown in the
-# example below to handle dependencies.
+# If, besides go.mod, your software has a directory named vendor in its
+# top level directory, the only thing you need to do is inherit the
+# eclass. If there is no vendor directory, you need to also populate
+# EGO_SUM and call go-module_set_globals as discussed below.
 #
 # Since Go programs are statically linked, it is important that your ebuild's
 # LICENSE= setting includes the licenses of all statically linked
 # dependencies. So please make sure it is accurate.
+# You can use a utility like dev-util/golicense (network connectivity is
+# required) to extract this information from the compiled binary.
 #
 # @EXAMPLE:
 #
@@ -33,19 +34,21 @@
 #
 # inherit go-module
 #
-# EGO_VENDOR=(
-#	"github.com/xenolf/lego 6cac0ea7d8b28c889f709ec7fa92e92b82f490dd"
-# "golang.org/x/crypto 453249f01cfeb54c3d549ddb75ff152ca243f9d8 github.com/golang/crypto"
+# EGO_SUM=(
+#   "github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59/go.mod h1:q/89r3U2H7sSsE2t6Kca0lfwTK8JdoNGS/yzM/4iH5I="
+#   "github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59 h1:WWB576BN5zNSZc/M9d/10pqEx5VHNhaQ/yOVAkmj5Yo="
 # )
 #
+# go-module_set_globals
+#
 # SRC_URI="https://github.com/example/${PN}/archive/v${PV}.tar.gz -> ${P}.tar.gz
-# $(go-module_vendor_uris)"
+#		   ${EGO_SUM_SRC_URI}"
 #
 # @CODE
 
 case ${EAPI:-0} in
 	7) ;;
-	*) die "${ECLASS} API in EAPI ${EAPI} not yet established."
+	*) die "${ECLASS} EAPI ${EAPI} is not supported."
 esac
 
 if [[ -z ${_GO_MODULE} ]]; then
@@ -64,10 +67,12 @@ export GO111MODULE=on
 export GOCACHE="${T}/go-build"
 
 # The following go flags should be used for all builds.
-# -mod=vendor stopps downloading of dependencies from the internet.
 # -v prints the names of packages as they are compiled
 # -x prints commands as they are executed
-export GOFLAGS="-mod=vendor -v -x"
+# -mod=readonly do not update go.mod/go.sum but fail if updates are needed
+# -mod=vendor use the vendor directory instead of downloading dependencies
+export GOFLAGS="-v -x -mod=readonly"
+[[ ${#EGO_VENDOR[@]} -gt 0 ]] && GOFLAGS+=" -mod=vendor"
 
 # Do not complain about CFLAGS etc since go projects do not use them.
 QA_FLAGS_IGNORED='.*'
@@ -77,28 +82,36 @@ RESTRICT="strip"
 
 EXPORT_FUNCTIONS src_unpack pkg_postinst
 
+# @ECLASS-VARIABLE: EGO_SUM
+# @DESCRIPTION:
+# This is an array based on the go.sum content from inside the target package.
+# Each array entry must be quoted and contain a single line from go.sum.
+#
+# The format of go.sum is described upstream here:
+# https://tip.golang.org/cmd/go/#hdr-Module_authentication_using_go_sum
+#
+# h1:<hash> is the Hash1 structure used by upstream Go
+# Note that Hash1 is MORE stable than Gentoo distfile hashing, and upstream
+# warns that it's conceptually possible for the Hash1 value to remain stable
+# while the upstream zipfiles change. E.g. it does NOT capture mtime changes in
+# files within a zipfile.
+
 # @ECLASS-VARIABLE: EGO_VENDOR
 # @DESCRIPTION:
-# This variable contains a list of vendored packages.
-# The items of this array are strings that contain the
-# import path and the git commit hash for a vendored package.
-# If the import path does not start with github.com, the third argument
-# can be used to point to a github repository.
+# This variable is deprecated and should no longer be used. Please
+# convert your ebuilds to use EGO_SUM.
 
 # @FUNCTION: go-module_vendor_uris
 # @DESCRIPTION:
-# Convert the information in EGO_VENDOR to a format suitable for
-# SRC_URI.
-# A call to this function should be added to SRC_URI in your ebuild if
-# the upstream package does not include vendored dependencies.
+# This function is deprecated.
 go-module_vendor_uris() {
 	local hash import line repo x
 	for line in "${EGO_VENDOR[@]}"; do
 		read -r import hash repo x <<< "${line}"
-		if [[ -n $x ]]; then
+		if [[ -n ${x} ]]; then
 			eerror "Trailing information in EGO_VENDOR in ${P}.ebuild"
 			eerror "${line}"
-			eerror "Trailing information is: \"$x\""
+			eerror "Trailing information is: \"${x}\""
 			die "Invalid EGO_VENDOR format"
 		fi
 		: "${repo:=${import}}"
@@ -106,18 +119,224 @@ go-module_vendor_uris() {
 	done
 }
 
+# @ECLASS-VARIABLE: _GOMODULE_GOPROXY_BASEURI
+# @DESCRIPTION:
+# Golang module proxy service to fetch module files from. Note that the module
+# proxy generally verifies modules via the Hash1 code.
+#
+# Users in China may find some mirrors in the default list blocked, and should
+# explicitly set an entry in /etc/portage/mirrors for goproxy to
+# https://goproxy.cn/ or another mirror that is not blocked in China.
+# See https://arslan.io/2019/08/02/why-you-should-use-a-go-module-proxy/ for
+# further details
+#
+# This variable is NOT intended for user-level configuration of mirrors, but
+# rather to cover go modules that might exist only on specific Goproxy
+# servers for non-technical reasons.
+#
+# This variable should NOT be present in user-level configuration e.g.
+# /etc/portage/make.conf, as it will violate metadata immutability!
+#
+# I am considering removing this and just hard coding mirror://goproxy
+# below, so please do not rely on it.
+: "${_GOMODULE_GOPROXY_BASEURI:=mirror://goproxy/}"
+
+# @ECLASS-VARIABLE: _GOMODULE_GOSUM_REVERSE_MAP
+# @DESCRIPTION:
+# Mapping back from Gentoo distfile name to upstream distfile path.
+# Associative array to avoid O(N*M) performance when populating the GOPROXY
+# directory structure.
+declare -A -g _GOMODULE_GOSUM_REVERSE_MAP
+
+# @FUNCTION: go-module_set_globals
+# @DESCRIPTION:
+# Convert the information in EGO_SUM for other usage in the ebuild.
+# - Populates EGO_SUM_SRC_URI that can be added to SRC_URI
+# - Exports _GOMODULE_GOSUM_REVERSE_MAP which provides reverse mapping from
+#   distfile back to the relative part of SRC_URI, as needed for
+#   GOPROXY=file:///...
+go-module_set_globals() {
+	local line exts
+	# for tracking go.sum errors
+	local error_in_gosum=0
+	local -a gosum_errorlines
+	# used make SRC_URI easier to read
+	local newline=$'\n'
+
+	# Now parse EGO_SUM
+	for line in "${EGO_SUM[@]}"; do
+		local module version modfile version_modfile kvs x
+		read -r module version_modfile kvs <<< "${line}"
+
+		# Split 'v0.3.0/go.mod' into 'v0.3.0' and '/go.mod'
+		# It might NOT have the trailing /go.mod
+		IFS=/ read -r version modfile x <<<"${version_modfile}"
+		# Reject multiple slashes
+		if [[ -n ${x} ]]; then
+			error_in_gosum=1
+			gosum_errorlines+=( "Bad version: ${version_modfile}" )
+			continue
+		fi
+
+		# The modfile variable should be either empty or '/go.mod'
+		# There is a chance that upstream Go might add something else here in
+		# future, and we should be prepared to capture it.
+		# The .info files do not need to be downloaded, they will be created
+		# based on the .mod file.
+		# See https://github.com/golang/go/issues/35922#issuecomment-584824275
+		exts=()
+		local errormsg=''
+		case "${modfile}" in
+			'') exts=( zip ) ;;
+			'go.mod'|'/go.mod') exts=( mod ) ;;
+			*) errormsg="Unknown modfile: line='${line}', modfile='${modfile}'" ;;
+		esac
+
+		# If it was a bad entry, restart the loop
+		if [[ -n ${errormsg} ]]; then
+			error_in_gosum=1
+			gosum_errorlines+=( "${errormsg} line='${line}', modfile='${modfile}'" )
+			continue
+		fi
+
+		_dir=$(_go-module_gomod_encode "${module}")
+
+		for _ext in "${exts[@]}" ; do
+			# Relative URI within a GOPROXY for a file
+			_reluri="${_dir}/@v/${version}.${_ext}"
+			# SRC_URI: LHS entry
+			_uri="${_GOMODULE_GOPROXY_BASEURI}/${_reluri}"
+#			_uri="mirror://goproxy/${_reluri}"
+			# SRC_URI: RHS entry, encode any slash in the path as
+			# %2F in the filename
+			_distfile="${_reluri//\//%2F}"
+
+			EGO_SUM_SRC_URI+=" ${_uri} -> ${_distfile}${newline}"
+			_GOMODULE_GOSUM_REVERSE_MAP["${_distfile}"]="${_reluri}"
+		done
+	done
+
+	if [[ ${error_in_gosum} != 0 ]]; then
+		eerror "Trailing information in EGO_SUM in ${P}.ebuild"
+		for line in "${gosum_errorlines[@]}" ; do
+			eerror "${line}"
+		done
+		die "Invalid EGO_SUM format"
+	fi
+
+	# Ensure these variables are not changed past this point
+	readonly EGO_SUM
+	readonly EGO_SUM_SRC_URI
+	readonly _GOMODULE_GOSUM_REVERSE_MAP
+
+	# Set the guard that we are safe
+	_GO_MODULE_SET_GLOBALS_CALLED=1
+}
+
 # @FUNCTION: go-module_src_unpack
 # @DESCRIPTION:
+# - If EGO_VENDOR is set, use the deprecated function to unpack the base
+#   tarballs and the tarballs indicated in EGO_VENDOR to the correct
+#   locations.
+# - Otherwise, if EGO_SUM is set, unpack the base tarball(s) and set up the
+#   local go proxy.
+# - Otherwise do a normal unpack.
+go-module_src_unpack() {
+	if [[ "${#EGO_VENDOR[@]}" -gt 0 ]]; then
+		_go-module_src_unpack_vendor
+	elif [[ "${#EGO_SUM[@]}" -gt 0 ]]; then
+		_go-module_src_unpack_gosum
+	else
+		default
+	fi
+}
+
+# @FUNCTION: _go-module_src_unpack_gosum
+# @DESCRIPTION:
+# Populate a GOPROXY directory hierarchy with distfiles from EGO_SUM and
+# unpack the base distfiles.
+#
+# Exports GOPROXY environment variable so that Go calls will source the
+# directory correctly.
+_go-module_src_unpack_gosum() {
+	# shellcheck disable=SC2120
+	debug-print-function "${FUNCNAME}" "$@"
+
+	if [[ ! ${_GO_MODULE_SET_GLOBALS_CALLED} ]]; then
+		die "go-module_set_globals must be called in global scope"
+	fi
+
+	local goproxy_dir="${T}/go-proxy"
+	mkdir -p "${goproxy_dir}" || die
+
+	# For each Golang module distfile, look up where it's supposed to go, and
+	# symlink into place.
+	local f
+	local goproxy_mod_dir
+	for f in ${A}; do
+		goproxy_mod_path="${_GOMODULE_GOSUM_REVERSE_MAP["${f}"]}"
+		if [[ -n "${goproxy_mod_path}" ]]; then
+			debug-print-function "Populating go proxy for ${goproxy_mod_path}"
+			# Build symlink hierarchy
+			goproxy_mod_dir=$( dirname "${goproxy_dir}"/"${goproxy_mod_path}" )
+			mkdir -p "${goproxy_mod_dir}" || die
+			ln -sf "${DISTDIR}"/"${f}" "${goproxy_dir}/${goproxy_mod_path}" ||
+				die "Failed to ln"
+			local v=${goproxy_mod_path}
+			v="${v%.mod}"
+			v="${v%.zip}"
+			v="${v//*\/}"
+			_go-module_gosum_synthesize_files "${goproxy_mod_dir}" "${v}"
+		else
+			unpack "$f"
+		fi
+	done
+	export GOPROXY="file://${goproxy_dir}"
+
+	# Validate the gosum now
+	_go-module_src_unpack_verify_gosum
+}
+
+# @FUNCTION: _go-module_gosum_synthesize_files
+# @DESCRIPTION:
+# Given a path &  version, populate all Goproxy metadata files which aren't
+# needed to be downloaded directly.
+# - .../@v/${version}.info
+# - .../@v/list
+_go-module_gosum_synthesize_files() {
+	local target=$1
+	local version=$2
+	# 'go get' doesn't care about the hash of the .info files, they
+	# just need a 'version' element!
+	# This saves a download of a tiny file
+	# The .time key is omitted, because that is the time a module was added
+	# to the upstream goproxy, and not metadata about the module itself.
+	cat >"${target}/${version}.info" <<-EOF
+	{
+		"Version": "${version}",
+		"shortName": "${version}",
+		"Name": "${version}"
+	}
+	EOF
+	listfile="${target}"/list
+	if ! grep -sq -x -e "${version}" "${listfile}" 2>/dev/null; then
+		echo "${version}" >>"${listfile}"
+	fi
+}
+
+# @FUNCTION: _go-module_src_unpack_vendor
+# @DESCRIPTION:
 # Extract all archives in ${a} which are not nentioned in ${EGO_VENDOR}
 # to their usual locations then extract all archives mentioned in
 # ${EGO_VENDOR} to ${S}/vendor.
-go-module_src_unpack() {
-	debug-print-function ${FUNCNAME} "$@"
+_go-module_src_unpack_vendor() {
+	# shellcheck disable=SC2120
+	debug-print-function "${FUNCNAME}" "$@"
 	local f hash import line repo tarball vendor_tarballs x
 	vendor_tarballs=()
 	for line in "${EGO_VENDOR[@]}"; do
 		read -r import hash repo x <<< "${line}"
-		if [[ -n $x ]]; then
+		if [[ -n ${x} ]]; then
 			eerror "Trailing information in EGO_VENDOR in ${P}.ebuild"
 			eerror "${line}"
 			die "Invalid EGO_VENDOR format"
@@ -125,10 +344,10 @@ go-module_src_unpack() {
 		: "${repo:=${import}}"
 		vendor_tarballs+=("${repo//\//-}-${hash}.tar.gz")
 	done
-	for f in $A; do
-		[[ -n ${vendor_tarballs[*]} ]] && has "$f" "${vendor_tarballs[@]}" &&
+	for f in ${A}; do
+		[[ -n ${vendor_tarballs[*]} ]] && has "${f}" "${vendor_tarballs[@]}" &&
 			continue
-		unpack "$f"
+		unpack "${f}"
 	done
 
 	[[ -z ${vendor_tarballs[*]} ]] && return
@@ -143,6 +362,42 @@ go-module_src_unpack() {
 			-f "${DISTDIR}/${tarball}" || die
 		eend
 	done
+	eqawarn "${P}.ebuild uses EGO_VENDOR. Please migrate to EGO_SUM."
+}
+
+# @FUNCTION: _go-module_src_unpack_verify_gosum
+# @DESCRIPTION:
+# Validate the Go modules declared by EGO_SUM are sufficient to cover building
+# the package, without actually building it yet.
+_go-module_src_unpack_verify_gosum() {
+	# shellcheck disable=SC2120
+	debug-print-function "${FUNCNAME}" "$@"
+
+	if [[ ! ${_GO_MODULE_SET_GLOBALS_CALLED} ]]; then
+		die "go-module_set_globals must be called in global scope"
+	fi
+
+	cd "${S}"
+
+	# Cleanup the modules before starting anything else
+	# This will print 'downloading' messages, but it's accessing content from
+	# the $GOPROXY file:/// URL!
+	einfo "Tidying go.mod/go.sum"
+	go mod tidy >/dev/null
+
+	# Verify that all needed modules are really present, by fetching everything
+	# in the package's main go.mod.  If the EGO_SUM was missing an entry then
+	# 'go mod tidy' && 'go get' will flag it.
+	# -v = verbose
+	# -d = download only, don't install
+	# -mod readonly = treat modules as readonly source
+	einfo "Verifying linked Golang modules"
+	go get \
+		-v \
+		-d \
+		-mod readonly \
+		all \
+		|| die "Some module is missing, update EGO_SUM"
 }
 
 # @FUNCTION: go-module_live_vendor
@@ -150,13 +405,14 @@ go-module_src_unpack() {
 # This function is used in live ebuilds to vendor the dependencies when
 # upstream doesn't vendor them.
 go-module_live_vendor() {
-	debug-print-function ${FUNCNAME} "$@"
+	debug-print-function "${FUNCNAME}" "$@"
 
+	# shellcheck disable=SC2086
 	has live ${PROPERTIES} ||
 		die "${FUNCNAME} only allowed in live ebuilds"
 	[[ "${EBUILD_PHASE}" == unpack ]] ||
 		die "${FUNCNAME} only allowed in src_unpack"
-	[[ -d "${S}"/vendor ]] ||
+	[[ -d "${S}"/vendor ]] &&
 		die "${FUNCNAME} only allowed when upstream isn't vendoring"
 
 	pushd "${S}" >& /dev/null || die
@@ -168,7 +424,7 @@ go-module_live_vendor() {
 # @DESCRIPTION:
 # Display a warning about security updates for Go programs.
 go-module_pkg_postinst() {
-	debug-print-function ${FUNCNAME} "$@"
+	debug-print-function "${FUNCNAME}" "$@"
 	[[ -n ${REPLACING_VERSIONS} ]] && return 0
 	ewarn "${PN} is written in the Go programming language."
 	ewarn "Since this language is statically linked, security"
@@ -179,4 +435,30 @@ go-module_pkg_postinst() {
 	ewarn "stable tree."
 }
 
+# @FUNCTION: _go-module_gomod_encode
+# @DESCRIPTION:
+# Encode the name(path) of a Golang module in the format expected by Goproxy.
+#
+# Upper letters are replaced by their lowercase version with a '!' prefix.
+#
+_go-module_gomod_encode() {
+	## Python:
+	# return re.sub('([A-Z]{1})', r'!\1', s).lower()
+
+	## Sed:
+	## This uses GNU Sed extension \l to downcase the match
+	#echo "${module}" |sed 's,[A-Z],!\l&,g'
+	#
+	# Bash variant:
+	debug-print-function "${FUNCNAME}" "$@"
+	#local re input lower
+	re='(.*)([A-Z])(.*)'
+	input="${1}"
+	while [[ ${input} =~ ${re} ]]; do
+		lower='!'"${BASH_REMATCH[2],}"
+		input="${BASH_REMATCH[1]}${lower}${BASH_REMATCH[3]}"
+	done
+	echo "${input}"
+}
+
 fi
-- 
2.24.1