Gentoo Archives: gentoo-dev

From: William Hubbs <williamh@g.o>
To: gentoo-dev@l.g.o
Cc: William Hubbs <williamh@g.o>
Subject: [gentoo-dev] [POC PATCH] go-module.eclass: remove deprecated code
Date: Thu, 10 Mar 2022 21:30:56
Message-Id: 20220310213038.9333-1-williamh@gentoo.org
1 This is not being merged. I am posting it now to show how I want
2 the eclass to change after the usage of EGO_SUM is removed from the tree.
3 I will re-post before I merge.
4
5 ---
6 eclass/go-module.eclass | 359 +---------------------------------------
7 1 file changed, 5 insertions(+), 354 deletions(-)
8
9 diff --git a/eclass/go-module.eclass b/eclass/go-module.eclass
10 index 817f8a9fd3e..557f3fb9654 100644
11 --- a/eclass/go-module.eclass
12 +++ b/eclass/go-module.eclass
13 @@ -104,85 +104,6 @@ QA_FLAGS_IGNORED='.*'
14 # Go packages should not be stripped with strip(1).
15 RESTRICT+=" strip"
16
17 -# @ECLASS-VARIABLE: EGO_SUM
18 -# @DEPRECATED:
19 -# @DESCRIPTION:
20 -# This is replaced by a dependency tarball, see above for how to create
21 -# one.
22 -#
23 -# This array is based on the contents of the go.sum file from the top
24 -# level directory of the software you are packaging. Each entry must be
25 -# quoted and contain the first two fields of a line from go.sum.
26 -#
27 -# You can use some combination of sed/awk/cut to extract the
28 -# contents of EGO_SUM or use the dev-go/get-ego-vendor tool.
29 -#
30 -# One manual way to do this is the following:
31 -#
32 -# @CODE
33 -#
34 -# cat go.sum | cut -d" " -f1,2 | awk '{print "\t\"" $0 "\""}'
35 -#
36 -# @CODE
37 -#
38 -# The format of go.sum is described upstream here:
39 -# https://go.dev/ref/mod#go-sum-files
40 -#
41 -# For inclusion in EGO_SUM, the h1: value and other future extensions SHOULD be
42 -# omitted at this time. The EGO_SUM parser will accept them for ease of ebuild
43 -# creation.
44 -#
45 -# h1:<hash> is the Hash1 structure used by upstream Go
46 -# The Hash1 is MORE stable than Gentoo distfile hashing, and upstream warns
47 -# that it's conceptually possible for the Hash1 value to remain stable while
48 -# the upstream zipfiles change. Here are examples that do NOT change the h1:
49 -# hash, but do change a regular checksum over all bytes of the file:
50 -# - Differing mtimes within zipfile
51 -# - Differing filename ordering with the zipfile
52 -# - Differing zipfile compression parameters
53 -# - Differing zipfile extra fields
54 -#
55 -# For Gentoo usage, the authors of this eclass feel that the h1: hash should
56 -# NOT be included in the EGO_SUM at this time in order to reduce size of the
57 -# ebuilds. This position will be reconsidered in future when a Go module
58 -# distfile collision comes to light, where the Hash1 value of two distfiles is
59 -# the same, but checksums over the file as a byte stream consider the files to
60 -# be different.
61 -#
62 -# This decision does NOT weaken Go module security, as Go will verify the
63 -# go.sum copy of the Hash1 values during building of the package.
64 -
65 -# @ECLASS-VARIABLE: _GOMODULE_GOPROXY_BASEURI
66 -# @DEPRECATED:
67 -# @DESCRIPTION:
68 -# Golang module proxy service to fetch module files from. Note that the module
69 -# proxy generally verifies modules via the Hash1 code.
70 -#
71 -# Users in China may find some mirrors in the default list blocked, and should
72 -# explicitly set an entry in /etc/portage/mirrors for goproxy to
73 -# https://goproxy.cn/ or another mirror that is not blocked in China.
74 -# See https://arslan.io/2019/08/02/why-you-should-use-a-go-module-proxy/ for
75 -# further details
76 -#
77 -# This variable is NOT intended for user-level configuration of mirrors, but
78 -# rather to cover go modules that might exist only on specific Goproxy
79 -# servers for non-technical reasons.
80 -#
81 -# This variable should NOT be present in user-level configuration e.g.
82 -# /etc/portage/make.conf, as it will violate metadata immutability!
83 -#
84 -# I am considering removing this and just hard coding mirror://goproxy
85 -# below, so please do not rely on it.
86 -: "${_GOMODULE_GOPROXY_BASEURI:=mirror://goproxy/}"
87 -
88 -# @ECLASS-VARIABLE: _GOMODULE_GOSUM_REVERSE_MAP
89 -# @DEPRECATED:
90 -# @DESCRIPTION:
91 -# Mapping back from Gentoo distfile name to upstream distfile path.
92 -# Associative array to avoid O(N*M) performance when populating the GOPROXY
93 -# directory structure.
94 -declare -A -g _GOMODULE_GOSUM_REVERSE_MAP
95 -
96 # @ECLASS-VARIABLE: GO_OPTIONAL
97 # @DEFAULT_UNSET
98 # @PRE_INHERIT
99 @@ -205,260 +126,17 @@ ego() {
100 "$@" || die -n "${*} failed"
101 }
102
103 -# @FUNCTION: go-module_set_globals
104 -# @DEPRECATED:
105 -# @DESCRIPTION:
106 -# Convert the information in EGO_SUM for other usage in the ebuild.
107 -# - Populates EGO_SUM_SRC_URI that can be added to SRC_URI
108 -# - Exports _GOMODULE_GOSUM_REVERSE_MAP which provides reverse mapping from
109 -# distfile back to the relative part of SRC_URI, as needed for
110 -# GOPROXY=file:///...
111 -go-module_set_globals() {
112 - local line exts
113 - # for tracking go.sum errors
114 - local error_in_gosum=0
115 - local -a gosum_errorlines
116 - # used make SRC_URI easier to read
117 - local newline=$'\n'
118 -
119 - # Now parse EGO_SUM
120 - for line in "${EGO_SUM[@]}"; do
121 - local module version modfile version_modfile kvs x
122 - read -r module version_modfile kvs <<< "${line}"
123 - # kvs contains the hash and may contain other data from
124 - # upstream in the future. We do not currently use any of this data.
125 -
126 - # Split 'v0.3.0/go.mod' into 'v0.3.0' and '/go.mod'
127 - # It might NOT have the trailing /go.mod
128 - IFS=/ read -r version modfile x <<<"${version_modfile}"
129 - # Reject multiple slashes
130 - if [[ -n ${x} ]]; then
131 - error_in_gosum=1
132 - gosum_errorlines+=( "Bad version: ${version_modfile}" )
133 - continue
134 - fi
135 -
136 - # The modfile variable should be either empty or '/go.mod'
137 - # There is a chance that upstream Go might add something else here in
138 - # the future, and we should be prepared to capture it.
139 - # The .info files do not need to be downloaded, they will be created
140 - # based on the .mod file.
141 - # See https://github.com/golang/go/issues/35922#issuecomment-584824275
142 - exts=()
143 - local errormsg=''
144 - case "${modfile}" in
145 - '') exts=( zip ) ;;
146 - 'go.mod'|'/go.mod') exts=( mod ) ;;
147 - *) errormsg="Unknown modfile: line='${line}', modfile='${modfile}'" ;;
148 - esac
149 -
150 - # If it was a bad entry, restart the loop
151 - if [[ -n ${errormsg} ]]; then
152 - error_in_gosum=1
153 - gosum_errorlines+=( "${errormsg} line='${line}', modfile='${modfile}'" )
154 - continue
155 - fi
156 -
157 - _dir=$(_go-module_gomod_encode "${module}")
158 -
159 - for _ext in "${exts[@]}" ; do
160 - # Relative URI within a GOPROXY for a file
161 - _reluri="${_dir}/@v/${version}.${_ext}"
162 - # SRC_URI: LHS entry
163 - _uri="${_GOMODULE_GOPROXY_BASEURI}/${_reluri}"
164 -# _uri="mirror://goproxy/${_reluri}"
165 - # SRC_URI: RHS entry, encode any slash in the path as
166 - # %2F in the filename
167 - _distfile="${_reluri//\//%2F}"
168 -
169 - EGO_SUM_SRC_URI+=" ${_uri} -> ${_distfile}${newline}"
170 - _GOMODULE_GOSUM_REVERSE_MAP["${_distfile}"]="${_reluri}"
171 - done
172 - done
173 -
174 - if [[ ${error_in_gosum} != 0 ]]; then
175 - eerror "Trailing information in EGO_SUM in ${P}.ebuild"
176 - for line in "${gosum_errorlines[@]}" ; do
177 - eerror "${line}"
178 - done
179 - die "Invalid EGO_SUM format"
180 - fi
181 -
182 - # Ensure these variables are not changed past this point
183 - readonly EGO_SUM
184 - readonly EGO_SUM_SRC_URI
185 - readonly _GOMODULE_GOSUM_REVERSE_MAP
186 -
187 - # export the GOPROXY setting
188 - export GOPROXY="file://${T}/go-proxy"
189 -
190 - # Set the guard that we are safe
191 - _GO_MODULE_SET_GLOBALS_CALLED=1
192 -}
193 -
194 -# @FUNCTION: go-module_setup_proxy
195 -# @DEPRECATED:
196 -# @DESCRIPTION:
197 -# If your ebuild redefines src_unpack and uses EGO_SUM you need to call
198 -# this function in src_unpack.
199 -# It sets up the go module proxy in the appropriate location.
200 -go-module_setup_proxy() {
201 - # shellcheck disable=SC2120
202 - debug-print-function "${FUNCNAME}" "$@"
203 -
204 - if [[ ! ${_GO_MODULE_SET_GLOBALS_CALLED} ]]; then
205 - die "go-module_set_globals must be called in global scope"
206 - fi
207 -
208 - local goproxy_dir="${GOPROXY/file:\/\//}"
209 - mkdir -p "${goproxy_dir}" || die
210 -
211 - # For each Golang module distfile, look up where it's supposed to go and
212 - # symlink it into place.
213 - local f
214 - local goproxy_mod_dir
215 - for f in ${A}; do
216 - goproxy_mod_path="${_GOMODULE_GOSUM_REVERSE_MAP["${f}"]}"
217 - if [[ -n "${goproxy_mod_path}" ]]; then
218 - debug-print-function "Populating go proxy for ${goproxy_mod_path}"
219 - # Build symlink hierarchy
220 - goproxy_mod_dir=$( dirname "${goproxy_dir}"/"${goproxy_mod_path}" )
221 - mkdir -p "${goproxy_mod_dir}" || die
222 - ln -sf "${DISTDIR}"/"${f}" "${goproxy_dir}/${goproxy_mod_path}" ||
223 - die "Failed to ln"
224 - local v=${goproxy_mod_path}
225 - v="${v%.mod}"
226 - v="${v%.zip}"
227 - v="${v//*\/}"
228 - _go-module_gosum_synthesize_files "${goproxy_mod_dir}" "${v}"
229 - fi
230 - done
231 -
232 - # Validate the gosum now
233 - _go-module_src_unpack_verify_gosum
234 -}
235 -
236 # @FUNCTION: go-module_src_unpack
237 # @DESCRIPTION:
238 -# If EGO_SUM is set, unpack the base tarball(s) and set up the
239 -# local go proxy. Also warn that this usage is deprecated.
240 -# - Otherwise, if EGO_VENDOR is set, bail out.
241 +# if EGO_SUM or EGO_VENDOR is set, bail out.
242 # - Otherwise do a normal unpack.
243 go-module_src_unpack() {
244 - if [[ "${#EGO_SUM[@]}" -gt 0 ]]; then
245 - eqawarn "This ebuild uses EGO_SUM which is deprecated"
246 - eqawarn "Please migrate to a dependency tarball"
247 - eqawarn "This will become a fatal error in the future"
248 - _go-module_src_unpack_gosum
249 - elif [[ "${#EGO_VENDOR[@]}" -gt 0 ]]; then
250 - eerror "${EBUILD} is using EGO_VENDOR which is no longer supported"
251 + if [[ "${#EGO_SUM[@]}" -gt 0 || "${#EGO_VENDOR[@]}" -gt 0 ]]; then
252 + eerror "${EBUILD} is using EGO_SUM or EGO_VENDOR"
253 + eeror "These are no longer supported"
254 die "Please update this ebuild"
255 - else
256 - default
257 - fi
258 -}
259 -
260 -# @FUNCTION: _go-module_src_unpack_gosum
261 -# @DEPRECATED:
262 -# @DESCRIPTION:
263 -# Populate a GOPROXY directory hierarchy with distfiles from EGO_SUM and
264 -# unpack the base distfiles.
265 -#
266 -# Exports GOPROXY environment variable so that Go calls will source the
267 -# directory correctly.
268 -_go-module_src_unpack_gosum() {
269 - # shellcheck disable=SC2120
270 - debug-print-function "${FUNCNAME}" "$@"
271 -
272 - if [[ ! ${_GO_MODULE_SET_GLOBALS_CALLED} ]]; then
273 - die "go-module_set_globals must be called in global scope"
274 - fi
275 -
276 - local goproxy_dir="${GOPROXY/file:\/\//}"
277 - mkdir -p "${goproxy_dir}" || die
278 -
279 - # For each Golang module distfile, look up where it's supposed to go, and
280 - # symlink into place.
281 - local f
282 - local goproxy_mod_dir
283 - for f in ${A}; do
284 - goproxy_mod_path="${_GOMODULE_GOSUM_REVERSE_MAP["${f}"]}"
285 - if [[ -n "${goproxy_mod_path}" ]]; then
286 - debug-print-function "Populating go proxy for ${goproxy_mod_path}"
287 - # Build symlink hierarchy
288 - goproxy_mod_dir=$( dirname "${goproxy_dir}"/"${goproxy_mod_path}" )
289 - mkdir -p "${goproxy_mod_dir}" || die
290 - ln -sf "${DISTDIR}"/"${f}" "${goproxy_dir}/${goproxy_mod_path}" ||
291 - die "Failed to ln"
292 - local v=${goproxy_mod_path}
293 - v="${v%.mod}"
294 - v="${v%.zip}"
295 - v="${v//*\/}"
296 - _go-module_gosum_synthesize_files "${goproxy_mod_dir}" "${v}"
297 - else
298 - unpack "$f"
299 - fi
300 - done
301 -
302 - # Validate the gosum now
303 - _go-module_src_unpack_verify_gosum
304 -}
305 -
306 -# @FUNCTION: _go-module_gosum_synthesize_files
307 -# @DEPRECATED:
308 -# @DESCRIPTION:
309 -# Given a path & version, populate all Goproxy metadata files which aren't
310 -# needed to be downloaded directly.
311 -# - .../@v/${version}.info
312 -# - .../@v/list
313 -_go-module_gosum_synthesize_files() {
314 - local target=$1
315 - local version=$2
316 - # 'go get' doesn't care about the hash of the .info files, they
317 - # just need a 'version' element!
318 - # This saves a download of a tiny file
319 - # The .time key is omitted, because that is the time a module was added
320 - # to the upstream goproxy, and not metadata about the module itself.
321 - cat >"${target}/${version}.info" <<-EOF
322 - {
323 - "Version": "${version}",
324 - "shortName": "${version}",
325 - "Name": "${version}"
326 - }
327 - EOF
328 - listfile="${target}"/list
329 - if ! grep -sq -x -e "${version}" "${listfile}" 2>/dev/null; then
330 - echo "${version}" >>"${listfile}"
331 fi
332 -}
333 -
334 -# @FUNCTION: _go-module_src_unpack_verify_gosum
335 -# @DEPRECATED:
336 -# @DESCRIPTION:
337 -# Validate the Go modules declared by EGO_SUM are sufficient to cover building
338 -# the package, without actually building it yet.
339 -_go-module_src_unpack_verify_gosum() {
340 - # shellcheck disable=SC2120
341 - debug-print-function "${FUNCNAME}" "$@"
342 -
343 - if [[ ! ${_GO_MODULE_SET_GLOBALS_CALLED} ]]; then
344 - die "go-module_set_globals must be called in global scope"
345 - fi
346 -
347 - cd "${S}" || die "cd failed"
348 -
349 - # Cleanup the modules before starting anything else
350 - # This will print 'downloading' messages, but it's accessing content from
351 - # the $GOPROXY file:/// URL!
352 - einfo "Tidying go.mod/go.sum"
353 - nonfatal ego mod tidy >/dev/null
354 -
355 - # This used to call 'go get' to verify by fetching everything from the main
356 - # go.mod. However 'go get' also turns out to recursively try to fetch
357 - # everything in dependencies, even materials that are used only for tests
358 - # of the dependencies, or code generation.
359 - # If EGO_SUM is missing an entry now, it will fail during the build process
360 - # rather than this helper function.
361 + default
362 }
363
364 # @FUNCTION: go-module_live_vendor
365 @@ -481,31 +159,4 @@ go-module_live_vendor() {
366 popd >& /dev/null || die
367 }
368
369 -# @FUNCTION: _go-module_gomod_encode
370 -# @DEPRECATED:
371 -# @DESCRIPTION:
372 -# Encode the name(path) of a Golang module in the format expected by Goproxy.
373 -#
374 -# Upper letters are replaced by their lowercase version with a '!' prefix.
375 -#
376 -_go-module_gomod_encode() {
377 - ## Python:
378 - # return re.sub('([A-Z]{1})', r'!\1', s).lower()
379 -
380 - ## Sed:
381 - ## This uses GNU Sed extension \l to downcase the match
382 - #echo "${module}" |sed 's,[A-Z],!\l&,g'
383 - #
384 - # Bash variant:
385 - debug-print-function "${FUNCNAME}" "$@"
386 - #local re input lower
387 - re='(.*)([A-Z])(.*)'
388 - input="${1}"
389 - while [[ ${input} =~ ${re} ]]; do
390 - lower='!'"${BASH_REMATCH[2],}"
391 - input="${BASH_REMATCH[1]}${lower}${BASH_REMATCH[3]}"
392 - done
393 - echo "${input}"
394 -}
395 -
396 fi
397 --
398 2.34.1