Gentoo Archives: gentoo-portage-dev

From: "Michał Górny" <mgorny@g.o>
To: gentoo-portage-dev@l.g.o
Cc: "Michał Górny" <mgorny@g.o>
Subject: [gentoo-portage-dev] [PATCH 2/2] ecompress: Replace with implementation from portage[mgorny]
Date: Mon, 17 Sep 2018 17:07:13
Message-Id: 20180917170703.14329-2-mgorny@gentoo.org
In Reply to: [gentoo-portage-dev] [PATCH 1/2] Replace implicit doc compression with dir compression in old EAPIs by "Michał Górny"
1 Replace the old ecompress/ecompressdir implementation with the one used
2 in portage[mgorny]. This is a squashed version of a long series of
3 commits that gutted off parts of old logic, introduced the new code,
4 fixed ongoing bugs and finally restored the missing features. Given
5 the scale of changes, it is easier to review it as completely new code.
6
7 The duplicate model of ecompress/ecompressdir (both not exactly par
8 on features) has been replaced by a single helper that compresses
9 everything uniformly. Instead of complex path-wise processing,
10 a mark-file logic is used -- the same as in estrip. Additionally,
11 the functions are moved out of ebuild path to clearly indicate they are
12 internal API (they were not used in any ebuilds).
13
14 The new implementation is par on features with the old one. It supports
15 exclusion suffix lists, automatic decompression of already-compressed
16 files (with eqawarn-ing about its unreliability), lower bound for
17 compressed file size, hardlink breaking and symlink fixing. In other
18 words, there should be no regression upon replacing it.
19 ---
20 bin/ebuild-helpers/ecompress | 161 --------------------
21 bin/ebuild-helpers/ecompressdir | 226 -----------------------------
22 bin/ecompress | 147 +++++++++++++++++++
23 bin/ecompress-file | 67 +++++++++
24 bin/misc-functions.sh | 127 +---------------
25 lib/portage/tests/bin/setup_env.py | 2 +-
26 6 files changed, 220 insertions(+), 510 deletions(-)
27 delete mode 100755 bin/ebuild-helpers/ecompress
28 delete mode 100755 bin/ebuild-helpers/ecompressdir
29 create mode 100755 bin/ecompress
30 create mode 100755 bin/ecompress-file
31
32 diff --git a/bin/ebuild-helpers/ecompress b/bin/ebuild-helpers/ecompress
33 deleted file mode 100755
34 index 50ee81129..000000000
35 --- a/bin/ebuild-helpers/ecompress
36 +++ /dev/null
37 @@ -1,161 +0,0 @@
38 -#!/bin/bash
39 -# Copyright 1999-2010 Gentoo Foundation
40 -# Distributed under the terms of the GNU General Public License v2
41 -
42 -source "${PORTAGE_BIN_PATH}"/isolated-functions.sh || exit 1
43 -
44 -if [[ -z $1 ]] ; then
45 - __helpers_die "${0##*/}: at least one argument needed"
46 - exit 1
47 -fi
48 -
49 -# setup compression stuff
50 -PORTAGE_COMPRESS=${PORTAGE_COMPRESS-bzip2}
51 -[[ -z ${PORTAGE_COMPRESS} ]] && exit 0
52 -
53 -if [[ ${PORTAGE_COMPRESS_FLAGS+set} != "set" ]] ; then
54 - case ${PORTAGE_COMPRESS} in
55 - bzip2|gzip) PORTAGE_COMPRESS_FLAGS="-9";;
56 - esac
57 -fi
58 -
59 -# decompress_args(suffix, binary)
60 -# - suffix: the compression suffix to work with
61 -# - binary: the program to execute that'll compress/decompress
62 -# new_args: global array used to return revised arguments
63 -decompress_args() {
64 - local suffix=$1 binary=$2
65 - shift 2
66 -
67 - # Initialize the global new_args array.
68 - new_args=()
69 - declare -a decompress_args=()
70 - local x i=0 decompress_count=0
71 - for x in "$@" ; do
72 - if [[ ${x%$suffix} = $x ]] ; then
73 - new_args[$i]=$x
74 - else
75 - new_args[$i]=${x%$suffix}
76 - decompress_args[$decompress_count]=$x
77 - ((decompress_count++))
78 - fi
79 - ((i++))
80 - done
81 -
82 - if [ $decompress_count -gt 0 ] ; then
83 - ${binary} "${decompress_args[@]}"
84 - if [ $? -ne 0 ] ; then
85 - # Apparently decompression failed for one or more files, so
86 - # drop those since we don't want to compress them twice.
87 - new_args=()
88 - local x i=0
89 - for x in "$@" ; do
90 - if [[ ${x%$suffix} = $x ]] ; then
91 - new_args[$i]=$x
92 - ((i++))
93 - elif [[ -f ${x%$suffix} ]] ; then
94 - new_args[$i]=${x%$suffix}
95 - ((i++))
96 - else
97 - # Apparently decompression failed for this one, so drop
98 - # it since we don't want to compress it twice.
99 - true
100 - fi
101 - done
102 - fi
103 - fi
104 -}
105 -
106 -case $1 in
107 - --suffix)
108 - [[ -n $2 ]] && __vecho "${0##*/}: --suffix takes no additional arguments" 1>&2
109 -
110 - if [[ ! -e ${T}/.ecompress.suffix ]] ; then
111 - set -e
112 - tmpdir="${T}"/.ecompress$$.${RANDOM}
113 - mkdir "${tmpdir}"
114 - cd "${tmpdir}"
115 - # we have to fill the file enough so that there is something
116 - # to compress as some programs will refuse to do compression
117 - # if it cannot actually compress the file
118 - echo {0..1000} > compressme
119 - ${PORTAGE_COMPRESS} ${PORTAGE_COMPRESS_FLAGS} compressme > /dev/null
120 - # If PORTAGE_COMPRESS_FLAGS contains -k then we need to avoid
121 - # having our glob match the uncompressed file here.
122 - suffix=$(echo compressme.*)
123 - [[ -z $suffix || "$suffix" == "compressme.*" ]] && \
124 - suffix=$(echo compressme*)
125 - suffix=${suffix#compressme}
126 - cd /
127 - rm -rf "${tmpdir}"
128 - echo "${suffix}" > "${T}/.ecompress.suffix"
129 - fi
130 - cat "${T}/.ecompress.suffix"
131 - ;;
132 - --bin)
133 - [[ -n $2 ]] && __vecho "${0##*/}: --bin takes no additional arguments" 1>&2
134 -
135 - echo "${PORTAGE_COMPRESS} ${PORTAGE_COMPRESS_FLAGS}"
136 - ;;
137 - --queue)
138 - shift
139 - ret=0
140 - for x in "${@/%/.ecompress.file}" ; do
141 - >> "$x"
142 - ((ret|=$?))
143 - done
144 - [[ $ret -ne 0 ]] && __helpers_die "${0##*/} failed"
145 - exit $ret
146 - ;;
147 - --dequeue)
148 - [[ -n $2 ]] && __vecho "${0##*/}: --dequeue takes no additional arguments" 1>&2
149 - find "${D}" -name '*.ecompress.file' -print0 \
150 - | sed -e 's:\.ecompress\.file::g' \
151 - | ${XARGS} -0 ecompress
152 - find "${D}" -name '*.ecompress.file' -print0 | ${XARGS} -0 rm -f
153 - ;;
154 - --*)
155 - __helpers_die "${0##*/}: unknown arguments '$*'"
156 - exit 1
157 - ;;
158 - *)
159 - # Since dodoc calls ecompress on files that are already compressed,
160 - # perform decompression here (similar to ecompressdir behavior).
161 - decompress_args ".Z" "gunzip -f" "$@"
162 - set -- "${new_args[@]}"
163 - decompress_args ".gz" "gunzip -f" "$@"
164 - set -- "${new_args[@]}"
165 - decompress_args ".bz2" "bunzip2 -f" "$@"
166 - set -- "${new_args[@]}"
167 -
168 - mask_ext_re=""
169 - set -f
170 - for x in $PORTAGE_COMPRESS_EXCLUDE_SUFFIXES ; do
171 - mask_ext_re+="|$x"
172 - done
173 - set +f
174 - mask_ext_re="^(${mask_ext_re:1})\$"
175 - declare -a filtered_args=()
176 - i=0
177 - for x in "$@" ; do
178 - [[ ${x##*.} =~ $mask_ext_re ]] && continue
179 - [[ -s ${x} ]] || continue
180 - filtered_args[$i]=$x
181 - ((i++))
182 - done
183 - [ $i -eq 0 ] && exit 0
184 - set -- "${filtered_args[@]}"
185 -
186 - # If a compressed version of the file already exists, simply
187 - # delete it so that the compressor doesn't whine (bzip2 will
188 - # complain and skip, gzip will prompt for input)
189 - suffix=$(ecompress --suffix)
190 - [[ -n ${suffix} ]] && echo -n "${@/%/${suffix}$'\001'}" | \
191 - tr '\001' '\000' | ${XARGS} -0 rm -f
192 - # Finally, let's actually do some real work
193 - "${PORTAGE_COMPRESS}" ${PORTAGE_COMPRESS_FLAGS} "$@"
194 - ret=$?
195 - [[ $ret -ne 0 ]] && __helpers_die "${0##*/} failed"
196 - exit $ret
197 - ;;
198 -esac
199 diff --git a/bin/ebuild-helpers/ecompressdir b/bin/ebuild-helpers/ecompressdir
200 deleted file mode 100755
201 index dacb857be..000000000
202 --- a/bin/ebuild-helpers/ecompressdir
203 +++ /dev/null
204 @@ -1,226 +0,0 @@
205 -#!/bin/bash
206 -# Copyright 1999-2018 Gentoo Foundation
207 -# Distributed under the terms of the GNU General Public License v2
208 -
209 -source "${PORTAGE_BIN_PATH}"/helper-functions.sh || exit 1
210 -
211 -if [[ -z $1 ]] ; then
212 - __helpers_die "${0##*/}: at least one argument needed"
213 - exit 1
214 -fi
215 -
216 -if ! ___eapi_has_prefix_variables; then
217 - ED=${D} EPREFIX=
218 -fi
219 -
220 -SIZE_LIMIT=''
221 -while [[ $# -gt 0 ]] ; do
222 - case $1 in
223 - --ignore)
224 - shift
225 - for skip in "$@" ; do
226 - [[ -d ${ED%/}/${skip#/} || -f ${ED%/}/${skip#/} ]] \
227 - && >> "${ED%/}/${skip#/}.ecompress.skip"
228 - done
229 - exit 0
230 - ;;
231 - --queue)
232 - shift
233 - set -- "${@#/}"
234 - set -- "${@/%/.ecompress.dir}"
235 - set -- "${@/#/${ED%/}/}"
236 - ret=0
237 - for x in "$@" ; do
238 - # Stash the limit in the .dir file so we can reload it later.
239 - printf "${SIZE_LIMIT}" > "${x}"
240 - ((ret|=$?))
241 - done
242 - [[ $ret -ne 0 ]] && __helpers_die "${0##*/} failed"
243 - exit $ret
244 - ;;
245 - --dequeue)
246 - [[ -n $2 ]] && __vecho "${0##*/}: --dequeue takes no additional arguments" 1>&2
247 - find "${ED}" -name '*.ecompress.dir' -print0 \
248 - | sed -e 's:\.ecompress\.dir::g' -e "s:${ED%/}::g" \
249 - | ${XARGS} -0 ecompressdir
250 - find "${ED}" -name '*.ecompress.skip' -print0 | ${XARGS} -0 rm -f
251 - exit 0
252 - ;;
253 - --limit)
254 - SIZE_LIMIT=$2
255 - shift
256 - ;;
257 - --*)
258 - __helpers_die "${0##*/}: unknown arguments '$*'"
259 - exit 1
260 - ;;
261 - *)
262 - break
263 - ;;
264 - esac
265 - shift
266 -done
267 -
268 -# figure out the new suffix
269 -suffix=$(ecompress --suffix)
270 -
271 -# funk_up_dir(action, suffix, binary, [size_limit])
272 -# - action: compress or decompress
273 -# - suffix: the compression suffix to work with
274 -# - binary: the program to execute that'll compress/decompress
275 -# - size_limit: if compressing, skip files smaller than this
276 -# The directory we act on is implied in the ${dir} variable
277 -funk_up_dir() {
278 - local act=$1 suffix=$2 binary=$3 size_limit=$4
279 -
280 - local negate=""
281 - [[ ${act} == "compress" ]] && negate="!"
282 -
283 - local ret=0
284 - # first we act on all the files
285 - local args=(
286 - -type f
287 - ${negate} -iname "*${suffix}"
288 - )
289 - [[ -n ${size_limit} ]] && args+=( -size "+${size_limit}c" )
290 - find "${dir}" "${args[@]}" -print0 | ${XARGS} -0 ${binary}
291 - ((ret|=$?))
292 -
293 - # Repeat until nothing changes, in order to handle multiple
294 - # levels of indirection (see bug #470916).
295 - local -i indirection=0
296 - while true ; do
297 - local something_changed=
298 - while read -r -d $'\0' brokenlink ; do
299 - [[ -e ${brokenlink} ]] && continue
300 - olddest=$(readlink "${brokenlink}")
301 - # Ignore temporarily broken symlinks due to
302 - # _relocate_skip_dirs (bug #399595), and handle
303 - # absolute symlinks to files that aren't merged
304 - # yet (bug #405327).
305 - if [[ ${olddest} == /* ]] ; then
306 - [ -e "${D%/}${olddest}" ] && continue
307 - skip_dir_dest=${T}/ecompress-skip/${olddest#${EPREFIX}}
308 - else
309 - skip_dir_dest=${T}/ecompress-skip/${actual_dir#${ED%/}}/${brokenlink%/*}/${olddest}
310 - fi
311 - [[ -e ${skip_dir_dest} ]] && continue
312 - if [[ ${act} == "compress" ]] ; then
313 - newdest=${olddest}${suffix}
314 - else
315 - [[ ${olddest} == *${suffix} ]] || continue
316 - newdest=${olddest%${suffix}}
317 - fi
318 - if [[ "${newdest}" == /* ]] ; then
319 - [[ -f "${D%/}${newdest}" ]] || continue
320 - else
321 - [[ -f "${dir}/${brokenlink%/*}/${newdest}" ]] || continue
322 - fi
323 - something_changed=${brokenlink}
324 - rm -f "${brokenlink}"
325 - [[ ${act} == "compress" ]] \
326 - && ln -snf "${newdest}" "${brokenlink}${suffix}" \
327 - || ln -snf "${newdest}" "${brokenlink%${suffix}}"
328 - ((ret|=$?))
329 - done < <(find "${dir}" -type l -print0)
330 - [[ -n ${something_changed} ]] || break
331 - (( indirection++ ))
332 - if (( indirection >= 100 )) ; then
333 - # Protect against possibility of a bug triggering an endless loop.
334 - eerror "ecompressdir: too many levels of indirection for" \
335 - "'${actual_dir#${ED%/}}/${something_changed#./}'"
336 - break
337 - fi
338 - done
339 - return ${ret}
340 -}
341 -
342 -# _relocate_skip_dirs(srctree, dsttree)
343 -# Move all files and directories we want to skip running compression
344 -# on from srctree to dsttree.
345 -_relocate_skip_dirs() {
346 - local srctree="${1%/}" dsttree="${2%/}"
347 -
348 - [[ -d ${srctree} ]] || return 0
349 -
350 - find "${srctree}" -name '*.ecompress.skip' -print0 | \
351 - while read -r -d $'\0' src ; do
352 - src=${src%.ecompress.skip}
353 - dst="${dsttree}${src#${srctree}}"
354 - parent=${dst%/*}
355 - mkdir -p "${parent}"
356 - mv "${src}" "${dst}"
357 - mv "${src}.ecompress.skip" "${dst}.ecompress.skip"
358 - done
359 -}
360 -hide_skip_dirs() { _relocate_skip_dirs "${ED}" "${T}"/ecompress-skip/ ; }
361 -restore_skip_dirs() { _relocate_skip_dirs "${T}"/ecompress-skip/ "${ED}" ; }
362 -
363 -ret=0
364 -
365 -rm -rf "${T}"/ecompress-skip
366 -
367 -decompressors=(
368 - ".Z" "gunzip -f"
369 - ".gz" "gunzip -f"
370 - ".bz2" "bunzip2 -f"
371 - ".xz" "unxz -f"
372 - ".lzma" "unxz -f"
373 -)
374 -
375 -__multijob_init
376 -
377 -for dir in "$@" ; do
378 - dir=${dir#/}
379 - dir="${ED%/}/${dir#/}"
380 - if [[ ! -d ${dir} ]] ; then
381 - __vecho "${0##*/}: /${dir#${ED%/}} does not exist!"
382 - continue
383 - fi
384 - cd "${dir}"
385 - actual_dir=${dir}
386 - dir=. # use relative path to avoid 'Argument list too long' errors
387 -
388 - # hide all the stuff we want to skip
389 - hide_skip_dirs "${dir}"
390 -
391 - # since we've been requested to compress the whole dir,
392 - # delete any individual queued requests
393 - size_limit=${SIZE_LIMIT:-$(<"${actual_dir}.ecompress.dir")}
394 - rm -f "${actual_dir}.ecompress.dir"
395 - find "${dir}" -type f -name '*.ecompress.file' -print0 | ${XARGS} -0 rm -f
396 -
397 - # not uncommon for packages to compress doc files themselves
398 - for (( i = 0; i < ${#decompressors[@]}; i += 2 )) ; do
399 - # It's faster to parallelize at this stage than to try to
400 - # parallelize the compressors. This is because the find|xargs
401 - # ends up launching less compressors overall, so the overhead
402 - # of forking children ends up dominating.
403 - (
404 - __multijob_child_init
405 - funk_up_dir "decompress" "${decompressors[i]}" "${decompressors[i+1]}"
406 - ) &
407 - __multijob_post_fork
408 - : $(( ret |= $? ))
409 - done
410 -
411 - __multijob_finish
412 - : $(( ret |= $? ))
413 -
414 - # forcibly break all hard links as some compressors whine about it
415 - find "${dir}" -type f -links +1 -exec env file="{}" sh -c \
416 - 'cp -p "${file}" "${file}.ecompress.break" ; mv -f "${file}.ecompress.break" "${file}"' \;
417 -
418 - # now lets do our work
419 - if [[ -n ${suffix} ]] ; then
420 - __vecho "${0##*/}: $(ecompress --bin) /${actual_dir#${ED%/}/}"
421 - funk_up_dir "compress" "${suffix}" "ecompress" "${size_limit}"
422 - : $(( ret |= $? ))
423 - fi
424 -
425 - # finally, restore the skipped stuff
426 - restore_skip_dirs
427 -done
428 -
429 -[[ $ret -ne 0 ]] && __helpers_die "${0##*/} failed"
430 -exit ${ret}
431 diff --git a/bin/ecompress b/bin/ecompress
432 new file mode 100755
433 index 000000000..bd50f235f
434 --- /dev/null
435 +++ b/bin/ecompress
436 @@ -0,0 +1,147 @@
437 +#!/bin/bash
438 +# Copyright 1999-2018 Gentoo Foundation
439 +# Distributed under the terms of the GNU General Public License v2
440 +
441 +source "${PORTAGE_BIN_PATH}"/isolated-functions.sh || exit 1
442 +
443 +if [[ -z $1 ]] ; then
444 + __helpers_die "${0##*/}: at least one argument needed"
445 + exit 1
446 +fi
447 +
448 +if ! ___eapi_has_prefix_variables; then
449 + ED=${D} EPREFIX=
450 +fi
451 +
452 +while [[ $# -gt 0 ]] ; do
453 + case $1 in
454 + --ignore)
455 + shift
456 +
457 + skip_dirs=()
458 + for skip; do
459 + if [[ -d ${ED%/}/${skip#/} ]]; then
460 + skip_dirs+=( "${ED%/}/${skip#/}" )
461 + else
462 + rm -f "${ED%/}/${skip#/}.ecompress" || die
463 + fi
464 + done
465 +
466 + if [[ ${skip_dirs[@]} ]]; then
467 + find "${skip_dirs[@]}" -name '*.ecompress' -delete || die
468 + fi
469 +
470 + exit 0
471 + ;;
472 + --queue)
473 + shift
474 +
475 + find_args=()
476 + for path; do
477 + if [[ -e ${ED%/}/${path#/} ]]; then
478 + find_args+=( "${ED%/}/${path#/}" )
479 + fi
480 + done
481 +
482 + if [[ ${find_args[@]} ]]; then
483 + find_args+=( -type f )
484 + [[ -n ${PORTAGE_DOCOMPRESS_SIZE_LIMIT} ]] &&
485 + find_args+=( -size "+${PORTAGE_DOCOMPRESS_SIZE_LIMIT}c" )
486 +
487 + while IFS= read -d '' -r path; do
488 + >> "${path}.ecompress" || die
489 + done < <(find "${find_args[@]}" -print0 || die)
490 + fi
491 +
492 + exit 0
493 + ;;
494 + --dequeue)
495 + [[ -n ${2} ]] && die "${0##*/}: --dequeue takes no additional arguments"
496 + break
497 + ;;
498 + *)
499 + die "${0##*/}: unknown arguments '$*'"
500 + exit 1
501 + ;;
502 + esac
503 + shift
504 +done
505 +
506 +# setup compression stuff
507 +PORTAGE_COMPRESS=${PORTAGE_COMPRESS-bzip2}
508 +[[ -z ${PORTAGE_COMPRESS} ]] && exit 0
509 +
510 +if [[ ${PORTAGE_COMPRESS_FLAGS+set} != "set" ]] ; then
511 + case ${PORTAGE_COMPRESS} in
512 + bzip2|gzip) PORTAGE_COMPRESS_FLAGS="-9";;
513 + esac
514 +fi
515 +
516 +guess_suffix() {
517 + set -e
518 + tmpdir="${T}"/.ecompress$$.${RANDOM}
519 + mkdir "${tmpdir}"
520 + cd "${tmpdir}"
521 + # we have to fill the file enough so that there is something
522 + # to compress as some programs will refuse to do compression
523 + # if it cannot actually compress the file
524 + echo {0..1000} > compressme
525 + ${PORTAGE_COMPRESS} ${PORTAGE_COMPRESS_FLAGS} compressme > /dev/null
526 + # If PORTAGE_COMPRESS_FLAGS contains -k then we need to avoid
527 + # having our glob match the uncompressed file here.
528 + suffix=$(echo compressme.*)
529 + [[ -z $suffix || "$suffix" == "compressme.*" ]] && \
530 + suffix=$(echo compressme*)
531 + suffix=${suffix#compressme}
532 + cd /
533 + rm -rf "${tmpdir}"
534 + echo "${suffix}"
535 +}
536 +
537 +# figure out the new suffix
538 +export PORTAGE_COMPRESS_SUFFIX=$(guess_suffix) || die
539 +
540 +fix_symlinks() {
541 + # Repeat until nothing changes, in order to handle multiple
542 + # levels of indirection (see bug #470916).
543 + local -i indirection=0
544 + while true ; do
545 + local something_changed=
546 + while read -r -d $'\0' brokenlink ; do
547 + [[ -e ${brokenlink} ]] && continue
548 + olddest=$(readlink "${brokenlink}")
549 + newdest=${olddest}${PORTAGE_COMPRESS_SUFFIX}
550 + if [[ "${newdest}" == /* ]] ; then
551 + [[ -f "${D%/}${newdest}" ]] || continue
552 + else
553 + [[ -f "${dir}/${brokenlink%/*}/${newdest}" ]] || continue
554 + fi
555 + something_changed=${brokenlink}
556 + rm -f "${brokenlink}" &&
557 + ln -snf "${newdest}" "${brokenlink}${PORTAGE_COMPRESS_SUFFIX}"
558 + ((ret|=$?))
559 + done < <(find "${ED}" -type l -print0 || die)
560 +
561 + [[ -n ${something_changed} ]] || break
562 + (( indirection++ ))
563 + if (( indirection >= 100 )) ; then
564 + # Protect against possibility of a bug triggering an endless loop.
565 + eerror "ecompress: too many levels of indirection for" \
566 + "'${actual_dir#${ED%/}}/${something_changed#./}'"
567 + break
568 + fi
569 + done
570 +
571 + return ${ret}
572 +}
573 +
574 +export PORTAGE_COMPRESS PORTAGE_COMPRESS_FLAGS
575 +find "${ED}" -name '*.ecompress' -delete \
576 + -exec "${PORTAGE_BIN_PATH}"/ecompress-file {} +
577 +ret=${?}
578 +
579 +fix_symlinks
580 +: $(( ret |= ${?} ))
581 +[[ $ret -ne 0 ]] && __helpers_die "${0##*/} failed"
582 +
583 +exit ${ret}
584 diff --git a/bin/ecompress-file b/bin/ecompress-file
585 new file mode 100755
586 index 000000000..8d6601a91
587 --- /dev/null
588 +++ b/bin/ecompress-file
589 @@ -0,0 +1,67 @@
590 +#!/bin/bash
591 +# Copyright 1999-2018 Gentoo Foundation
592 +# Distributed under the terms of the GNU General Public License v2
593 +
594 +source "${PORTAGE_BIN_PATH}"/isolated-functions.sh || exit 1
595 +
596 +compress_file() {
597 + mask_ext_re=""
598 + set -f
599 + for x in $PORTAGE_COMPRESS_EXCLUDE_SUFFIXES ; do
600 + mask_ext_re+="|$x"
601 + done
602 + set +f
603 + mask_ext_re="^(${mask_ext_re:1})\$"
604 + local filtered_args=()
605 + local had_precompressed=
606 + for x in "$@" ; do
607 + [[ ${x##*.} =~ $mask_ext_re ]] && continue
608 + [[ -s ${x} ]] || continue
609 +
610 + # handle precompressed files
611 + case ${x} in
612 + *.gz|*.Z)
613 + gunzip -f "${x}" || __helpers_die "gunzip failed"
614 + x=${x%.*}
615 + had_precompressed=1;;
616 + *.bz2)
617 + bunzip2 -f "${x}" || __helpers_die "bunzip2 failed"
618 + x=${x%.bz2}
619 + had_precompressed=1;;
620 + *.lzma|*.xz)
621 + unxz -f "${x}" || __helpers_die "unxz failed"
622 + x=${x%.*}
623 + had_precompressed=1;;
624 + esac
625 +
626 + filtered_args+=( "$x" )
627 + done
628 + [[ -z ${filtered_args[@]} ]] && return 0
629 + set -- "${filtered_args[@]}"
630 +
631 + if [[ ${had_precompressed} ]]; then
632 + eqawarn "One or more compressed files were found in docompress-ed directories."
633 + eqawarn "Please fix the ebuild not to install compressed files (manpages,"
634 + eqawarn "documentation) when automatic compression is used."
635 + fi
636 +
637 + # If a compressed version of the file already exists, simply
638 + # delete it so that the compressor doesn't whine (bzip2 will
639 + # complain and skip, gzip will prompt for input)
640 + [[ -n ${PORTAGE_COMPRESS_SUFFIX} ]] && echo -n "${@/%/${PORTAGE_COMPRESS_SUFFIX}$'\001'}" | \
641 + tr '\001' '\000' | ${XARGS} -0 rm -f
642 +
643 + # forcibly break all hard links as some compressors whine about it
644 + while IFS= read -d '' -r x ; do
645 + cp -p "${x}" "${x}.ecompress.break" || die
646 + mv -f "${x}.ecompress.break" "${x}" || die
647 + done < <(find "${@}" -type f -links +1 -print0)
648 +
649 + # Finally, let's actually do some real work
650 + "${PORTAGE_COMPRESS}" ${PORTAGE_COMPRESS_FLAGS} "$@"
651 + ret=$?
652 + [[ $ret -ne 0 ]] && __helpers_die "${0##*/} failed"
653 + return ${ret}
654 +}
655 +
656 +compress_file "${@%.ecompress}"
657 diff --git a/bin/misc-functions.sh b/bin/misc-functions.sh
658 index ed66e90ca..8c88f8f35 100755
659 --- a/bin/misc-functions.sh
660 +++ b/bin/misc-functions.sh
661 @@ -43,125 +43,6 @@ install_symlink_html_docs() {
662 fi
663 }
664
665 -# replacement for "readlink -f" or "realpath"
666 -READLINK_F_WORKS=""
667 -canonicalize() {
668 - if [[ -z ${READLINK_F_WORKS} ]] ; then
669 - if [[ $(readlink -f -- /../ 2>/dev/null) == "/" ]] ; then
670 - READLINK_F_WORKS=true
671 - else
672 - READLINK_F_WORKS=false
673 - fi
674 - fi
675 - if ${READLINK_F_WORKS} ; then
676 - readlink -f -- "$@"
677 - return
678 - fi
679 -
680 - local f=$1 b n=10 wd=$(pwd)
681 - while (( n-- > 0 )); do
682 - while [[ ${f: -1} = / && ${#f} -gt 1 ]]; do
683 - f=${f%/}
684 - done
685 - b=${f##*/}
686 - cd "${f%"${b}"}" 2>/dev/null || break
687 - if [[ ! -L ${b} ]]; then
688 - f=$(pwd -P)
689 - echo "${f%/}/${b}"
690 - cd "${wd}"
691 - return 0
692 - fi
693 - f=$(readlink "${b}")
694 - done
695 - cd "${wd}"
696 - return 1
697 -}
698 -
699 -prepcompress() {
700 - local -a include exclude incl_d incl_f
701 - local f g i real_f real_d
702 - if ! ___eapi_has_prefix_variables; then
703 - local ED=${D}
704 - fi
705 -
706 - # Canonicalize path names and check for their existence.
707 - real_d=$(canonicalize "${ED}")
708 - for (( i = 0; i < ${#PORTAGE_DOCOMPRESS[@]}; i++ )); do
709 - real_f=$(canonicalize "${ED%/}/${PORTAGE_DOCOMPRESS[i]#/}")
710 - f=${real_f#"${real_d}"}
711 - if [[ ${real_f} != "${f}" ]] && [[ -d ${real_f} || -f ${real_f} ]]
712 - then
713 - include[${#include[@]}]=${f:-/}
714 - elif [[ ${i} -ge 3 ]]; then
715 - ewarn "prepcompress:" \
716 - "ignoring nonexistent path '${PORTAGE_DOCOMPRESS[i]}'"
717 - fi
718 - done
719 - for (( i = 0; i < ${#PORTAGE_DOCOMPRESS_SKIP[@]}; i++ )); do
720 - real_f=$(canonicalize "${ED%/}/${PORTAGE_DOCOMPRESS_SKIP[i]#/}")
721 - f=${real_f#"${real_d}"}
722 - if [[ ${real_f} != "${f}" ]] && [[ -d ${real_f} || -f ${real_f} ]]
723 - then
724 - exclude[${#exclude[@]}]=${f:-/}
725 - elif [[ ${i} -ge 1 ]]; then
726 - ewarn "prepcompress:" \
727 - "ignoring nonexistent path '${PORTAGE_DOCOMPRESS_SKIP[i]}'"
728 - fi
729 - done
730 -
731 - # Remove redundant entries from lists.
732 - # For the include list, remove any entries that are:
733 - # a) contained in a directory in the include or exclude lists, or
734 - # b) identical with an entry in the exclude list.
735 - for (( i = ${#include[@]} - 1; i >= 0; i-- )); do
736 - f=${include[i]}
737 - for g in "${include[@]}"; do
738 - if [[ ${f} == "${g%/}"/* ]]; then
739 - unset include[i]
740 - continue 2
741 - fi
742 - done
743 - for g in "${exclude[@]}"; do
744 - if [[ ${f} = "${g}" || ${f} == "${g%/}"/* ]]; then
745 - unset include[i]
746 - continue 2
747 - fi
748 - done
749 - done
750 - # For the exclude list, remove any entries that are:
751 - # a) contained in a directory in the exclude list, or
752 - # b) _not_ contained in a directory in the include list.
753 - for (( i = ${#exclude[@]} - 1; i >= 0; i-- )); do
754 - f=${exclude[i]}
755 - for g in "${exclude[@]}"; do
756 - if [[ ${f} == "${g%/}"/* ]]; then
757 - unset exclude[i]
758 - continue 2
759 - fi
760 - done
761 - for g in "${include[@]}"; do
762 - [[ ${f} == "${g%/}"/* ]] && continue 2
763 - done
764 - unset exclude[i]
765 - done
766 -
767 - # Split the include list into directories and files
768 - for f in "${include[@]}"; do
769 - if [[ -d ${ED%/}/${f#/} ]]; then
770 - incl_d[${#incl_d[@]}]=${f}
771 - else
772 - incl_f[${#incl_f[@]}]=${f}
773 - fi
774 - done
775 -
776 - # Queue up for compression.
777 - # ecompress{,dir} doesn't like to be called with empty argument lists.
778 - [[ ${#incl_d[@]} -gt 0 ]] && ecompressdir --limit ${PORTAGE_DOCOMPRESS_SIZE_LIMIT:-0} --queue "${incl_d[@]}"
779 - [[ ${#incl_f[@]} -gt 0 ]] && ecompress --queue "${incl_f[@]/#/${ED%/}}"
780 - [[ ${#exclude[@]} -gt 0 ]] && ecompressdir --ignore "${exclude[@]}"
781 - return 0
782 -}
783 -
784 __prepall() {
785 if has chflags $FEATURES ; then
786 # Save all the file flags for restoration at the end of prepall.
787 @@ -173,6 +54,11 @@ __prepall() {
788
789 [[ -d ${ED%/}/usr/share/info ]] && prepinfo
790
791 + # Apply compression.
792 + "${PORTAGE_BIN_PATH}"/ecompress --queue "${PORTAGE_DOCOMPRESS[@]}"
793 + "${PORTAGE_BIN_PATH}"/ecompress --ignore "${PORTAGE_DOCOMPRESS_SKIP[@]}"
794 + "${PORTAGE_BIN_PATH}"/ecompress --dequeue
795 +
796 ___eapi_has_dostrip || prepallstrip
797
798 if has chflags $FEATURES ; then
799 @@ -239,9 +125,6 @@ install_qa_check() {
800
801 export STRIP_MASK
802 __prepall
803 - prepcompress
804 - ecompressdir --dequeue
805 - ecompress --dequeue
806
807 if ___eapi_has_dostrip; then
808 "${PORTAGE_BIN_PATH}"/estrip --queue "${PORTAGE_DOSTRIP[@]}"
809 diff --git a/lib/portage/tests/bin/setup_env.py b/lib/portage/tests/bin/setup_env.py
810 index 9cc26df08..33b167989 100644
811 --- a/lib/portage/tests/bin/setup_env.py
812 +++ b/lib/portage/tests/bin/setup_env.py
813 @@ -82,6 +82,6 @@ for bin in os.listdir(os.path.join(bindir, 'ebuild-helpers')):
814 if bin.startswith('do') or \
815 bin.startswith('new') or \
816 bin.startswith('prep') or \
817 - bin in ('ecompress', 'ecompressdir', 'fowners', 'fperms'):
818 + bin in ('fowners', 'fperms'):
819 globals()[bin] = create_portage_wrapper(
820 os.path.join(bindir, 'ebuild-helpers', bin))
821 --
822 2.19.0