1 |
Replace the old ecompress/ecompressdir implementation with the one used |
2 |
in portage[mgorny]. This is a squashed version of a long series of |
3 |
commits that gutted off parts of old logic, introduced the new code, |
4 |
fixed ongoing bugs and finally restored the missing features. Given |
5 |
the scale of changes, it is easier to review it as completely new code. |
6 |
|
7 |
The duplicate model of ecompress/ecompressdir (both not exactly par |
8 |
on features) has been replaced by a single helper that compresses |
9 |
everything uniformly. Instead of complex path-wise processing, |
10 |
a mark-file logic is used -- the same as in estrip. Additionally, |
11 |
the functions are moved out of ebuild path to clearly indicate they are |
12 |
internal API (they were not used in any ebuilds). |
13 |
|
14 |
The new implementation is par on features with the old one. It supports |
15 |
exclusion suffix lists, automatic decompression of already-compressed |
16 |
files (with eqawarn-ing about its unreliability), lower bound for |
17 |
compressed file size, hardlink breaking and symlink fixing. In other |
18 |
words, there should be no regression upon replacing it. |
19 |
--- |
20 |
bin/ebuild-helpers/ecompress | 161 -------------------- |
21 |
bin/ebuild-helpers/ecompressdir | 226 ----------------------------- |
22 |
bin/ecompress | 147 +++++++++++++++++++ |
23 |
bin/ecompress-file | 67 +++++++++ |
24 |
bin/misc-functions.sh | 127 +--------------- |
25 |
lib/portage/tests/bin/setup_env.py | 2 +- |
26 |
6 files changed, 220 insertions(+), 510 deletions(-) |
27 |
delete mode 100755 bin/ebuild-helpers/ecompress |
28 |
delete mode 100755 bin/ebuild-helpers/ecompressdir |
29 |
create mode 100755 bin/ecompress |
30 |
create mode 100755 bin/ecompress-file |
31 |
|
32 |
diff --git a/bin/ebuild-helpers/ecompress b/bin/ebuild-helpers/ecompress |
33 |
deleted file mode 100755 |
34 |
index 50ee81129..000000000 |
35 |
--- a/bin/ebuild-helpers/ecompress |
36 |
+++ /dev/null |
37 |
@@ -1,161 +0,0 @@ |
38 |
-#!/bin/bash |
39 |
-# Copyright 1999-2010 Gentoo Foundation |
40 |
-# Distributed under the terms of the GNU General Public License v2 |
41 |
- |
42 |
-source "${PORTAGE_BIN_PATH}"/isolated-functions.sh || exit 1 |
43 |
- |
44 |
-if [[ -z $1 ]] ; then |
45 |
- __helpers_die "${0##*/}: at least one argument needed" |
46 |
- exit 1 |
47 |
-fi |
48 |
- |
49 |
-# setup compression stuff |
50 |
-PORTAGE_COMPRESS=${PORTAGE_COMPRESS-bzip2} |
51 |
-[[ -z ${PORTAGE_COMPRESS} ]] && exit 0 |
52 |
- |
53 |
-if [[ ${PORTAGE_COMPRESS_FLAGS+set} != "set" ]] ; then |
54 |
- case ${PORTAGE_COMPRESS} in |
55 |
- bzip2|gzip) PORTAGE_COMPRESS_FLAGS="-9";; |
56 |
- esac |
57 |
-fi |
58 |
- |
59 |
-# decompress_args(suffix, binary) |
60 |
-# - suffix: the compression suffix to work with |
61 |
-# - binary: the program to execute that'll compress/decompress |
62 |
-# new_args: global array used to return revised arguments |
63 |
-decompress_args() { |
64 |
- local suffix=$1 binary=$2 |
65 |
- shift 2 |
66 |
- |
67 |
- # Initialize the global new_args array. |
68 |
- new_args=() |
69 |
- declare -a decompress_args=() |
70 |
- local x i=0 decompress_count=0 |
71 |
- for x in "$@" ; do |
72 |
- if [[ ${x%$suffix} = $x ]] ; then |
73 |
- new_args[$i]=$x |
74 |
- else |
75 |
- new_args[$i]=${x%$suffix} |
76 |
- decompress_args[$decompress_count]=$x |
77 |
- ((decompress_count++)) |
78 |
- fi |
79 |
- ((i++)) |
80 |
- done |
81 |
- |
82 |
- if [ $decompress_count -gt 0 ] ; then |
83 |
- ${binary} "${decompress_args[@]}" |
84 |
- if [ $? -ne 0 ] ; then |
85 |
- # Apparently decompression failed for one or more files, so |
86 |
- # drop those since we don't want to compress them twice. |
87 |
- new_args=() |
88 |
- local x i=0 |
89 |
- for x in "$@" ; do |
90 |
- if [[ ${x%$suffix} = $x ]] ; then |
91 |
- new_args[$i]=$x |
92 |
- ((i++)) |
93 |
- elif [[ -f ${x%$suffix} ]] ; then |
94 |
- new_args[$i]=${x%$suffix} |
95 |
- ((i++)) |
96 |
- else |
97 |
- # Apparently decompression failed for this one, so drop |
98 |
- # it since we don't want to compress it twice. |
99 |
- true |
100 |
- fi |
101 |
- done |
102 |
- fi |
103 |
- fi |
104 |
-} |
105 |
- |
106 |
-case $1 in |
107 |
- --suffix) |
108 |
- [[ -n $2 ]] && __vecho "${0##*/}: --suffix takes no additional arguments" 1>&2 |
109 |
- |
110 |
- if [[ ! -e ${T}/.ecompress.suffix ]] ; then |
111 |
- set -e |
112 |
- tmpdir="${T}"/.ecompress$$.${RANDOM} |
113 |
- mkdir "${tmpdir}" |
114 |
- cd "${tmpdir}" |
115 |
- # we have to fill the file enough so that there is something |
116 |
- # to compress as some programs will refuse to do compression |
117 |
- # if it cannot actually compress the file |
118 |
- echo {0..1000} > compressme |
119 |
- ${PORTAGE_COMPRESS} ${PORTAGE_COMPRESS_FLAGS} compressme > /dev/null |
120 |
- # If PORTAGE_COMPRESS_FLAGS contains -k then we need to avoid |
121 |
- # having our glob match the uncompressed file here. |
122 |
- suffix=$(echo compressme.*) |
123 |
- [[ -z $suffix || "$suffix" == "compressme.*" ]] && \ |
124 |
- suffix=$(echo compressme*) |
125 |
- suffix=${suffix#compressme} |
126 |
- cd / |
127 |
- rm -rf "${tmpdir}" |
128 |
- echo "${suffix}" > "${T}/.ecompress.suffix" |
129 |
- fi |
130 |
- cat "${T}/.ecompress.suffix" |
131 |
- ;; |
132 |
- --bin) |
133 |
- [[ -n $2 ]] && __vecho "${0##*/}: --bin takes no additional arguments" 1>&2 |
134 |
- |
135 |
- echo "${PORTAGE_COMPRESS} ${PORTAGE_COMPRESS_FLAGS}" |
136 |
- ;; |
137 |
- --queue) |
138 |
- shift |
139 |
- ret=0 |
140 |
- for x in "${@/%/.ecompress.file}" ; do |
141 |
- >> "$x" |
142 |
- ((ret|=$?)) |
143 |
- done |
144 |
- [[ $ret -ne 0 ]] && __helpers_die "${0##*/} failed" |
145 |
- exit $ret |
146 |
- ;; |
147 |
- --dequeue) |
148 |
- [[ -n $2 ]] && __vecho "${0##*/}: --dequeue takes no additional arguments" 1>&2 |
149 |
- find "${D}" -name '*.ecompress.file' -print0 \ |
150 |
- | sed -e 's:\.ecompress\.file::g' \ |
151 |
- | ${XARGS} -0 ecompress |
152 |
- find "${D}" -name '*.ecompress.file' -print0 | ${XARGS} -0 rm -f |
153 |
- ;; |
154 |
- --*) |
155 |
- __helpers_die "${0##*/}: unknown arguments '$*'" |
156 |
- exit 1 |
157 |
- ;; |
158 |
- *) |
159 |
- # Since dodoc calls ecompress on files that are already compressed, |
160 |
- # perform decompression here (similar to ecompressdir behavior). |
161 |
- decompress_args ".Z" "gunzip -f" "$@" |
162 |
- set -- "${new_args[@]}" |
163 |
- decompress_args ".gz" "gunzip -f" "$@" |
164 |
- set -- "${new_args[@]}" |
165 |
- decompress_args ".bz2" "bunzip2 -f" "$@" |
166 |
- set -- "${new_args[@]}" |
167 |
- |
168 |
- mask_ext_re="" |
169 |
- set -f |
170 |
- for x in $PORTAGE_COMPRESS_EXCLUDE_SUFFIXES ; do |
171 |
- mask_ext_re+="|$x" |
172 |
- done |
173 |
- set +f |
174 |
- mask_ext_re="^(${mask_ext_re:1})\$" |
175 |
- declare -a filtered_args=() |
176 |
- i=0 |
177 |
- for x in "$@" ; do |
178 |
- [[ ${x##*.} =~ $mask_ext_re ]] && continue |
179 |
- [[ -s ${x} ]] || continue |
180 |
- filtered_args[$i]=$x |
181 |
- ((i++)) |
182 |
- done |
183 |
- [ $i -eq 0 ] && exit 0 |
184 |
- set -- "${filtered_args[@]}" |
185 |
- |
186 |
- # If a compressed version of the file already exists, simply |
187 |
- # delete it so that the compressor doesn't whine (bzip2 will |
188 |
- # complain and skip, gzip will prompt for input) |
189 |
- suffix=$(ecompress --suffix) |
190 |
- [[ -n ${suffix} ]] && echo -n "${@/%/${suffix}$'\001'}" | \ |
191 |
- tr '\001' '\000' | ${XARGS} -0 rm -f |
192 |
- # Finally, let's actually do some real work |
193 |
- "${PORTAGE_COMPRESS}" ${PORTAGE_COMPRESS_FLAGS} "$@" |
194 |
- ret=$? |
195 |
- [[ $ret -ne 0 ]] && __helpers_die "${0##*/} failed" |
196 |
- exit $ret |
197 |
- ;; |
198 |
-esac |
199 |
diff --git a/bin/ebuild-helpers/ecompressdir b/bin/ebuild-helpers/ecompressdir |
200 |
deleted file mode 100755 |
201 |
index dacb857be..000000000 |
202 |
--- a/bin/ebuild-helpers/ecompressdir |
203 |
+++ /dev/null |
204 |
@@ -1,226 +0,0 @@ |
205 |
-#!/bin/bash |
206 |
-# Copyright 1999-2018 Gentoo Foundation |
207 |
-# Distributed under the terms of the GNU General Public License v2 |
208 |
- |
209 |
-source "${PORTAGE_BIN_PATH}"/helper-functions.sh || exit 1 |
210 |
- |
211 |
-if [[ -z $1 ]] ; then |
212 |
- __helpers_die "${0##*/}: at least one argument needed" |
213 |
- exit 1 |
214 |
-fi |
215 |
- |
216 |
-if ! ___eapi_has_prefix_variables; then |
217 |
- ED=${D} EPREFIX= |
218 |
-fi |
219 |
- |
220 |
-SIZE_LIMIT='' |
221 |
-while [[ $# -gt 0 ]] ; do |
222 |
- case $1 in |
223 |
- --ignore) |
224 |
- shift |
225 |
- for skip in "$@" ; do |
226 |
- [[ -d ${ED%/}/${skip#/} || -f ${ED%/}/${skip#/} ]] \ |
227 |
- && >> "${ED%/}/${skip#/}.ecompress.skip" |
228 |
- done |
229 |
- exit 0 |
230 |
- ;; |
231 |
- --queue) |
232 |
- shift |
233 |
- set -- "${@#/}" |
234 |
- set -- "${@/%/.ecompress.dir}" |
235 |
- set -- "${@/#/${ED%/}/}" |
236 |
- ret=0 |
237 |
- for x in "$@" ; do |
238 |
- # Stash the limit in the .dir file so we can reload it later. |
239 |
- printf "${SIZE_LIMIT}" > "${x}" |
240 |
- ((ret|=$?)) |
241 |
- done |
242 |
- [[ $ret -ne 0 ]] && __helpers_die "${0##*/} failed" |
243 |
- exit $ret |
244 |
- ;; |
245 |
- --dequeue) |
246 |
- [[ -n $2 ]] && __vecho "${0##*/}: --dequeue takes no additional arguments" 1>&2 |
247 |
- find "${ED}" -name '*.ecompress.dir' -print0 \ |
248 |
- | sed -e 's:\.ecompress\.dir::g' -e "s:${ED%/}::g" \ |
249 |
- | ${XARGS} -0 ecompressdir |
250 |
- find "${ED}" -name '*.ecompress.skip' -print0 | ${XARGS} -0 rm -f |
251 |
- exit 0 |
252 |
- ;; |
253 |
- --limit) |
254 |
- SIZE_LIMIT=$2 |
255 |
- shift |
256 |
- ;; |
257 |
- --*) |
258 |
- __helpers_die "${0##*/}: unknown arguments '$*'" |
259 |
- exit 1 |
260 |
- ;; |
261 |
- *) |
262 |
- break |
263 |
- ;; |
264 |
- esac |
265 |
- shift |
266 |
-done |
267 |
- |
268 |
-# figure out the new suffix |
269 |
-suffix=$(ecompress --suffix) |
270 |
- |
271 |
-# funk_up_dir(action, suffix, binary, [size_limit]) |
272 |
-# - action: compress or decompress |
273 |
-# - suffix: the compression suffix to work with |
274 |
-# - binary: the program to execute that'll compress/decompress |
275 |
-# - size_limit: if compressing, skip files smaller than this |
276 |
-# The directory we act on is implied in the ${dir} variable |
277 |
-funk_up_dir() { |
278 |
- local act=$1 suffix=$2 binary=$3 size_limit=$4 |
279 |
- |
280 |
- local negate="" |
281 |
- [[ ${act} == "compress" ]] && negate="!" |
282 |
- |
283 |
- local ret=0 |
284 |
- # first we act on all the files |
285 |
- local args=( |
286 |
- -type f |
287 |
- ${negate} -iname "*${suffix}" |
288 |
- ) |
289 |
- [[ -n ${size_limit} ]] && args+=( -size "+${size_limit}c" ) |
290 |
- find "${dir}" "${args[@]}" -print0 | ${XARGS} -0 ${binary} |
291 |
- ((ret|=$?)) |
292 |
- |
293 |
- # Repeat until nothing changes, in order to handle multiple |
294 |
- # levels of indirection (see bug #470916). |
295 |
- local -i indirection=0 |
296 |
- while true ; do |
297 |
- local something_changed= |
298 |
- while read -r -d $'\0' brokenlink ; do |
299 |
- [[ -e ${brokenlink} ]] && continue |
300 |
- olddest=$(readlink "${brokenlink}") |
301 |
- # Ignore temporarily broken symlinks due to |
302 |
- # _relocate_skip_dirs (bug #399595), and handle |
303 |
- # absolute symlinks to files that aren't merged |
304 |
- # yet (bug #405327). |
305 |
- if [[ ${olddest} == /* ]] ; then |
306 |
- [ -e "${D%/}${olddest}" ] && continue |
307 |
- skip_dir_dest=${T}/ecompress-skip/${olddest#${EPREFIX}} |
308 |
- else |
309 |
- skip_dir_dest=${T}/ecompress-skip/${actual_dir#${ED%/}}/${brokenlink%/*}/${olddest} |
310 |
- fi |
311 |
- [[ -e ${skip_dir_dest} ]] && continue |
312 |
- if [[ ${act} == "compress" ]] ; then |
313 |
- newdest=${olddest}${suffix} |
314 |
- else |
315 |
- [[ ${olddest} == *${suffix} ]] || continue |
316 |
- newdest=${olddest%${suffix}} |
317 |
- fi |
318 |
- if [[ "${newdest}" == /* ]] ; then |
319 |
- [[ -f "${D%/}${newdest}" ]] || continue |
320 |
- else |
321 |
- [[ -f "${dir}/${brokenlink%/*}/${newdest}" ]] || continue |
322 |
- fi |
323 |
- something_changed=${brokenlink} |
324 |
- rm -f "${brokenlink}" |
325 |
- [[ ${act} == "compress" ]] \ |
326 |
- && ln -snf "${newdest}" "${brokenlink}${suffix}" \ |
327 |
- || ln -snf "${newdest}" "${brokenlink%${suffix}}" |
328 |
- ((ret|=$?)) |
329 |
- done < <(find "${dir}" -type l -print0) |
330 |
- [[ -n ${something_changed} ]] || break |
331 |
- (( indirection++ )) |
332 |
- if (( indirection >= 100 )) ; then |
333 |
- # Protect against possibility of a bug triggering an endless loop. |
334 |
- eerror "ecompressdir: too many levels of indirection for" \ |
335 |
- "'${actual_dir#${ED%/}}/${something_changed#./}'" |
336 |
- break |
337 |
- fi |
338 |
- done |
339 |
- return ${ret} |
340 |
-} |
341 |
- |
342 |
-# _relocate_skip_dirs(srctree, dsttree) |
343 |
-# Move all files and directories we want to skip running compression |
344 |
-# on from srctree to dsttree. |
345 |
-_relocate_skip_dirs() { |
346 |
- local srctree="${1%/}" dsttree="${2%/}" |
347 |
- |
348 |
- [[ -d ${srctree} ]] || return 0 |
349 |
- |
350 |
- find "${srctree}" -name '*.ecompress.skip' -print0 | \ |
351 |
- while read -r -d $'\0' src ; do |
352 |
- src=${src%.ecompress.skip} |
353 |
- dst="${dsttree}${src#${srctree}}" |
354 |
- parent=${dst%/*} |
355 |
- mkdir -p "${parent}" |
356 |
- mv "${src}" "${dst}" |
357 |
- mv "${src}.ecompress.skip" "${dst}.ecompress.skip" |
358 |
- done |
359 |
-} |
360 |
-hide_skip_dirs() { _relocate_skip_dirs "${ED}" "${T}"/ecompress-skip/ ; } |
361 |
-restore_skip_dirs() { _relocate_skip_dirs "${T}"/ecompress-skip/ "${ED}" ; } |
362 |
- |
363 |
-ret=0 |
364 |
- |
365 |
-rm -rf "${T}"/ecompress-skip |
366 |
- |
367 |
-decompressors=( |
368 |
- ".Z" "gunzip -f" |
369 |
- ".gz" "gunzip -f" |
370 |
- ".bz2" "bunzip2 -f" |
371 |
- ".xz" "unxz -f" |
372 |
- ".lzma" "unxz -f" |
373 |
-) |
374 |
- |
375 |
-__multijob_init |
376 |
- |
377 |
-for dir in "$@" ; do |
378 |
- dir=${dir#/} |
379 |
- dir="${ED%/}/${dir#/}" |
380 |
- if [[ ! -d ${dir} ]] ; then |
381 |
- __vecho "${0##*/}: /${dir#${ED%/}} does not exist!" |
382 |
- continue |
383 |
- fi |
384 |
- cd "${dir}" |
385 |
- actual_dir=${dir} |
386 |
- dir=. # use relative path to avoid 'Argument list too long' errors |
387 |
- |
388 |
- # hide all the stuff we want to skip |
389 |
- hide_skip_dirs "${dir}" |
390 |
- |
391 |
- # since we've been requested to compress the whole dir, |
392 |
- # delete any individual queued requests |
393 |
- size_limit=${SIZE_LIMIT:-$(<"${actual_dir}.ecompress.dir")} |
394 |
- rm -f "${actual_dir}.ecompress.dir" |
395 |
- find "${dir}" -type f -name '*.ecompress.file' -print0 | ${XARGS} -0 rm -f |
396 |
- |
397 |
- # not uncommon for packages to compress doc files themselves |
398 |
- for (( i = 0; i < ${#decompressors[@]}; i += 2 )) ; do |
399 |
- # It's faster to parallelize at this stage than to try to |
400 |
- # parallelize the compressors. This is because the find|xargs |
401 |
- # ends up launching less compressors overall, so the overhead |
402 |
- # of forking children ends up dominating. |
403 |
- ( |
404 |
- __multijob_child_init |
405 |
- funk_up_dir "decompress" "${decompressors[i]}" "${decompressors[i+1]}" |
406 |
- ) & |
407 |
- __multijob_post_fork |
408 |
- : $(( ret |= $? )) |
409 |
- done |
410 |
- |
411 |
- __multijob_finish |
412 |
- : $(( ret |= $? )) |
413 |
- |
414 |
- # forcibly break all hard links as some compressors whine about it |
415 |
- find "${dir}" -type f -links +1 -exec env file="{}" sh -c \ |
416 |
- 'cp -p "${file}" "${file}.ecompress.break" ; mv -f "${file}.ecompress.break" "${file}"' \; |
417 |
- |
418 |
- # now lets do our work |
419 |
- if [[ -n ${suffix} ]] ; then |
420 |
- __vecho "${0##*/}: $(ecompress --bin) /${actual_dir#${ED%/}/}" |
421 |
- funk_up_dir "compress" "${suffix}" "ecompress" "${size_limit}" |
422 |
- : $(( ret |= $? )) |
423 |
- fi |
424 |
- |
425 |
- # finally, restore the skipped stuff |
426 |
- restore_skip_dirs |
427 |
-done |
428 |
- |
429 |
-[[ $ret -ne 0 ]] && __helpers_die "${0##*/} failed" |
430 |
-exit ${ret} |
431 |
diff --git a/bin/ecompress b/bin/ecompress |
432 |
new file mode 100755 |
433 |
index 000000000..bd50f235f |
434 |
--- /dev/null |
435 |
+++ b/bin/ecompress |
436 |
@@ -0,0 +1,147 @@ |
437 |
+#!/bin/bash |
438 |
+# Copyright 1999-2018 Gentoo Foundation |
439 |
+# Distributed under the terms of the GNU General Public License v2 |
440 |
+ |
441 |
+source "${PORTAGE_BIN_PATH}"/isolated-functions.sh || exit 1 |
442 |
+ |
443 |
+if [[ -z $1 ]] ; then |
444 |
+ __helpers_die "${0##*/}: at least one argument needed" |
445 |
+ exit 1 |
446 |
+fi |
447 |
+ |
448 |
+if ! ___eapi_has_prefix_variables; then |
449 |
+ ED=${D} EPREFIX= |
450 |
+fi |
451 |
+ |
452 |
+while [[ $# -gt 0 ]] ; do |
453 |
+ case $1 in |
454 |
+ --ignore) |
455 |
+ shift |
456 |
+ |
457 |
+ skip_dirs=() |
458 |
+ for skip; do |
459 |
+ if [[ -d ${ED%/}/${skip#/} ]]; then |
460 |
+ skip_dirs+=( "${ED%/}/${skip#/}" ) |
461 |
+ else |
462 |
+ rm -f "${ED%/}/${skip#/}.ecompress" || die |
463 |
+ fi |
464 |
+ done |
465 |
+ |
466 |
+ if [[ ${skip_dirs[@]} ]]; then |
467 |
+ find "${skip_dirs[@]}" -name '*.ecompress' -delete || die |
468 |
+ fi |
469 |
+ |
470 |
+ exit 0 |
471 |
+ ;; |
472 |
+ --queue) |
473 |
+ shift |
474 |
+ |
475 |
+ find_args=() |
476 |
+ for path; do |
477 |
+ if [[ -e ${ED%/}/${path#/} ]]; then |
478 |
+ find_args+=( "${ED%/}/${path#/}" ) |
479 |
+ fi |
480 |
+ done |
481 |
+ |
482 |
+ if [[ ${find_args[@]} ]]; then |
483 |
+ find_args+=( -type f ) |
484 |
+ [[ -n ${PORTAGE_DOCOMPRESS_SIZE_LIMIT} ]] && |
485 |
+ find_args+=( -size "+${PORTAGE_DOCOMPRESS_SIZE_LIMIT}c" ) |
486 |
+ |
487 |
+ while IFS= read -d '' -r path; do |
488 |
+ >> "${path}.ecompress" || die |
489 |
+ done < <(find "${find_args[@]}" -print0 || die) |
490 |
+ fi |
491 |
+ |
492 |
+ exit 0 |
493 |
+ ;; |
494 |
+ --dequeue) |
495 |
+ [[ -n ${2} ]] && die "${0##*/}: --dequeue takes no additional arguments" |
496 |
+ break |
497 |
+ ;; |
498 |
+ *) |
499 |
+ die "${0##*/}: unknown arguments '$*'" |
500 |
+ exit 1 |
501 |
+ ;; |
502 |
+ esac |
503 |
+ shift |
504 |
+done |
505 |
+ |
506 |
+# setup compression stuff |
507 |
+PORTAGE_COMPRESS=${PORTAGE_COMPRESS-bzip2} |
508 |
+[[ -z ${PORTAGE_COMPRESS} ]] && exit 0 |
509 |
+ |
510 |
+if [[ ${PORTAGE_COMPRESS_FLAGS+set} != "set" ]] ; then |
511 |
+ case ${PORTAGE_COMPRESS} in |
512 |
+ bzip2|gzip) PORTAGE_COMPRESS_FLAGS="-9";; |
513 |
+ esac |
514 |
+fi |
515 |
+ |
516 |
+guess_suffix() { |
517 |
+ set -e |
518 |
+ tmpdir="${T}"/.ecompress$$.${RANDOM} |
519 |
+ mkdir "${tmpdir}" |
520 |
+ cd "${tmpdir}" |
521 |
+ # we have to fill the file enough so that there is something |
522 |
+ # to compress as some programs will refuse to do compression |
523 |
+ # if it cannot actually compress the file |
524 |
+ echo {0..1000} > compressme |
525 |
+ ${PORTAGE_COMPRESS} ${PORTAGE_COMPRESS_FLAGS} compressme > /dev/null |
526 |
+ # If PORTAGE_COMPRESS_FLAGS contains -k then we need to avoid |
527 |
+ # having our glob match the uncompressed file here. |
528 |
+ suffix=$(echo compressme.*) |
529 |
+ [[ -z $suffix || "$suffix" == "compressme.*" ]] && \ |
530 |
+ suffix=$(echo compressme*) |
531 |
+ suffix=${suffix#compressme} |
532 |
+ cd / |
533 |
+ rm -rf "${tmpdir}" |
534 |
+ echo "${suffix}" |
535 |
+} |
536 |
+ |
537 |
+# figure out the new suffix |
538 |
+export PORTAGE_COMPRESS_SUFFIX=$(guess_suffix) || die |
539 |
+ |
540 |
+fix_symlinks() { |
541 |
+ # Repeat until nothing changes, in order to handle multiple |
542 |
+ # levels of indirection (see bug #470916). |
543 |
+ local -i indirection=0 |
544 |
+ while true ; do |
545 |
+ local something_changed= |
546 |
+ while read -r -d $'\0' brokenlink ; do |
547 |
+ [[ -e ${brokenlink} ]] && continue |
548 |
+ olddest=$(readlink "${brokenlink}") |
549 |
+ newdest=${olddest}${PORTAGE_COMPRESS_SUFFIX} |
550 |
+ if [[ "${newdest}" == /* ]] ; then |
551 |
+ [[ -f "${D%/}${newdest}" ]] || continue |
552 |
+ else |
553 |
+ [[ -f "${dir}/${brokenlink%/*}/${newdest}" ]] || continue |
554 |
+ fi |
555 |
+ something_changed=${brokenlink} |
556 |
+ rm -f "${brokenlink}" && |
557 |
+ ln -snf "${newdest}" "${brokenlink}${PORTAGE_COMPRESS_SUFFIX}" |
558 |
+ ((ret|=$?)) |
559 |
+ done < <(find "${ED}" -type l -print0 || die) |
560 |
+ |
561 |
+ [[ -n ${something_changed} ]] || break |
562 |
+ (( indirection++ )) |
563 |
+ if (( indirection >= 100 )) ; then |
564 |
+ # Protect against possibility of a bug triggering an endless loop. |
565 |
+ eerror "ecompress: too many levels of indirection for" \ |
566 |
+ "'${actual_dir#${ED%/}}/${something_changed#./}'" |
567 |
+ break |
568 |
+ fi |
569 |
+ done |
570 |
+ |
571 |
+ return ${ret} |
572 |
+} |
573 |
+ |
574 |
+export PORTAGE_COMPRESS PORTAGE_COMPRESS_FLAGS |
575 |
+find "${ED}" -name '*.ecompress' -delete \ |
576 |
+ -exec "${PORTAGE_BIN_PATH}"/ecompress-file {} + |
577 |
+ret=${?} |
578 |
+ |
579 |
+fix_symlinks |
580 |
+: $(( ret |= ${?} )) |
581 |
+[[ $ret -ne 0 ]] && __helpers_die "${0##*/} failed" |
582 |
+ |
583 |
+exit ${ret} |
584 |
diff --git a/bin/ecompress-file b/bin/ecompress-file |
585 |
new file mode 100755 |
586 |
index 000000000..8d6601a91 |
587 |
--- /dev/null |
588 |
+++ b/bin/ecompress-file |
589 |
@@ -0,0 +1,67 @@ |
590 |
+#!/bin/bash |
591 |
+# Copyright 1999-2018 Gentoo Foundation |
592 |
+# Distributed under the terms of the GNU General Public License v2 |
593 |
+ |
594 |
+source "${PORTAGE_BIN_PATH}"/isolated-functions.sh || exit 1 |
595 |
+ |
596 |
+compress_file() { |
597 |
+ mask_ext_re="" |
598 |
+ set -f |
599 |
+ for x in $PORTAGE_COMPRESS_EXCLUDE_SUFFIXES ; do |
600 |
+ mask_ext_re+="|$x" |
601 |
+ done |
602 |
+ set +f |
603 |
+ mask_ext_re="^(${mask_ext_re:1})\$" |
604 |
+ local filtered_args=() |
605 |
+ local had_precompressed= |
606 |
+ for x in "$@" ; do |
607 |
+ [[ ${x##*.} =~ $mask_ext_re ]] && continue |
608 |
+ [[ -s ${x} ]] || continue |
609 |
+ |
610 |
+ # handle precompressed files |
611 |
+ case ${x} in |
612 |
+ *.gz|*.Z) |
613 |
+ gunzip -f "${x}" || __helpers_die "gunzip failed" |
614 |
+ x=${x%.*} |
615 |
+ had_precompressed=1;; |
616 |
+ *.bz2) |
617 |
+ bunzip2 -f "${x}" || __helpers_die "bunzip2 failed" |
618 |
+ x=${x%.bz2} |
619 |
+ had_precompressed=1;; |
620 |
+ *.lzma|*.xz) |
621 |
+ unxz -f "${x}" || __helpers_die "unxz failed" |
622 |
+ x=${x%.*} |
623 |
+ had_precompressed=1;; |
624 |
+ esac |
625 |
+ |
626 |
+ filtered_args+=( "$x" ) |
627 |
+ done |
628 |
+ [[ -z ${filtered_args[@]} ]] && return 0 |
629 |
+ set -- "${filtered_args[@]}" |
630 |
+ |
631 |
+ if [[ ${had_precompressed} ]]; then |
632 |
+ eqawarn "One or more compressed files were found in docompress-ed directories." |
633 |
+ eqawarn "Please fix the ebuild not to install compressed files (manpages," |
634 |
+ eqawarn "documentation) when automatic compression is used." |
635 |
+ fi |
636 |
+ |
637 |
+ # If a compressed version of the file already exists, simply |
638 |
+ # delete it so that the compressor doesn't whine (bzip2 will |
639 |
+ # complain and skip, gzip will prompt for input) |
640 |
+ [[ -n ${PORTAGE_COMPRESS_SUFFIX} ]] && echo -n "${@/%/${PORTAGE_COMPRESS_SUFFIX}$'\001'}" | \ |
641 |
+ tr '\001' '\000' | ${XARGS} -0 rm -f |
642 |
+ |
643 |
+ # forcibly break all hard links as some compressors whine about it |
644 |
+ while IFS= read -d '' -r x ; do |
645 |
+ cp -p "${x}" "${x}.ecompress.break" || die |
646 |
+ mv -f "${x}.ecompress.break" "${x}" || die |
647 |
+ done < <(find "${@}" -type f -links +1 -print0) |
648 |
+ |
649 |
+ # Finally, let's actually do some real work |
650 |
+ "${PORTAGE_COMPRESS}" ${PORTAGE_COMPRESS_FLAGS} "$@" |
651 |
+ ret=$? |
652 |
+ [[ $ret -ne 0 ]] && __helpers_die "${0##*/} failed" |
653 |
+ return ${ret} |
654 |
+} |
655 |
+ |
656 |
+compress_file "${@%.ecompress}" |
657 |
diff --git a/bin/misc-functions.sh b/bin/misc-functions.sh |
658 |
index ed66e90ca..8c88f8f35 100755 |
659 |
--- a/bin/misc-functions.sh |
660 |
+++ b/bin/misc-functions.sh |
661 |
@@ -43,125 +43,6 @@ install_symlink_html_docs() { |
662 |
fi |
663 |
} |
664 |
|
665 |
-# replacement for "readlink -f" or "realpath" |
666 |
-READLINK_F_WORKS="" |
667 |
-canonicalize() { |
668 |
- if [[ -z ${READLINK_F_WORKS} ]] ; then |
669 |
- if [[ $(readlink -f -- /../ 2>/dev/null) == "/" ]] ; then |
670 |
- READLINK_F_WORKS=true |
671 |
- else |
672 |
- READLINK_F_WORKS=false |
673 |
- fi |
674 |
- fi |
675 |
- if ${READLINK_F_WORKS} ; then |
676 |
- readlink -f -- "$@" |
677 |
- return |
678 |
- fi |
679 |
- |
680 |
- local f=$1 b n=10 wd=$(pwd) |
681 |
- while (( n-- > 0 )); do |
682 |
- while [[ ${f: -1} = / && ${#f} -gt 1 ]]; do |
683 |
- f=${f%/} |
684 |
- done |
685 |
- b=${f##*/} |
686 |
- cd "${f%"${b}"}" 2>/dev/null || break |
687 |
- if [[ ! -L ${b} ]]; then |
688 |
- f=$(pwd -P) |
689 |
- echo "${f%/}/${b}" |
690 |
- cd "${wd}" |
691 |
- return 0 |
692 |
- fi |
693 |
- f=$(readlink "${b}") |
694 |
- done |
695 |
- cd "${wd}" |
696 |
- return 1 |
697 |
-} |
698 |
- |
699 |
-prepcompress() { |
700 |
- local -a include exclude incl_d incl_f |
701 |
- local f g i real_f real_d |
702 |
- if ! ___eapi_has_prefix_variables; then |
703 |
- local ED=${D} |
704 |
- fi |
705 |
- |
706 |
- # Canonicalize path names and check for their existence. |
707 |
- real_d=$(canonicalize "${ED}") |
708 |
- for (( i = 0; i < ${#PORTAGE_DOCOMPRESS[@]}; i++ )); do |
709 |
- real_f=$(canonicalize "${ED%/}/${PORTAGE_DOCOMPRESS[i]#/}") |
710 |
- f=${real_f#"${real_d}"} |
711 |
- if [[ ${real_f} != "${f}" ]] && [[ -d ${real_f} || -f ${real_f} ]] |
712 |
- then |
713 |
- include[${#include[@]}]=${f:-/} |
714 |
- elif [[ ${i} -ge 3 ]]; then |
715 |
- ewarn "prepcompress:" \ |
716 |
- "ignoring nonexistent path '${PORTAGE_DOCOMPRESS[i]}'" |
717 |
- fi |
718 |
- done |
719 |
- for (( i = 0; i < ${#PORTAGE_DOCOMPRESS_SKIP[@]}; i++ )); do |
720 |
- real_f=$(canonicalize "${ED%/}/${PORTAGE_DOCOMPRESS_SKIP[i]#/}") |
721 |
- f=${real_f#"${real_d}"} |
722 |
- if [[ ${real_f} != "${f}" ]] && [[ -d ${real_f} || -f ${real_f} ]] |
723 |
- then |
724 |
- exclude[${#exclude[@]}]=${f:-/} |
725 |
- elif [[ ${i} -ge 1 ]]; then |
726 |
- ewarn "prepcompress:" \ |
727 |
- "ignoring nonexistent path '${PORTAGE_DOCOMPRESS_SKIP[i]}'" |
728 |
- fi |
729 |
- done |
730 |
- |
731 |
- # Remove redundant entries from lists. |
732 |
- # For the include list, remove any entries that are: |
733 |
- # a) contained in a directory in the include or exclude lists, or |
734 |
- # b) identical with an entry in the exclude list. |
735 |
- for (( i = ${#include[@]} - 1; i >= 0; i-- )); do |
736 |
- f=${include[i]} |
737 |
- for g in "${include[@]}"; do |
738 |
- if [[ ${f} == "${g%/}"/* ]]; then |
739 |
- unset include[i] |
740 |
- continue 2 |
741 |
- fi |
742 |
- done |
743 |
- for g in "${exclude[@]}"; do |
744 |
- if [[ ${f} = "${g}" || ${f} == "${g%/}"/* ]]; then |
745 |
- unset include[i] |
746 |
- continue 2 |
747 |
- fi |
748 |
- done |
749 |
- done |
750 |
- # For the exclude list, remove any entries that are: |
751 |
- # a) contained in a directory in the exclude list, or |
752 |
- # b) _not_ contained in a directory in the include list. |
753 |
- for (( i = ${#exclude[@]} - 1; i >= 0; i-- )); do |
754 |
- f=${exclude[i]} |
755 |
- for g in "${exclude[@]}"; do |
756 |
- if [[ ${f} == "${g%/}"/* ]]; then |
757 |
- unset exclude[i] |
758 |
- continue 2 |
759 |
- fi |
760 |
- done |
761 |
- for g in "${include[@]}"; do |
762 |
- [[ ${f} == "${g%/}"/* ]] && continue 2 |
763 |
- done |
764 |
- unset exclude[i] |
765 |
- done |
766 |
- |
767 |
- # Split the include list into directories and files |
768 |
- for f in "${include[@]}"; do |
769 |
- if [[ -d ${ED%/}/${f#/} ]]; then |
770 |
- incl_d[${#incl_d[@]}]=${f} |
771 |
- else |
772 |
- incl_f[${#incl_f[@]}]=${f} |
773 |
- fi |
774 |
- done |
775 |
- |
776 |
- # Queue up for compression. |
777 |
- # ecompress{,dir} doesn't like to be called with empty argument lists. |
778 |
- [[ ${#incl_d[@]} -gt 0 ]] && ecompressdir --limit ${PORTAGE_DOCOMPRESS_SIZE_LIMIT:-0} --queue "${incl_d[@]}" |
779 |
- [[ ${#incl_f[@]} -gt 0 ]] && ecompress --queue "${incl_f[@]/#/${ED%/}}" |
780 |
- [[ ${#exclude[@]} -gt 0 ]] && ecompressdir --ignore "${exclude[@]}" |
781 |
- return 0 |
782 |
-} |
783 |
- |
784 |
__prepall() { |
785 |
if has chflags $FEATURES ; then |
786 |
# Save all the file flags for restoration at the end of prepall. |
787 |
@@ -173,6 +54,11 @@ __prepall() { |
788 |
|
789 |
[[ -d ${ED%/}/usr/share/info ]] && prepinfo |
790 |
|
791 |
+ # Apply compression. |
792 |
+ "${PORTAGE_BIN_PATH}"/ecompress --queue "${PORTAGE_DOCOMPRESS[@]}" |
793 |
+ "${PORTAGE_BIN_PATH}"/ecompress --ignore "${PORTAGE_DOCOMPRESS_SKIP[@]}" |
794 |
+ "${PORTAGE_BIN_PATH}"/ecompress --dequeue |
795 |
+ |
796 |
___eapi_has_dostrip || prepallstrip |
797 |
|
798 |
if has chflags $FEATURES ; then |
799 |
@@ -239,9 +125,6 @@ install_qa_check() { |
800 |
|
801 |
export STRIP_MASK |
802 |
__prepall |
803 |
- prepcompress |
804 |
- ecompressdir --dequeue |
805 |
- ecompress --dequeue |
806 |
|
807 |
if ___eapi_has_dostrip; then |
808 |
"${PORTAGE_BIN_PATH}"/estrip --queue "${PORTAGE_DOSTRIP[@]}" |
809 |
diff --git a/lib/portage/tests/bin/setup_env.py b/lib/portage/tests/bin/setup_env.py |
810 |
index 9cc26df08..33b167989 100644 |
811 |
--- a/lib/portage/tests/bin/setup_env.py |
812 |
+++ b/lib/portage/tests/bin/setup_env.py |
813 |
@@ -82,6 +82,6 @@ for bin in os.listdir(os.path.join(bindir, 'ebuild-helpers')): |
814 |
if bin.startswith('do') or \ |
815 |
bin.startswith('new') or \ |
816 |
bin.startswith('prep') or \ |
817 |
- bin in ('ecompress', 'ecompressdir', 'fowners', 'fperms'): |
818 |
+ bin in ('fowners', 'fperms'): |
819 |
globals()[bin] = create_portage_wrapper( |
820 |
os.path.join(bindir, 'ebuild-helpers', bin)) |
821 |
-- |
822 |
2.19.0 |