Gentoo Archives: gentoo-commits

From: Thomas Deutschmann <whissi@g.o>
To: gentoo-commits@l.g.o
Subject: [gentoo-commits] proj/genkernel:master commit in: /, doc/, defaults/
Date: Thu, 16 Jul 2020 15:03:32
Message-Id: 1594909755.944da1fc61bd64a47c2bc6415da5e5d5611b6ff5.whissi@gentoo
1 commit: 944da1fc61bd64a47c2bc6415da5e5d5611b6ff5
2 Author: Thomas Deutschmann <whissi <AT> gentoo <DOT> org>
3 AuthorDate: Tue Jul 14 14:17:30 2020 +0000
4 Commit: Thomas Deutschmann <whissi <AT> gentoo <DOT> org>
5 CommitDate: Thu Jul 16 14:29:15 2020 +0000
6 URL: https://gitweb.gentoo.org/proj/genkernel.git/commit/?id=944da1fc
7
8 gen_initramfs.sh: create_initramfs(): Move and unify validation of --compress-initramfs-type to determine_real_args()
9
10 This will allow us to error out early if required user space tool
11 needed to compress initramfs based on specified --compress-initramfs-type
12 is missing or selected type is invalid/unsupported.
13
14 Best/fastest list is based on results from [Link1][Link2][Link3].
15
16 Link1: https://events.static.linuxfound.org/sites/events/files/lcjpcojp13_klee.pdf
17 Link2: https://kernel.ubuntu.com/~cking/boot-speed-eoan-5.3/kernel-compression-method.txt
18 Link3: https://lwn.net/Articles/817134/
19 Signed-off-by: Thomas Deutschmann <whissi <AT> gentoo.org>
20
21 defaults/compression_methods.sh | 37 +++++++++++++++++
22 doc/genkernel.8.txt | 10 +++--
23 gen_cmdline.sh | 3 +-
24 gen_determineargs.sh | 91 +++++++++++++++++++++++++++++++++++++++++
25 gen_funcs.sh | 24 +++++++++++
26 gen_initramfs.sh | 67 +++---------------------------
27 genkernel | 1 +
28 7 files changed, 168 insertions(+), 65 deletions(-)
29
30 diff --git a/defaults/compression_methods.sh b/defaults/compression_methods.sh
31 new file mode 100644
32 index 0000000..41ee9c4
33 --- /dev/null
34 +++ b/defaults/compression_methods.sh
35 @@ -0,0 +1,37 @@
36 +# NOTE:
37 +# - This file (compression_methods.sh) is sourced by genkernel.
38 +# Rather than changing this very file, please override specific variables
39 +# somewhere in /etc/genkernel.conf .
40 +#
41 +# - This file should not override previously defined variables, as their values may
42 +# originate from user changes to /etc/genkernel.conf .
43 +
44 +GKICM_BZ2_KOPTNAME="BZIP2"
45 +GKICM_BZ2_CMD="bzip2 -z -f -9"
46 +GKICM_BZ2_EXT=".bz2"
47 +GKICM_BZ2_PKG="app-arch/bzip2"
48 +
49 +GKICM_GZ_KOPTNAME="GZIP"
50 +GKICM_GZ_CMD="gzip -f -9"
51 +GKICM_GZ_EXT=".gz"
52 +GKICM_GZ_PKG="app-arch/gzip"
53 +
54 +GKICM_LZO_KOPTNAME="LZO"
55 +GKICM_LZO_CMD="lzop -f -9"
56 +GKICM_LZO_EXT=".lzo"
57 +GKICM_LZO_PKG="app-arch/lzop"
58 +
59 +GKICM_LZ4_KOPTNAME="LZ4"
60 +GKICM_LZ4_CMD="lz4 -f -9 -l -q"
61 +GKICM_LZ4_EXT=".lz4"
62 +GKICM_LZ4_PKG="app-arch/lz4"
63 +
64 +GKICM_LZMA_KOPTNAME="LZMA"
65 +GKICM_LZMA_CMD="lzma -z -f -9"
66 +GKICM_LZMA_EXT=".lzma"
67 +GKICM_LZMA_PKG="app-arch/xz-utils"
68 +
69 +GKICM_XZ_KOPTNAME="XZ"
70 +GKICM_XZ_CMD="xz -e --check=none -z -f -9"
71 +GKICM_XZ_EXT=".xz"
72 +GKICM_XZ_PKG="app-arch/xz-utils"
73
74 diff --git a/doc/genkernel.8.txt b/doc/genkernel.8.txt
75 index 494a29c..40c591e 100644
76 --- a/doc/genkernel.8.txt
77 +++ b/doc/genkernel.8.txt
78 @@ -598,9 +598,13 @@ NOTE: System.map filename and System.map symlink name must be different.
79 Deprecated alias for *--*[*no-*]*compress-initramfs*.
80
81 *--compress-initramfs-type*=<arg>::
82 - Compression type for initramfs (best, xz, lzma, bzip2, gzip, lzop, lz4).
83 - "Best" causes selection of the best available algorithm that is selected
84 - in your kernel configuration.
85 + Compression type for initramfs (best, bzip2, fastest, gzip, lz4, lzma, lza, xz).
86 ++
87 +*best* will select the algorithm providing best compression
88 +from those selected in your kernel configuration.
89 ++
90 +*fastest* will select the algorithm providing fastest decompression
91 +from those selected in your kernel configuration.
92
93 *--strip*=<all|kernel|modules|none>::
94 Strip debug symbols from none, all, installed kernel (obsolete) or
95
96 diff --git a/gen_cmdline.sh b/gen_cmdline.sh
97 index 9e59ed2..1daee5c 100755
98 --- a/gen_cmdline.sh
99 +++ b/gen_cmdline.sh
100 @@ -243,7 +243,8 @@ longusage() {
101 echo " --compress-initrd Deprecated alias for --compress-initramfs"
102 echo " --no-compress-initrd Deprecated alias for --no-compress-initramfs"
103 echo " --compress-initramfs-type=<arg>"
104 - echo " Compression type for initramfs (best, xz, lzma, bzip2, gzip, lzop, lz4)"
105 + echo " Compression type for initramfs (best, bzip2, fastest, gzip, lz4,"
106 + echo " lzma, lza, xz)"
107 echo " --strip=(all|kernel|modules|none)"
108 echo " Strip debug symbols from none, all, installed kernel (obsolete) or"
109 echo " modules (default)"
110
111 diff --git a/gen_determineargs.sh b/gen_determineargs.sh
112 index 0cbba36..406d1d5 100755
113 --- a/gen_determineargs.sh
114 +++ b/gen_determineargs.sh
115 @@ -619,6 +619,73 @@ determine_real_args() {
116 done
117 unset v pn pn_varname pkg_prefixes
118
119 + declare -gA GKICM_LOOKUP_TABLE_CMD=()
120 + declare -gA GKICM_LOOKUP_TABLE_EXT=()
121 + declare -gA GKICM_LOOKUP_TABLE_PKG=()
122 + local known_initramfs_compression_methods_by_compression=( $(get_initramfs_compression_method_by_compression) )
123 + local known_initramfs_compression_methods_by_speed=( $(get_initramfs_compression_method_by_speed) )
124 + local initramfs_compression_methods=( $(compgen -A variable |grep '^GKICM_.*_KOPTNAME$') )
125 + local initramfs_compression_method key var_name var_prefix
126 + for initramfs_compression_method in "${initramfs_compression_methods[@]}"
127 + do
128 + if [ -z "${!initramfs_compression_method}" ]
129 + then
130 + gen_die "Invalid config found: Check value of '${initramfs_compression_method}'!"
131 + fi
132 +
133 + if [[ "${known_initramfs_compression_methods_by_compression[@]} " != *"${!initramfs_compression_method}"* ]]
134 + then
135 + gen_die "Internal error: Initramfs compression method '${!initramfs_compression_method}' was not added to get_initramfs_compression_method_by_compression()!"
136 + else
137 + known_initramfs_compression_methods_by_compression=( $(printf '%s\n' "${known_initramfs_compression_methods_by_compression[@]//${!initramfs_compression_method}/}") )
138 + fi
139 +
140 + if [[ "${known_initramfs_compression_methods_by_speed[@]} " != *"${!initramfs_compression_method}"* ]]
141 + then
142 + gen_die "Internal error: Initramfs compression method '${!initramfs_compression_method}' was not added to get_initramfs_compression_method_by_speed()!"
143 + else
144 + known_initramfs_compression_methods_by_speed=( $(printf '%s\n' "${known_initramfs_compression_methods_by_speed[@]//${!initramfs_compression_method}/}") )
145 + fi
146 +
147 + var_prefix="${initramfs_compression_method%_KOPTNAME}"
148 +
149 + for key in CMD EXT PKG
150 + do
151 + var_name="${var_prefix}_${key}"
152 + if [ -z "${!var_name}" ]
153 + then
154 + gen_die "Internal error: Variable '${var_name}' is not set!"
155 + fi
156 +
157 + case ${key} in
158 + CMD)
159 + GKICM_LOOKUP_TABLE_CMD[${!initramfs_compression_method}]="${!var_name}"
160 + ;;
161 + EXT)
162 + GKICM_LOOKUP_TABLE_EXT[${!initramfs_compression_method}]="${!var_name}"
163 + ;;
164 + PKG)
165 + GKICM_LOOKUP_TABLE_PKG[${!initramfs_compression_method}]="${!var_name}"
166 + ;;
167 + esac
168 + done
169 + done
170 + unset initramfs_compression_methods initramfs_compression_method key var_name var_prefix
171 +
172 + # It is enough to check just one data set because we validated
173 + # both data sets above.
174 + if [[ ${#known_initramfs_compression_methods_by_compression[@]} -gt 0 ]]
175 + then
176 + local unhandled_method
177 + for unhandled_method in "${known_initramfs_compression_methods_by_compression[@]}"
178 + do
179 + print_error 1 "Do not know how to handle initramfs compression type '${unhandled_method}'!"
180 + done
181 +
182 + gen_die "Internal error: Not all known initramfs compression methods are defined!"
183 + fi
184 + unset known_initramfs_compression_methods_by_compression known_initramfs_compression_methods_by_speed
185 +
186 if [ -n "${CMD_BOOTLOADER}" ]
187 then
188 BOOTLOADER="${CMD_BOOTLOADER}"
189 @@ -1009,6 +1076,30 @@ determine_real_args() {
190 gen_die "'\"${LDDTREE_COMMAND}\" -l \"${CPIO_COMMAND}\"' failed -- cannot generate initramfs without working lddtree!"
191 fi
192
193 + if isTrue "${COMPRESS_INITRD}"
194 + then
195 + local pattern_auto='^(BEST|FASTEST)$'
196 + local pattern_manual="$(get_initramfs_compression_method_by_speed)"
197 + pattern_manual=${pattern_manual// /|}
198 + pattern_manual="^(${pattern_manual})$"
199 +
200 + if [[ "${COMPRESS_INITRD_TYPE^^}" =~ ${pattern_auto} ]]
201 + then
202 + # Will be handled in set_initramfs_compression_method()
203 + :;
204 + elif [[ ! "${COMPRESS_INITRD_TYPE^^}" =~ ${pattern_manual} ]]
205 + then
206 + gen_die "Specified --compress-initramfs-type '${COMPRESS_INITRD_TYPE}' is unknown"
207 + elif ! hash ${GKICM_LOOKUP_TABLE_CMD[${COMPRESS_INITRD_TYPE^^}]/%\ */} &>/dev/null
208 + then
209 + gen_die "'${GKICM_LOOKUP_TABLE_CMD[${COMPRESS_INITRD_TYPE^^}]/%\ */}', the tool to compress initramfs based on selected --compress-initramfs-type was not found. Is ${GKICM_LOOKUP_TABLE_PKG[${COMPRESS_INITRD_TYPE^^}]} installed?"
210 + fi
211 + unset pattern_auto pattern_manual
212 +
213 + # Ensure that value matches keys in GKICM_* arrays
214 + COMPRESS_INITRD_TYPE=${COMPRESS_INITRD_TYPE^^}
215 + fi
216 +
217 SANDBOX_COMMAND=
218 if isTrue "${SANDBOX}"
219 then
220
221 diff --git a/gen_funcs.sh b/gen_funcs.sh
222 index 8c6f412..e681c1e 100755
223 --- a/gen_funcs.sh
224 +++ b/gen_funcs.sh
225 @@ -464,6 +464,30 @@ get_indent() {
226 echo "${_indent}"
227 }
228
229 +get_initramfs_compression_method_by_compression() {
230 + local -a methods=()
231 + methods+=( XZ )
232 + methods+=( LZMA )
233 + methods+=( GZIP )
234 + methods+=( BZIP2 )
235 + methods+=( LZO )
236 + methods+=( LZ4 )
237 +
238 + echo "${methods[@]}"
239 +}
240 +
241 +get_initramfs_compression_method_by_speed() {
242 + local -a methods=()
243 + methods+=( LZ4 )
244 + methods+=( LZO )
245 + methods+=( GZIP )
246 + methods+=( BZIP2 )
247 + methods+=( LZMA )
248 + methods+=( XZ )
249 +
250 + echo "${methods[@]}"
251 +}
252 +
253 setup_cache_dir() {
254 if [ ! -d "${GK_V_CACHEDIR}" ]
255 then
256
257 diff --git a/gen_initramfs.sh b/gen_initramfs.sh
258 index fd9583b..617c656 100755
259 --- a/gen_initramfs.sh
260 +++ b/gen_initramfs.sh
261 @@ -1770,7 +1770,6 @@ append_data() {
262 }
263
264 create_initramfs() {
265 - local compress_ext=""
266 print_info 1 "initramfs: >> Initializing ..."
267
268 # Create empty cpio
269 @@ -1894,6 +1893,8 @@ create_initramfs() {
270 print_info 1 "$(get_indent 1)>> --integrated-initramfs is set; Setting CONFIG_INITRAMFS_* options ..."
271
272 [ -f "${KCONFIG_MODIFIED_MARKER}" ] && rm "${KCONFIG_MODIFIED_MARKER}"
273 + [ -f "${KCONFIG_REQUIRED_OPTIONS}" ] && rm "${KCONFIG_REQUIRED_OPTIONS}"
274 +
275 kconfig_set_opt "${KERNEL_OUTPUTDIR}/.config" "CONFIG_BLK_DEV_INITRD" "y"
276 kconfig_set_opt "${KERNEL_OUTPUTDIR}/.config" "CONFIG_INITRAMFS_SOURCE" "\"${CPIO_ARCHIVE}.cpio\""
277 kconfig_set_opt "${KERNEL_OUTPUTDIR}/.config" "CONFIG_INITRAMFS_ROOT_UID" "0"
278 @@ -1911,66 +1912,10 @@ create_initramfs() {
279 else
280 if isTrue "${COMPRESS_INITRD}"
281 then
282 - local cmd_xz=$(type -p xz)
283 - local cmd_lzma=$(type -p lzma)
284 - local cmd_bzip2=$(type -p bzip2)
285 - local cmd_gzip=$(type -p gzip)
286 - local cmd_lzop=$(type -p lzop)
287 - local cmd_lz4=$(type -p lz4)
288 - local pkg_xz='app-arch/xz-utils'
289 - local pkg_lzma='app-arch/xz-utils'
290 - local pkg_bzip2='app-arch/bzip2'
291 - local pkg_gzip='app-arch/gzip'
292 - local pkg_lzop='app-arch/lzop'
293 - local pkg_lz4='app-arch/lz4'
294 - local compression
295 - case ${COMPRESS_INITRD_TYPE} in
296 - xz|lzma|bzip2|gzip|lzop|lz4) compression=${COMPRESS_INITRD_TYPE} ;;
297 - lzo) compression=lzop ;;
298 - best|fastest)
299 - local tuple
300 - for tuple in \
301 - 'CONFIG_RD_XZ cmd_xz xz' \
302 - 'CONFIG_RD_LZMA cmd_lzma lzma' \
303 - 'CONFIG_RD_BZIP2 cmd_bzip2 bzip2' \
304 - 'CONFIG_RD_GZIP cmd_gzip gzip' \
305 - 'CONFIG_RD_LZO cmd_lzop lzop' \
306 - 'CONFIG_RD_LZ4 cmd_lz4 lz4' \
307 - ; do
308 - set -- ${tuple}
309 - local kernel_option=${1}
310 - local cmd_variable_name=${2}
311 - if ${CONFGREP} -q "^${kernel_option}=y" "${ACTUAL_KERNEL_CONFIG}" && test -n "${!cmd_variable_name}" ; then
312 - compression=${3}
313 - [[ ${COMPRESS_INITRD_TYPE} == best ]] && break
314 - fi
315 - done
316 - [[ -z "${compression}" ]] && gen_die "None of the initramfs compression methods we tried are supported by your kernel (config file \"${ACTUAL_KERNEL_CONFIG}\"), strange!?"
317 - ;;
318 - *)
319 - gen_die "Compression '${COMPRESS_INITRD_TYPE}' unknown"
320 - ;;
321 - esac
322 -
323 - # Check for actual availability
324 - local cmd_variable_name=cmd_${compression}
325 - local pkg_variable_name=pkg_${compression}
326 - [[ -z "${!cmd_variable_name}" ]] && gen_die "Compression '${compression}' is not available. Please install package '${!pkg_variable_name}'."
327 -
328 - local compress_ext compress_cmd
329 - case ${compression} in
330 - xz) compress_ext='.xz' compress_cmd="${cmd_xz} -e --check=none -z -f -9" ;;
331 - lzma) compress_ext='.lzma' compress_cmd="${cmd_lzma} -z -f -9" ;;
332 - bzip2) compress_ext='.bz2' compress_cmd="${cmd_bzip2} -z -f -9" ;;
333 - gzip) compress_ext='.gz' compress_cmd="${cmd_gzip} -f -9" ;;
334 - lzop) compress_ext='.lzo' compress_cmd="${cmd_lzop} -f -9" ;;
335 - lz4) compress_ext='.lz4' compress_cmd="${cmd_lz4} -f -9 -l -q" ;;
336 - esac
337 -
338 - print_info 1 "$(get_indent 1)>> Compressing cpio data (${compress_ext}) ..."
339 - print_info 3 "COMMAND: ${compress_cmd} ${CPIO_ARCHIVE}" 1 0 1
340 - ${compress_cmd} "${CPIO_ARCHIVE}" || gen_die "Compression (${compress_cmd}) failed"
341 - mv -f "${CPIO_ARCHIVE}${compress_ext}" "${CPIO_ARCHIVE}" || gen_die "Rename failed"
342 + print_info 1 "$(get_indent 1)>> Compressing cpio data (${GKICM_LOOKUP_TABLE_EXT[${COMPRESS_INITRD_TYPE}]}) ..."
343 + print_info 3 "COMMAND: ${GKICM_LOOKUP_TABLE_CMD[${COMPRESS_INITRD_TYPE}]} ${CPIO_ARCHIVE}" 1 0 1
344 + ${GKICM_LOOKUP_TABLE_CMD[${COMPRESS_INITRD_TYPE}]} "${CPIO_ARCHIVE}" || gen_die "Initramfs compression using '${GKICM_LOOKUP_TABLE_CMD[${COMPRESS_INITRD_TYPE}]}' failed"
345 + mv -f "${CPIO_ARCHIVE}${GKICM_LOOKUP_TABLE_EXT[${COMPRESS_INITRD_TYPE}]}" "${CPIO_ARCHIVE}" || gen_die "Rename failed"
346 else
347 print_info 3 "$(get_indent 1)>> --no-compress-initramfs is set; Skipping compression of initramfs ..."
348 fi
349
350 diff --git a/genkernel b/genkernel
351 index 0856e25..c0fd96d 100755
352 --- a/genkernel
353 +++ b/genkernel
354 @@ -69,6 +69,7 @@ source "${_GENKERNEL_CONF}" || small_die "Could not read ${_GENKERNEL_CONF}"
355 LOGLEVEL=${LOGLEVEL:-1}
356
357 # Start sourcing other scripts
358 +source "${GK_SHARE}"/defaults/compression_methods.sh || small_die "Could not read '${GK_SHARE}/defaults/compression_methods.sh'"
359 source "${GK_SHARE}"/defaults/software.sh || small_die "Could not read '${GK_SHARE}/defaults/software.sh'"
360 source "${GK_SHARE}"/defaults/config.sh || small_die "Could not read '${GK_SHARE}/defaults/config.sh'"
361 source "${GK_SHARE}"/gen_funcs.sh || small_die "Could not read '${GK_SHARE}/gen_funcs.sh'"