Gentoo Archives: gentoo-portage-dev

From: Zac Medico <zmedico@g.o>
To: gentoo-portage-dev@l.g.o
Cc: Hidehiko Abe <hidehiko@××××××××.org>
Subject: [gentoo-portage-dev] [PATCH] Rewrite doins in python (bug 624526)
Date: Mon, 14 Aug 2017 07:39:53
Message-Id: 20170814073921.20837-1-zmedico@gentoo.org
1 From: Hidehiko Abe <hidehiko@××××××××.org>
2
3 doins is written in bash. However, specifically in case that
4 too many files are installed, it is very slow.
5 This CL rewrites the script in python for performance.
6
7 BUG=chromium:712659
8 TEST=time (./setup_board --forace && \
9 ./build_package --withdev && \
10 ./build_image --noenable_rootfs_verification test)
11 ===Before===
12 real 21m35.445s
13 user 93m40.588s
14 sys 21m31.224s
15
16 ===After===
17 real 17m30.106s
18 user 94m1.812s
19 sys 20m13.468s
20
21 Change-Id: Ib10f623961ba316753d58397cff5e72fbc343339
22 Reviewed-on: https://chromium-review.googlesource.com/559225
23 X-Chromium-Bug: 712659
24 X-Chromium-Bug-url: https://bugs.chromium.org/p/chromium/issues/detail?id=712659
25 X-Gentoo-Bug: 624526
26 X-Gentoo-Bug-url: https://bugs.gentoo.org/624526
27 ---
28 bin/doins.py | 341 +++++++++++++++++++++++++++++++++++++++++++++++
29 bin/ebuild-helpers/doins | 123 ++---------------
30 2 files changed, 353 insertions(+), 111 deletions(-)
31 create mode 100644 bin/doins.py
32
33 diff --git a/bin/doins.py b/bin/doins.py
34 new file mode 100644
35 index 000000000..4a17287ca
36 --- /dev/null
37 +++ b/bin/doins.py
38 @@ -0,0 +1,341 @@
39 +#!/usr/bin/python -b
40 +# Copyright 2017 The Chromium OS Authors. All rights reserved.
41 +# Use of this source code is governed by a BSD-style license that can be
42 +# found in the LICENSE file.
43 +
44 +from __future__ import print_function
45 +
46 +import argparse
47 +import errno
48 +import grp
49 +import logging
50 +import os
51 +import pwd
52 +import re
53 +import shlex
54 +import shutil
55 +import stat
56 +import subprocess
57 +import sys
58 +
59 +from portage.util import movefile
60 +
61 +# Change back to original cwd _after_ all imports (bug #469338).
62 +# See also bin/ebuild-helpers/doins, which sets the cwd.
63 +os.chdir(os.environ["__PORTAGE_HELPER_CWD"])
64 +
65 +_PORTAGE_ACTUAL_DISTDIR = (
66 + (os.environb if sys.version_info.major >= 3 else os.environ).get(
67 + b'PORTAGE_ACTUAL_DISTDIR', b'') + b'/')
68 +
69 +
70 +def _parse_install_options(
71 + options, inprocess_runner_class, subprocess_runner_class):
72 + """Parses command line arguments for install command."""
73 + parser = argparse.ArgumentParser()
74 + parser.add_argument('-g', '--group', default=-1, type=_parse_group)
75 + parser.add_argument('-o', '--owner', default=-1, type=_parse_user)
76 + # "install"'s --mode option is complicated. So here is partially
77 + # supported.
78 + parser.add_argument('-m', '--mode', type=_parse_mode)
79 + split_options = shlex.split(options)
80 + namespace, remaining = parser.parse_known_args(split_options)
81 + if remaining:
82 + print('Unknown install options: %s, %r' % (options, remaining),
83 + file=sys.stderr)
84 + if os.environ.get('DOINSSTRICTOPTION', '') == '1':
85 + sys.exit(1)
86 + print('Continue with falling back to \'install\' '
87 + 'command execution, which can be slower.',
88 + file=sys.stderr)
89 + return subprocess_runner_class(split_options)
90 + return inprocess_runner_class(namespace)
91 +
92 +
93 +def _parse_group(group):
94 + """Parses gid."""
95 + g = grp.getgrnam(group)
96 + if g:
97 + return g.gr_gid
98 + return int(group)
99 +
100 +
101 +def _parse_user(user):
102 + """Parses uid."""
103 + u = pwd.getpwnam(user)
104 + if u:
105 + return u.pw_uid
106 + return int(user)
107 +
108 +
109 +def _parse_mode(mode):
110 + # "install"'s --mode option is complicated. So here is partially
111 + # supported.
112 + # In Python 3, the prefix of octal int must be '0o' rather than '0'.
113 + # So, set base explicitly in that case.
114 + return int(mode, 8 if re.search(r'^0[0-7]*$', mode) else 0)
115 +
116 +
117 +def _set_attributes(options, path):
118 + """Sets attributes the file/dir at given |path|.
119 +
120 + Args:
121 + options: object which has |owner|, |group| and |mode| fields.
122 + |owner| is int value representing uid. Similary |group|
123 + represents gid.
124 + If -1 is set, just unchanged.
125 + |mode| is the bits of permissions.
126 + path: File/directory path.
127 + """
128 + if options.owner != -1 or options.group != -1:
129 + os.lchown(path, options.owner, options.group)
130 + if options.mode is not None:
131 + os.chmod(path, options.mode)
132 +
133 +
134 +class _InsInProcessInstallRunner(object):
135 + def __init__(self, parsed_options):
136 + self._parsed_options = parsed_options
137 + self._copy_xattr = (
138 + 'xattr' in os.environ.get('FEATURES', '').split())
139 + if self._copy_xattr:
140 + self._xattr_exclude = os.environ.get(
141 + 'PORTAGE_XATTR_EXCLUDE',
142 + 'security.* system.nfs4_acl')
143 +
144 + def run(self, source, dest_dir):
145 + """Installs a file at |source| into |dest_dir| in process."""
146 + dest = os.path.join(dest_dir, os.path.basename(source))
147 + try:
148 + # TODO: Consider to use portage.util.file_copy.copyfile
149 + # introduced by
150 + # https://gitweb.gentoo.org/proj/portage.git/commit/
151 + # ?id=8ab5c8835931fd9ec098dbf4c5f416eb32e4a3a4
152 + # after uprev.
153 + shutil.copyfile(source, dest)
154 + _set_attributes(self._parsed_options, dest)
155 + if self._copy_xattr:
156 + movefile._copyxattr(
157 + source, dest,
158 + exclude=self._xattr_exclude)
159 + except Exception:
160 + logging.exception(
161 + 'Failed to copy file: '
162 + '_parsed_options=%r, source=%r, dest_dir=%r',
163 + self._parsed_options, source, dest_dir)
164 + return False
165 + return True
166 +
167 +
168 +class _InsSubprocessInstallRunner(object):
169 + def __init__(self, split_options):
170 + self._split_options = split_options
171 +
172 + def run(self, source, dest_dir):
173 + """Installs a file at |source| into |dest_dir| by 'install'."""
174 + command = ['install'] + self._split_options + [source, dest_dir]
175 + return subprocess.call(command) == 0
176 +
177 +
178 +class _DirInProcessInstallRunner(object):
179 + def __init__(self, parsed_options):
180 + self._parsed_options = parsed_options
181 +
182 + def run(self, dest):
183 + """Installs a dir into |dest| in process."""
184 + try:
185 + os.makedirs(dest)
186 + except OSError as e:
187 + if e.errno != errno.EEXIST or not os.path.isdir(dest):
188 + raise
189 + _set_attributes(self._parsed_options, dest)
190 +
191 +
192 +class _DirSubprocessInstallRunner(object):
193 + """Runs real 'install' command to install files or directories."""
194 +
195 + def __init__(self, split_options):
196 + self._split_options = split_options
197 +
198 + def run(self, dest):
199 + """Installs a dir into |dest| by 'install' command."""
200 + command = ['install', '-d'] + self._split_options + [dest]
201 + subprocess.call(command)
202 +
203 +
204 +class _InstallRunner(object):
205 + def __init__(self):
206 + self._ins_runner = _parse_install_options(
207 + os.environ.get('INSOPTIONS', ''),
208 + _InsInProcessInstallRunner,
209 + _InsSubprocessInstallRunner)
210 + self._dir_runner = _parse_install_options(
211 + os.environ.get('DIROPTIONS', ''),
212 + _DirInProcessInstallRunner,
213 + _DirSubprocessInstallRunner)
214 +
215 + def install_file(self, source, dest_dir):
216 + """Installs a file at |source| into |dest_dir| directory."""
217 + return self._ins_runner.run(source, dest_dir)
218 +
219 + def install_dir(self, dest):
220 + """Creates a directory at |dest|."""
221 + self._dir_runner.run(dest)
222 +
223 +
224 +def _doins(args, install_runner, relpath, source_root):
225 + """Installs a file as if 'install' command runs.
226 +
227 + Installs a file at |source_root|/|relpath| into |args.dest|/|relpath|.
228 + If |args.preserve_symlinks| is set, creates symlink if the source is a
229 + symlink.
230 +
231 + Args:
232 + args: parsed arguments. It should have following fields.
233 + - preserve_symlinks: bool representing whether symlinks
234 + needs to be preserved.
235 + - dest: Destination root directory.
236 + - insoptions: options for 'install' command.
237 + intall_runner: _InstallRunner instance for file install.
238 + relpath: Relative path of the file being installed.
239 + source_root: Source root directory.
240 +
241 + Returns: True on success.
242 + """
243 + source = os.path.join(source_root, relpath)
244 + dest = os.path.join(args.dest, relpath)
245 + if os.path.islink(source):
246 + # Our fake $DISTDIR contains symlinks that should not be
247 + # reproduced inside $D. In order to ensure that things like
248 + # dodoc "$DISTDIR"/foo.pdf work as expected, we dereference
249 + # symlinked files that refer to absolute paths inside
250 + # $PORTAGE_ACTUAL_DISTDIR/.
251 + try:
252 + if (args.preserve_symlinks and
253 + not os.readlink(source).startswith(
254 + _PORTAGE_ACTUAL_DISTDIR)):
255 + linkto = os.readlink(source)
256 + shutil.rmtree(dest, ignore_errors=True)
257 + os.symlink(linkto, dest)
258 + return True
259 + except Exception:
260 + logging.exception(
261 + 'Failed to create symlink: '
262 + 'args=%r, relpath=%r, source_root=%r',
263 + args, relpath, source_root)
264 + return False
265 +
266 + return install_runner.install_file(source, os.path.dirname(dest))
267 +
268 +
269 +def _parse_args():
270 + """Parses the command line arguments."""
271 + parser = argparse.ArgumentParser()
272 + parser.add_argument(
273 + '--recursive', action='store_true',
274 + help='If set, installs files recursively. Otherwise, '
275 + 'just skips directories.')
276 + parser.add_argument(
277 + '--preserve_symlinks', action='store_true',
278 + help='If set, a symlink will be installed as symlink.')
279 + # If helper is dodoc, it changes the behavior for the directory
280 + # install without --recursive.
281 + parser.add_argument('--helper', help='Name of helper.')
282 + parser.add_argument(
283 + '--dest',
284 + help='Destination where the files are installed.')
285 + parser.add_argument(
286 + 'sources', nargs='*',
287 + help='Source file/directory paths to be installed.')
288 +
289 + args = parser.parse_args()
290 +
291 + # Encode back to the original byte stream. Please see
292 + # http://bugs.python.org/issue8776.
293 + if sys.version_info.major >= 3:
294 + args.dest = os.fsencode(args.dest)
295 + args.sources = [os.fsencode(source) for source in args.sources]
296 +
297 + return args
298 +
299 +
300 +def _install_dir(args, install_runner, source):
301 + """Installs directory at |source|.
302 +
303 + Returns:
304 + True on success, False on failure, or None on skipped.
305 + """
306 + if not args.recursive:
307 + if args.helper == 'dodoc':
308 + print('!!! %s: %s is a directory' % (
309 + args.helper, source),
310 + file=sys.stderr)
311 + return False
312 + # Neither success nor fail. Return None to indicate skipped.
313 + return None
314 +
315 + # Strip trailing '/'s.
316 + source = source.rstrip(b'/')
317 + source_root = os.path.dirname(source)
318 + dest_dir = os.path.join(args.dest, os.path.basename(source))
319 + install_runner.install_dir(dest_dir)
320 +
321 + relpath_list = []
322 + for dirpath, dirnames, filenames in os.walk(source):
323 + for dirname in dirnames:
324 + relpath = os.path.relpath(
325 + os.path.join(dirpath, dirname),
326 + source_root)
327 + dest = os.path.join(args.dest, relpath)
328 + install_runner.install_dir(dest)
329 + relpath_list.extend(
330 + os.path.relpath(
331 + os.path.join(dirpath, filename), source_root)
332 + for filename in filenames)
333 +
334 + if not relpath_list:
335 + # NOTE: Even if only an empty directory is installed here, it
336 + # still counts as success, since an empty directory given as
337 + # an argument to doins -r should not trigger failure.
338 + return True
339 + success = True
340 + for relpath in relpath_list:
341 + if not _doins(args, install_runner, relpath, source_root):
342 + success = False
343 + return success
344 +
345 +
346 +def main():
347 + args = _parse_args()
348 + install_runner = _InstallRunner()
349 +
350 + if not os.path.isdir(args.dest):
351 + install_runner.install_dir(args.dest)
352 +
353 + any_success = False
354 + any_failure = False
355 + for source in args.sources:
356 + if (os.path.isdir(source) and
357 + (not args.preserve_symlinks or
358 + not os.path.islink(source))):
359 + ret = _install_dir(args, install_runner, source)
360 + if ret is None:
361 + continue
362 + if ret:
363 + any_success = True
364 + else:
365 + any_failure = True
366 + else:
367 + if _doins(
368 + args, install_runner,
369 + os.path.basename(source),
370 + os.path.dirname(source)):
371 + any_success = True
372 + else:
373 + any_failure = True
374 +
375 + return 0 if not any_failure and any_success else 1
376 +
377 +
378 +if __name__ == '__main__':
379 + sys.exit(main())
380 diff --git a/bin/ebuild-helpers/doins b/bin/ebuild-helpers/doins
381 index 93052c2ea..de559780c 100755
382 --- a/bin/ebuild-helpers/doins
383 +++ b/bin/ebuild-helpers/doins
384 @@ -24,10 +24,10 @@ if [ $# -lt 1 ] ; then
385 fi
386
387 if [[ "$1" == "-r" ]] ; then
388 - DOINSRECUR=y
389 + RECURSIVE_OPTION='--recursive'
390 shift
391 else
392 - DOINSRECUR=n
393 + RECURSIVE_OPTION=''
394 fi
395
396 if ! ___eapi_has_prefix_variables; then
397 @@ -44,116 +44,17 @@ if [[ ${INSDESTTREE#${ED}} != "${INSDESTTREE}" ]]; then
398 fi
399
400 if ___eapi_doins_and_newins_preserve_symlinks; then
401 - PRESERVE_SYMLINKS=y
402 + SYMLINK_OPTION='--preserve_symlinks'
403 else
404 - PRESERVE_SYMLINKS=n
405 + SYMLINK_OPTION=''
406 fi
407
408 -export TMP=$(mktemp -d "${T}/.doins_tmp_XXXXXX")
409 -# Use separate directories to avoid potential name collisions.
410 -mkdir -p "$TMP"/{1,2}
411 +# Use safe cwd, avoiding unsafe import for bug #469338.
412 +export __PORTAGE_HELPER_CWD=${PWD}
413 +cd "${PORTAGE_PYM_PATH:-/usr/lib/portage/pym}"
414
415 -[[ ! -d ${ED}${INSDESTTREE} ]] && dodir "${INSDESTTREE}"
416 -
417 -_doins() {
418 - local mysrc="$1" mydir="$2" cleanup="" rval
419 -
420 - if [ -L "$mysrc" ] ; then
421 - # Our fake $DISTDIR contains symlinks that should
422 - # not be reproduced inside $D. In order to ensure
423 - # that things like dodoc "$DISTDIR"/foo.pdf work
424 - # as expected, we dereference symlinked files that
425 - # refer to absolute paths inside
426 - # $PORTAGE_ACTUAL_DISTDIR/.
427 - if [ $PRESERVE_SYMLINKS = y ] && \
428 - ! [[ $(readlink "$mysrc") == "$PORTAGE_ACTUAL_DISTDIR"/* ]] ; then
429 - rm -rf "${ED}$INSDESTTREE/$mydir/${mysrc##*/}" || return $?
430 - cp -P "$mysrc" "${ED}$INSDESTTREE/$mydir/${mysrc##*/}"
431 - return $?
432 - else
433 - cp "$mysrc" "$TMP/2/${mysrc##*/}" || return $?
434 - mysrc="$TMP/2/${mysrc##*/}"
435 - cleanup=$mysrc
436 - fi
437 - fi
438 -
439 - install ${INSOPTIONS} "${mysrc}" "${ED}${INSDESTTREE}/${mydir}"
440 - rval=$?
441 - [[ -n ${cleanup} ]] && rm -f "${cleanup}"
442 - [ $rval -ne 0 ] && echo "!!! ${helper}: $mysrc does not exist" 1>&2
443 - return $rval
444 -}
445 -
446 -_xdoins() {
447 - local -i failed=0
448 - while read -r -d $'\0' x ; do
449 - _doins "$x" "${x%/*}"
450 - ((failed|=$?))
451 - done
452 - return $failed
453 -}
454 -
455 -success=0
456 -failed=0
457 -
458 -for x in "$@" ; do
459 - if [[ $PRESERVE_SYMLINKS = n && -d $x ]] || \
460 - [[ $PRESERVE_SYMLINKS = y && -d $x && ! -L $x ]] ; then
461 - if [ "${DOINSRECUR}" == "n" ] ; then
462 - if [[ ${helper} == dodoc ]] ; then
463 - echo "!!! ${helper}: $x is a directory" 1>&2
464 - ((failed|=1))
465 - fi
466 - continue
467 - fi
468 -
469 - while [ "$x" != "${x%/}" ] ; do
470 - x=${x%/}
471 - done
472 - if [ "$x" = "${x%/*}" ] ; then
473 - pushd "$PWD" >/dev/null
474 - else
475 - pushd "${x%/*}" >/dev/null
476 - fi
477 - x=${x##*/}
478 - x_orig=$x
479 - # Follow any symlinks recursively until we've got
480 - # a normal directory for 'find' to traverse. The
481 - # name of the symlink will be used for the name
482 - # of the installed directory, as discussed in
483 - # bug #239529.
484 - while [ -L "$x" ] ; do
485 - pushd "$(readlink "$x")" >/dev/null
486 - x=${PWD##*/}
487 - pushd "${PWD%/*}" >/dev/null
488 - done
489 - if [[ $x != $x_orig ]] ; then
490 - mv "$x" "$TMP/1/$x_orig"
491 - pushd "$TMP/1" >/dev/null
492 - fi
493 - find "$x_orig" -type d -exec dodir "${INSDESTTREE}/{}" \;
494 - find "$x_orig" \( -type f -or -type l \) -print0 | _xdoins
495 - if [[ ${PIPESTATUS[1]} -eq 0 ]] ; then
496 - # NOTE: Even if only an empty directory is installed here, it
497 - # still counts as success, since an empty directory given as
498 - # an argument to doins -r should not trigger failure.
499 - ((success|=1))
500 - else
501 - ((failed|=1))
502 - fi
503 - if [[ $x != $x_orig ]] ; then
504 - popd >/dev/null
505 - mv "$TMP/1/$x_orig" "$x"
506 - fi
507 - while popd >/dev/null 2>&1 ; do true ; done
508 - else
509 - _doins "${x}"
510 - if [[ $? -eq 0 ]] ; then
511 - ((success|=1))
512 - else
513 - ((failed|=1))
514 - fi
515 - fi
516 -done
517 -rm -rf "$TMP"
518 -[[ $failed -ne 0 || $success -eq 0 ]] && { __helpers_die "${helper} failed"; exit 1; } || exit 0
519 +"${PORTAGE_PYTHON:-/usr/bin/python}" \
520 + "${PORTAGE_BIN_PATH:-/usr/lib/portage/bin}"/doins.py \
521 + ${RECURSIVE_OPTION} ${SYMLINK_OPTION} \
522 + --helper "${helper}" --dest "${ED}${INSDESTTREE}" "$@" || \
523 +{ __helpers_die "${helper} failed"; exit 1; }
524 --
525 2.13.0

Replies