1 |
From: Hidehiko Abe <hidehiko@××××××××.org> |
2 |
|
3 |
doins is written in bash. However, specifically in case that |
4 |
too many files are installed, it is very slow. |
5 |
This CL rewrites the script in python for performance. |
6 |
|
7 |
BUG=chromium:712659 |
8 |
TEST=time (./setup_board --forace && \ |
9 |
./build_package --withdev && \ |
10 |
./build_image --noenable_rootfs_verification test) |
11 |
===Before=== |
12 |
real 21m35.445s |
13 |
user 93m40.588s |
14 |
sys 21m31.224s |
15 |
|
16 |
===After=== |
17 |
real 17m30.106s |
18 |
user 94m1.812s |
19 |
sys 20m13.468s |
20 |
|
21 |
Change-Id: Ib10f623961ba316753d58397cff5e72fbc343339 |
22 |
Reviewed-on: https://chromium-review.googlesource.com/559225 |
23 |
X-Chromium-Bug: 712659 |
24 |
X-Chromium-Bug-url: https://bugs.chromium.org/p/chromium/issues/detail?id=712659 |
25 |
X-Gentoo-Bug: 624526 |
26 |
X-Gentoo-Bug-url: https://bugs.gentoo.org/624526 |
27 |
--- |
28 |
bin/doins.py | 341 +++++++++++++++++++++++++++++++++++++++++++++++ |
29 |
bin/ebuild-helpers/doins | 123 ++--------------- |
30 |
2 files changed, 353 insertions(+), 111 deletions(-) |
31 |
create mode 100644 bin/doins.py |
32 |
|
33 |
diff --git a/bin/doins.py b/bin/doins.py |
34 |
new file mode 100644 |
35 |
index 000000000..4a17287ca |
36 |
--- /dev/null |
37 |
+++ b/bin/doins.py |
38 |
@@ -0,0 +1,341 @@ |
39 |
+#!/usr/bin/python -b |
40 |
+# Copyright 2017 The Chromium OS Authors. All rights reserved. |
41 |
+# Use of this source code is governed by a BSD-style license that can be |
42 |
+# found in the LICENSE file. |
43 |
+ |
44 |
+from __future__ import print_function |
45 |
+ |
46 |
+import argparse |
47 |
+import errno |
48 |
+import grp |
49 |
+import logging |
50 |
+import os |
51 |
+import pwd |
52 |
+import re |
53 |
+import shlex |
54 |
+import shutil |
55 |
+import stat |
56 |
+import subprocess |
57 |
+import sys |
58 |
+ |
59 |
+from portage.util import movefile |
60 |
+ |
61 |
+# Change back to original cwd _after_ all imports (bug #469338). |
62 |
+# See also bin/ebuild-helpers/doins, which sets the cwd. |
63 |
+os.chdir(os.environ["__PORTAGE_HELPER_CWD"]) |
64 |
+ |
65 |
+_PORTAGE_ACTUAL_DISTDIR = ( |
66 |
+ (os.environb if sys.version_info.major >= 3 else os.environ).get( |
67 |
+ b'PORTAGE_ACTUAL_DISTDIR', b'') + b'/') |
68 |
+ |
69 |
+ |
70 |
+def _parse_install_options( |
71 |
+ options, inprocess_runner_class, subprocess_runner_class): |
72 |
+ """Parses command line arguments for install command.""" |
73 |
+ parser = argparse.ArgumentParser() |
74 |
+ parser.add_argument('-g', '--group', default=-1, type=_parse_group) |
75 |
+ parser.add_argument('-o', '--owner', default=-1, type=_parse_user) |
76 |
+ # "install"'s --mode option is complicated. So here is partially |
77 |
+ # supported. |
78 |
+ parser.add_argument('-m', '--mode', type=_parse_mode) |
79 |
+ split_options = shlex.split(options) |
80 |
+ namespace, remaining = parser.parse_known_args(split_options) |
81 |
+ if remaining: |
82 |
+ print('Unknown install options: %s, %r' % (options, remaining), |
83 |
+ file=sys.stderr) |
84 |
+ if os.environ.get('DOINSSTRICTOPTION', '') == '1': |
85 |
+ sys.exit(1) |
86 |
+ print('Continue with falling back to \'install\' ' |
87 |
+ 'command execution, which can be slower.', |
88 |
+ file=sys.stderr) |
89 |
+ return subprocess_runner_class(split_options) |
90 |
+ return inprocess_runner_class(namespace) |
91 |
+ |
92 |
+ |
93 |
+def _parse_group(group): |
94 |
+ """Parses gid.""" |
95 |
+ g = grp.getgrnam(group) |
96 |
+ if g: |
97 |
+ return g.gr_gid |
98 |
+ return int(group) |
99 |
+ |
100 |
+ |
101 |
+def _parse_user(user): |
102 |
+ """Parses uid.""" |
103 |
+ u = pwd.getpwnam(user) |
104 |
+ if u: |
105 |
+ return u.pw_uid |
106 |
+ return int(user) |
107 |
+ |
108 |
+ |
109 |
+def _parse_mode(mode): |
110 |
+ # "install"'s --mode option is complicated. So here is partially |
111 |
+ # supported. |
112 |
+ # In Python 3, the prefix of octal int must be '0o' rather than '0'. |
113 |
+ # So, set base explicitly in that case. |
114 |
+ return int(mode, 8 if re.search(r'^0[0-7]*$', mode) else 0) |
115 |
+ |
116 |
+ |
117 |
+def _set_attributes(options, path): |
118 |
+ """Sets attributes the file/dir at given |path|. |
119 |
+ |
120 |
+ Args: |
121 |
+ options: object which has |owner|, |group| and |mode| fields. |
122 |
+ |owner| is int value representing uid. Similary |group| |
123 |
+ represents gid. |
124 |
+ If -1 is set, just unchanged. |
125 |
+ |mode| is the bits of permissions. |
126 |
+ path: File/directory path. |
127 |
+ """ |
128 |
+ if options.owner != -1 or options.group != -1: |
129 |
+ os.lchown(path, options.owner, options.group) |
130 |
+ if options.mode is not None: |
131 |
+ os.chmod(path, options.mode) |
132 |
+ |
133 |
+ |
134 |
+class _InsInProcessInstallRunner(object): |
135 |
+ def __init__(self, parsed_options): |
136 |
+ self._parsed_options = parsed_options |
137 |
+ self._copy_xattr = ( |
138 |
+ 'xattr' in os.environ.get('FEATURES', '').split()) |
139 |
+ if self._copy_xattr: |
140 |
+ self._xattr_exclude = os.environ.get( |
141 |
+ 'PORTAGE_XATTR_EXCLUDE', |
142 |
+ 'security.* system.nfs4_acl') |
143 |
+ |
144 |
+ def run(self, source, dest_dir): |
145 |
+ """Installs a file at |source| into |dest_dir| in process.""" |
146 |
+ dest = os.path.join(dest_dir, os.path.basename(source)) |
147 |
+ try: |
148 |
+ # TODO: Consider to use portage.util.file_copy.copyfile |
149 |
+ # introduced by |
150 |
+ # https://gitweb.gentoo.org/proj/portage.git/commit/ |
151 |
+ # ?id=8ab5c8835931fd9ec098dbf4c5f416eb32e4a3a4 |
152 |
+ # after uprev. |
153 |
+ shutil.copyfile(source, dest) |
154 |
+ _set_attributes(self._parsed_options, dest) |
155 |
+ if self._copy_xattr: |
156 |
+ movefile._copyxattr( |
157 |
+ source, dest, |
158 |
+ exclude=self._xattr_exclude) |
159 |
+ except Exception: |
160 |
+ logging.exception( |
161 |
+ 'Failed to copy file: ' |
162 |
+ '_parsed_options=%r, source=%r, dest_dir=%r', |
163 |
+ self._parsed_options, source, dest_dir) |
164 |
+ return False |
165 |
+ return True |
166 |
+ |
167 |
+ |
168 |
+class _InsSubprocessInstallRunner(object): |
169 |
+ def __init__(self, split_options): |
170 |
+ self._split_options = split_options |
171 |
+ |
172 |
+ def run(self, source, dest_dir): |
173 |
+ """Installs a file at |source| into |dest_dir| by 'install'.""" |
174 |
+ command = ['install'] + self._split_options + [source, dest_dir] |
175 |
+ return subprocess.call(command) == 0 |
176 |
+ |
177 |
+ |
178 |
+class _DirInProcessInstallRunner(object): |
179 |
+ def __init__(self, parsed_options): |
180 |
+ self._parsed_options = parsed_options |
181 |
+ |
182 |
+ def run(self, dest): |
183 |
+ """Installs a dir into |dest| in process.""" |
184 |
+ try: |
185 |
+ os.makedirs(dest) |
186 |
+ except OSError as e: |
187 |
+ if e.errno != errno.EEXIST or not os.path.isdir(dest): |
188 |
+ raise |
189 |
+ _set_attributes(self._parsed_options, dest) |
190 |
+ |
191 |
+ |
192 |
+class _DirSubprocessInstallRunner(object): |
193 |
+ """Runs real 'install' command to install files or directories.""" |
194 |
+ |
195 |
+ def __init__(self, split_options): |
196 |
+ self._split_options = split_options |
197 |
+ |
198 |
+ def run(self, dest): |
199 |
+ """Installs a dir into |dest| by 'install' command.""" |
200 |
+ command = ['install', '-d'] + self._split_options + [dest] |
201 |
+ subprocess.call(command) |
202 |
+ |
203 |
+ |
204 |
+class _InstallRunner(object): |
205 |
+ def __init__(self): |
206 |
+ self._ins_runner = _parse_install_options( |
207 |
+ os.environ.get('INSOPTIONS', ''), |
208 |
+ _InsInProcessInstallRunner, |
209 |
+ _InsSubprocessInstallRunner) |
210 |
+ self._dir_runner = _parse_install_options( |
211 |
+ os.environ.get('DIROPTIONS', ''), |
212 |
+ _DirInProcessInstallRunner, |
213 |
+ _DirSubprocessInstallRunner) |
214 |
+ |
215 |
+ def install_file(self, source, dest_dir): |
216 |
+ """Installs a file at |source| into |dest_dir| directory.""" |
217 |
+ return self._ins_runner.run(source, dest_dir) |
218 |
+ |
219 |
+ def install_dir(self, dest): |
220 |
+ """Creates a directory at |dest|.""" |
221 |
+ self._dir_runner.run(dest) |
222 |
+ |
223 |
+ |
224 |
+def _doins(args, install_runner, relpath, source_root): |
225 |
+ """Installs a file as if 'install' command runs. |
226 |
+ |
227 |
+ Installs a file at |source_root|/|relpath| into |args.dest|/|relpath|. |
228 |
+ If |args.preserve_symlinks| is set, creates symlink if the source is a |
229 |
+ symlink. |
230 |
+ |
231 |
+ Args: |
232 |
+ args: parsed arguments. It should have following fields. |
233 |
+ - preserve_symlinks: bool representing whether symlinks |
234 |
+ needs to be preserved. |
235 |
+ - dest: Destination root directory. |
236 |
+ - insoptions: options for 'install' command. |
237 |
+ intall_runner: _InstallRunner instance for file install. |
238 |
+ relpath: Relative path of the file being installed. |
239 |
+ source_root: Source root directory. |
240 |
+ |
241 |
+ Returns: True on success. |
242 |
+ """ |
243 |
+ source = os.path.join(source_root, relpath) |
244 |
+ dest = os.path.join(args.dest, relpath) |
245 |
+ if os.path.islink(source): |
246 |
+ # Our fake $DISTDIR contains symlinks that should not be |
247 |
+ # reproduced inside $D. In order to ensure that things like |
248 |
+ # dodoc "$DISTDIR"/foo.pdf work as expected, we dereference |
249 |
+ # symlinked files that refer to absolute paths inside |
250 |
+ # $PORTAGE_ACTUAL_DISTDIR/. |
251 |
+ try: |
252 |
+ if (args.preserve_symlinks and |
253 |
+ not os.readlink(source).startswith( |
254 |
+ _PORTAGE_ACTUAL_DISTDIR)): |
255 |
+ linkto = os.readlink(source) |
256 |
+ shutil.rmtree(dest, ignore_errors=True) |
257 |
+ os.symlink(linkto, dest) |
258 |
+ return True |
259 |
+ except Exception: |
260 |
+ logging.exception( |
261 |
+ 'Failed to create symlink: ' |
262 |
+ 'args=%r, relpath=%r, source_root=%r', |
263 |
+ args, relpath, source_root) |
264 |
+ return False |
265 |
+ |
266 |
+ return install_runner.install_file(source, os.path.dirname(dest)) |
267 |
+ |
268 |
+ |
269 |
+def _parse_args(): |
270 |
+ """Parses the command line arguments.""" |
271 |
+ parser = argparse.ArgumentParser() |
272 |
+ parser.add_argument( |
273 |
+ '--recursive', action='store_true', |
274 |
+ help='If set, installs files recursively. Otherwise, ' |
275 |
+ 'just skips directories.') |
276 |
+ parser.add_argument( |
277 |
+ '--preserve_symlinks', action='store_true', |
278 |
+ help='If set, a symlink will be installed as symlink.') |
279 |
+ # If helper is dodoc, it changes the behavior for the directory |
280 |
+ # install without --recursive. |
281 |
+ parser.add_argument('--helper', help='Name of helper.') |
282 |
+ parser.add_argument( |
283 |
+ '--dest', |
284 |
+ help='Destination where the files are installed.') |
285 |
+ parser.add_argument( |
286 |
+ 'sources', nargs='*', |
287 |
+ help='Source file/directory paths to be installed.') |
288 |
+ |
289 |
+ args = parser.parse_args() |
290 |
+ |
291 |
+ # Encode back to the original byte stream. Please see |
292 |
+ # http://bugs.python.org/issue8776. |
293 |
+ if sys.version_info.major >= 3: |
294 |
+ args.dest = os.fsencode(args.dest) |
295 |
+ args.sources = [os.fsencode(source) for source in args.sources] |
296 |
+ |
297 |
+ return args |
298 |
+ |
299 |
+ |
300 |
+def _install_dir(args, install_runner, source): |
301 |
+ """Installs directory at |source|. |
302 |
+ |
303 |
+ Returns: |
304 |
+ True on success, False on failure, or None on skipped. |
305 |
+ """ |
306 |
+ if not args.recursive: |
307 |
+ if args.helper == 'dodoc': |
308 |
+ print('!!! %s: %s is a directory' % ( |
309 |
+ args.helper, source), |
310 |
+ file=sys.stderr) |
311 |
+ return False |
312 |
+ # Neither success nor fail. Return None to indicate skipped. |
313 |
+ return None |
314 |
+ |
315 |
+ # Strip trailing '/'s. |
316 |
+ source = source.rstrip(b'/') |
317 |
+ source_root = os.path.dirname(source) |
318 |
+ dest_dir = os.path.join(args.dest, os.path.basename(source)) |
319 |
+ install_runner.install_dir(dest_dir) |
320 |
+ |
321 |
+ relpath_list = [] |
322 |
+ for dirpath, dirnames, filenames in os.walk(source): |
323 |
+ for dirname in dirnames: |
324 |
+ relpath = os.path.relpath( |
325 |
+ os.path.join(dirpath, dirname), |
326 |
+ source_root) |
327 |
+ dest = os.path.join(args.dest, relpath) |
328 |
+ install_runner.install_dir(dest) |
329 |
+ relpath_list.extend( |
330 |
+ os.path.relpath( |
331 |
+ os.path.join(dirpath, filename), source_root) |
332 |
+ for filename in filenames) |
333 |
+ |
334 |
+ if not relpath_list: |
335 |
+ # NOTE: Even if only an empty directory is installed here, it |
336 |
+ # still counts as success, since an empty directory given as |
337 |
+ # an argument to doins -r should not trigger failure. |
338 |
+ return True |
339 |
+ success = True |
340 |
+ for relpath in relpath_list: |
341 |
+ if not _doins(args, install_runner, relpath, source_root): |
342 |
+ success = False |
343 |
+ return success |
344 |
+ |
345 |
+ |
346 |
+def main(): |
347 |
+ args = _parse_args() |
348 |
+ install_runner = _InstallRunner() |
349 |
+ |
350 |
+ if not os.path.isdir(args.dest): |
351 |
+ install_runner.install_dir(args.dest) |
352 |
+ |
353 |
+ any_success = False |
354 |
+ any_failure = False |
355 |
+ for source in args.sources: |
356 |
+ if (os.path.isdir(source) and |
357 |
+ (not args.preserve_symlinks or |
358 |
+ not os.path.islink(source))): |
359 |
+ ret = _install_dir(args, install_runner, source) |
360 |
+ if ret is None: |
361 |
+ continue |
362 |
+ if ret: |
363 |
+ any_success = True |
364 |
+ else: |
365 |
+ any_failure = True |
366 |
+ else: |
367 |
+ if _doins( |
368 |
+ args, install_runner, |
369 |
+ os.path.basename(source), |
370 |
+ os.path.dirname(source)): |
371 |
+ any_success = True |
372 |
+ else: |
373 |
+ any_failure = True |
374 |
+ |
375 |
+ return 0 if not any_failure and any_success else 1 |
376 |
+ |
377 |
+ |
378 |
+if __name__ == '__main__': |
379 |
+ sys.exit(main()) |
380 |
diff --git a/bin/ebuild-helpers/doins b/bin/ebuild-helpers/doins |
381 |
index 93052c2ea..de559780c 100755 |
382 |
--- a/bin/ebuild-helpers/doins |
383 |
+++ b/bin/ebuild-helpers/doins |
384 |
@@ -24,10 +24,10 @@ if [ $# -lt 1 ] ; then |
385 |
fi |
386 |
|
387 |
if [[ "$1" == "-r" ]] ; then |
388 |
- DOINSRECUR=y |
389 |
+ RECURSIVE_OPTION='--recursive' |
390 |
shift |
391 |
else |
392 |
- DOINSRECUR=n |
393 |
+ RECURSIVE_OPTION='' |
394 |
fi |
395 |
|
396 |
if ! ___eapi_has_prefix_variables; then |
397 |
@@ -44,116 +44,17 @@ if [[ ${INSDESTTREE#${ED}} != "${INSDESTTREE}" ]]; then |
398 |
fi |
399 |
|
400 |
if ___eapi_doins_and_newins_preserve_symlinks; then |
401 |
- PRESERVE_SYMLINKS=y |
402 |
+ SYMLINK_OPTION='--preserve_symlinks' |
403 |
else |
404 |
- PRESERVE_SYMLINKS=n |
405 |
+ SYMLINK_OPTION='' |
406 |
fi |
407 |
|
408 |
-export TMP=$(mktemp -d "${T}/.doins_tmp_XXXXXX") |
409 |
-# Use separate directories to avoid potential name collisions. |
410 |
-mkdir -p "$TMP"/{1,2} |
411 |
+# Use safe cwd, avoiding unsafe import for bug #469338. |
412 |
+export __PORTAGE_HELPER_CWD=${PWD} |
413 |
+cd "${PORTAGE_PYM_PATH:-/usr/lib/portage/pym}" |
414 |
|
415 |
-[[ ! -d ${ED}${INSDESTTREE} ]] && dodir "${INSDESTTREE}" |
416 |
- |
417 |
-_doins() { |
418 |
- local mysrc="$1" mydir="$2" cleanup="" rval |
419 |
- |
420 |
- if [ -L "$mysrc" ] ; then |
421 |
- # Our fake $DISTDIR contains symlinks that should |
422 |
- # not be reproduced inside $D. In order to ensure |
423 |
- # that things like dodoc "$DISTDIR"/foo.pdf work |
424 |
- # as expected, we dereference symlinked files that |
425 |
- # refer to absolute paths inside |
426 |
- # $PORTAGE_ACTUAL_DISTDIR/. |
427 |
- if [ $PRESERVE_SYMLINKS = y ] && \ |
428 |
- ! [[ $(readlink "$mysrc") == "$PORTAGE_ACTUAL_DISTDIR"/* ]] ; then |
429 |
- rm -rf "${ED}$INSDESTTREE/$mydir/${mysrc##*/}" || return $? |
430 |
- cp -P "$mysrc" "${ED}$INSDESTTREE/$mydir/${mysrc##*/}" |
431 |
- return $? |
432 |
- else |
433 |
- cp "$mysrc" "$TMP/2/${mysrc##*/}" || return $? |
434 |
- mysrc="$TMP/2/${mysrc##*/}" |
435 |
- cleanup=$mysrc |
436 |
- fi |
437 |
- fi |
438 |
- |
439 |
- install ${INSOPTIONS} "${mysrc}" "${ED}${INSDESTTREE}/${mydir}" |
440 |
- rval=$? |
441 |
- [[ -n ${cleanup} ]] && rm -f "${cleanup}" |
442 |
- [ $rval -ne 0 ] && echo "!!! ${helper}: $mysrc does not exist" 1>&2 |
443 |
- return $rval |
444 |
-} |
445 |
- |
446 |
-_xdoins() { |
447 |
- local -i failed=0 |
448 |
- while read -r -d $'\0' x ; do |
449 |
- _doins "$x" "${x%/*}" |
450 |
- ((failed|=$?)) |
451 |
- done |
452 |
- return $failed |
453 |
-} |
454 |
- |
455 |
-success=0 |
456 |
-failed=0 |
457 |
- |
458 |
-for x in "$@" ; do |
459 |
- if [[ $PRESERVE_SYMLINKS = n && -d $x ]] || \ |
460 |
- [[ $PRESERVE_SYMLINKS = y && -d $x && ! -L $x ]] ; then |
461 |
- if [ "${DOINSRECUR}" == "n" ] ; then |
462 |
- if [[ ${helper} == dodoc ]] ; then |
463 |
- echo "!!! ${helper}: $x is a directory" 1>&2 |
464 |
- ((failed|=1)) |
465 |
- fi |
466 |
- continue |
467 |
- fi |
468 |
- |
469 |
- while [ "$x" != "${x%/}" ] ; do |
470 |
- x=${x%/} |
471 |
- done |
472 |
- if [ "$x" = "${x%/*}" ] ; then |
473 |
- pushd "$PWD" >/dev/null |
474 |
- else |
475 |
- pushd "${x%/*}" >/dev/null |
476 |
- fi |
477 |
- x=${x##*/} |
478 |
- x_orig=$x |
479 |
- # Follow any symlinks recursively until we've got |
480 |
- # a normal directory for 'find' to traverse. The |
481 |
- # name of the symlink will be used for the name |
482 |
- # of the installed directory, as discussed in |
483 |
- # bug #239529. |
484 |
- while [ -L "$x" ] ; do |
485 |
- pushd "$(readlink "$x")" >/dev/null |
486 |
- x=${PWD##*/} |
487 |
- pushd "${PWD%/*}" >/dev/null |
488 |
- done |
489 |
- if [[ $x != $x_orig ]] ; then |
490 |
- mv "$x" "$TMP/1/$x_orig" |
491 |
- pushd "$TMP/1" >/dev/null |
492 |
- fi |
493 |
- find "$x_orig" -type d -exec dodir "${INSDESTTREE}/{}" \; |
494 |
- find "$x_orig" \( -type f -or -type l \) -print0 | _xdoins |
495 |
- if [[ ${PIPESTATUS[1]} -eq 0 ]] ; then |
496 |
- # NOTE: Even if only an empty directory is installed here, it |
497 |
- # still counts as success, since an empty directory given as |
498 |
- # an argument to doins -r should not trigger failure. |
499 |
- ((success|=1)) |
500 |
- else |
501 |
- ((failed|=1)) |
502 |
- fi |
503 |
- if [[ $x != $x_orig ]] ; then |
504 |
- popd >/dev/null |
505 |
- mv "$TMP/1/$x_orig" "$x" |
506 |
- fi |
507 |
- while popd >/dev/null 2>&1 ; do true ; done |
508 |
- else |
509 |
- _doins "${x}" |
510 |
- if [[ $? -eq 0 ]] ; then |
511 |
- ((success|=1)) |
512 |
- else |
513 |
- ((failed|=1)) |
514 |
- fi |
515 |
- fi |
516 |
-done |
517 |
-rm -rf "$TMP" |
518 |
-[[ $failed -ne 0 || $success -eq 0 ]] && { __helpers_die "${helper} failed"; exit 1; } || exit 0 |
519 |
+"${PORTAGE_PYTHON:-/usr/bin/python}" \ |
520 |
+ "${PORTAGE_BIN_PATH:-/usr/lib/portage/bin}"/doins.py \ |
521 |
+ ${RECURSIVE_OPTION} ${SYMLINK_OPTION} \ |
522 |
+ --helper "${helper}" --dest "${ED}${INSDESTTREE}" "$@" || \ |
523 |
+{ __helpers_die "${helper} failed"; exit 1; } |
524 |
-- |
525 |
2.13.0 |