Gentoo Archives: gentoo-commits

From: "Fabian Groffen (grobian)" <grobian@g.o>
To: gentoo-commits@l.g.o
Subject: [gentoo-commits] portage r9375 - in main/branches/prefix: bin pym/_emerge pym/portage pym/portage/dbapi pym/portage/sets pym/repoman
Date: Sat, 23 Feb 2008 23:46:31
Message-Id: E1JT44Z-0002aC-30@stork.gentoo.org
1 Author: grobian
2 Date: 2008-02-23 23:46:25 +0000 (Sat, 23 Feb 2008)
3 New Revision: 9375
4
5 Modified:
6 main/branches/prefix/bin/ebuild.sh
7 main/branches/prefix/bin/emerge-webrsync
8 main/branches/prefix/bin/misc-functions.sh
9 main/branches/prefix/bin/repoman
10 main/branches/prefix/pym/_emerge/__init__.py
11 main/branches/prefix/pym/portage/dbapi/porttree.py
12 main/branches/prefix/pym/portage/dbapi/vartree.py
13 main/branches/prefix/pym/portage/sets/__init__.py
14 main/branches/prefix/pym/portage/sets/dbapi.py
15 main/branches/prefix/pym/portage/sets/files.py
16 main/branches/prefix/pym/portage/sets/security.py
17 main/branches/prefix/pym/portage/util.py
18 main/branches/prefix/pym/repoman/utilities.py
19 Log:
20 Merged from trunk 9351:9372
21
22 | 9352 | Revert back to some known working code from the 2.1.2 |
23 | zmedico | branch. |
24
25 | 9353 | Use parse_use_local_desc() from the repoman.utilities |
26 | zmedico | module. |
27
28 | 9354 | Make PreservedLibsRegistry.store() use atomic_ofstream so |
29 | zmedico | that we don't lose the whole registry if an error such as |
30 | | 'out of space' occurs. |
31
32 | 9355 | Bug #210372 - Fix suidctl.conf parser to look for paths |
33 | zmedico | with a leading slash. |
34
35 | 9357 | Bug #210449 - Reset exeinto(), docinto(), insinto(), and |
36 | zmedico | into() state variables in case the user is running the |
37 | | install phase multiple times consecutively via the ebuild |
38 | | command. |
39
40 | 9358 | - outsource logic for boolean options into a central |
41 | genone | function - enable support for greedy behavior in |
42 | | StaticFileSet |
43
44 | 9359 | For greedy slot behavior, in addition to any installed |
45 | zmedico | slots also try to pull in the latest new slot that may be |
46 | | available. |
47
48 | 9360 | When showing an unsatisfied dep, never include installed |
49 | zmedico | packages. |
50
51 | 9361 | Make stack_dicts() treat an empty variable assignment just |
52 | zmedico | like a non-empty assignment. This allows the profile to set |
53 | | and empty variable in make.defaults and have it override a |
54 | | non-empty value as one would expect. |
55
56 | 9362 | In the circular dependency display, drop PDEPEND (aka |
57 | zmedico | MEDIUM_SOFT) from the graph so there's less noise. |
58
59 | 9363 | Optimize dblink.isowner() to use fewer stat calls by doing |
60 | zmedico | a basename comparison to try and eliminate the file before |
61 | | resorting to inode comparison. This speeds up the `portageq |
62 | | owners` command as well as the search that is done when |
63 | | collision-protect finds a collision. |
64
65 | 9364 | Make LibraryPackageMap.update() sort the contents of the |
66 | zmedico | library_consumers cache so that it's nice and orderly. |
67
68 | 9365 | Bug #211067 - Make the "portdir_overlay" and "mydir" |
69 | zmedico | variables contain paths that are consistent wrt eachother |
70 | | regardless of any path irregularities that can be induced |
71 | | by symlinks. Consistency is achieved by regenerating one of |
72 | | the paths to ensure that both paths have the exact same |
73 | | mapping between inodes and paths. This consistency ensures |
74 | | that the path manipulations used to calculate "repolevel" |
75 | | will work as intended. |
76
77 | 9367 | Add support for file paths as arguments to emerge. If an |
78 | zmedico | argument starts with / and it's not recognized as a tbz2 or |
79 | | ebuild then we try to find and owner in the vdb and |
80 | | generate a slot atom from it. Thanks to solar for the |
81 | | suggestion. |
82
83 | 9368 | Revert portdbapi category auto-detection code since it's |
84 | zmedico | not really needed and can cause problems if we need to add |
85 | | new non-category directories in the future. We only really |
86 | | need the category auto-detection for bindbapi and vardbapi |
87 | | anyway. |
88
89 | 9369 | cleanup main option parsing |
90 | SpankMan | |
91
92 | 9370 | tighten up output from script |
93 | SpankMan | |
94
95 | 9371 | add support for lzma/gz snapshots |
96 | SpankMan | |
97
98 | 9372 | only try to fetch compression schemes which the host |
99 | SpankMan | supports |
100
101
102 Modified: main/branches/prefix/bin/ebuild.sh
103 ===================================================================
104 --- main/branches/prefix/bin/ebuild.sh 2008-02-23 23:37:49 UTC (rev 9374)
105 +++ main/branches/prefix/bin/ebuild.sh 2008-02-23 23:46:25 UTC (rev 9375)
106 @@ -263,10 +263,6 @@
107 #if no perms are specified, dirs/files will have decent defaults
108 #(not secretive, but not stupid)
109 umask 022
110 -export DESTTREE=/usr
111 -export INSDESTTREE=""
112 -export _E_EXEDESTTREE_=""
113 -export _E_DOCDESTTREE_=""
114 export INSOPTIONS="-m0644"
115 export EXEOPTIONS="-m0755"
116 export LIBOPTIONS="-m0644"
117 @@ -986,6 +982,15 @@
118 #some packages uses an alternative to $S to build in, cause
119 #our libtool to create problematic .la files
120 export PWORKDIR="$WORKDIR"
121 +
122 + # Reset exeinto(), docinto(), insinto(), and into() state variables
123 + # in case the user is running the install phase multiple times
124 + # consecutively via the ebuild command.
125 + export DESTTREE=/usr
126 + export INSDESTTREE=""
127 + export _E_EXEDESTTREE_=""
128 + export _E_DOCDESTTREE_=""
129 +
130 ebuild_phase src_install
131 touch "${PORTAGE_BUILDDIR}/.installed"
132 vecho ">>> Completed installing ${PF} into ${ED}"
133
134 Modified: main/branches/prefix/bin/emerge-webrsync
135 ===================================================================
136 --- main/branches/prefix/bin/emerge-webrsync 2008-02-23 23:37:49 UTC (rev 9374)
137 +++ main/branches/prefix/bin/emerge-webrsync 2008-02-23 23:46:25 UTC (rev 9375)
138 @@ -1,5 +1,5 @@
139 #!@BASH@
140 -# Copyright 1999-2006 Gentoo Foundation
141 +# Copyright 1999-2008 Gentoo Foundation
142 # Distributed under the terms of the GNU General Public License v2
143 # $Id$
144 # Author: Karl Trygve Kalleberg <karltk@g.o>
145 @@ -7,6 +7,10 @@
146 # Author: Alon Bar-Lev <alon.barlev@×××××.com>
147 # Major rewrite from Karl's scripts.
148
149 +# TODO:
150 +# - all output should prob be converted to e* funcs
151 +# - add support for ROOT
152 +
153 #
154 # gpg key import
155 # KEY_ID=0x7DDAD20D
156 @@ -14,7 +18,20 @@
157 # gpg --homedir /etc/portage/gnupg --edit-key $KEY_ID trust
158 #
159
160 -type portageq > /dev/null || exit $?
161 +# Only echo if in verbose mode
162 +vvecho() { [[ ${do_verbose} -eq 1 ]] && echo "$@" ; }
163 +# Only echo if not in verbose mode
164 +nvecho() { [[ ${do_verbose} -eq 0 ]] && echo "$@" ; }
165 +# warning echos
166 +wecho() { echo "${argv0}: warning: $*" 1>&2 ; }
167 +# error echos
168 +eecho() { echo "${argv0}: error: $*" 1>&2 ; }
169 +
170 +argv0=$0
171 +if ! type portageq > /dev/null ; then
172 + eecho "could not find 'portageq'; aborting"
173 + exit 1
174 +fi
175 eval $(portageq envvar -v FEATURES FETCHCOMMAND GENTOO_MIRRORS \
176 PORTAGE_BIN_PATH PORTAGE_GPG_DIR PORTAGE_INST_UID PORTAGE_INST_GID \
177 PORTAGE_NICENESS PORTAGE_RSYNC_EXTRA_OPTS PORTAGE_TMPDIR PORTDIR \
178 @@ -31,6 +48,7 @@
179 source "${PORTAGE_BIN_PATH}"/isolated-functions.sh || exit 1
180
181 do_verbose=0
182 +do_debug=0
183
184 if hasq webrsync-gpg ${FEATURES} ; then
185 WEBSYNC_VERIFY_SIGNATURE=1
186 @@ -38,10 +56,24 @@
187 WEBSYNC_VERIFY_SIGNATURE=0
188 fi
189 if [ ${WEBSYNC_VERIFY_SIGNATURE} != 0 -a -z "${PORTAGE_GPG_DIR}" ]; then
190 - echo "Error: Please set PORTAGE_GPG_DIR in make.conf"
191 + eecho "please set PORTAGE_GPG_DIR in make.conf"
192 exit 1
193 fi
194
195 +do_tar() {
196 + local file=$1; shift
197 + local decompressor
198 + case ${file} in
199 + *.lzma) decompressor="lzcat" ;;
200 + *.bz2) decompressor="bzcat" ;;
201 + *.gz) decompressor="zcat" ;;
202 + *) decompressor="cat" ;;
203 + esac
204 + ${decompressor} "${file}" | tar "$@"
205 + _pipestatus=${PIPESTATUS[*]}
206 + [[ ${_pipestatus// /} -eq 0 ]]
207 +}
208 +
209 get_utc_date_in_seconds() {
210 date -u +"%s"
211 }
212 @@ -78,25 +110,17 @@
213 local opts
214
215 if [ "${FETCHCOMMAND/wget/}" != "${FETCHCOMMAND}" ]; then
216 - opts="--continue"
217 -
218 - [ "${do_verbose}" == 0 ] && opts="$opts -q"
219 + opts="--continue $(nvecho -q)"
220 elif [ "${FETCHCOMMAND/curl/}" != "${FETCHCOMMAND}" ]; then
221 - opts="--continue-at -"
222 -
223 - [ "${do_verbose}" == 0 ] && opts="$opts -s -f"
224 + opts="--continue-at - $(nvecho -s -f)"
225 else
226 rm -f "${FILE}"
227 fi
228
229 - echo "Fetching file ${FILE}..."
230 -
231 - #already set DISTDIR=
232 - if [ "${do_verbose}" == 0 ] ; then
233 - eval "${FETCHCOMMAND}" ${opts} > /dev/null && [ -s "${FILE}" ]
234 - else
235 - eval "${FETCHCOMMAND}" ${opts} && [ -s "${FILE}" ]
236 - fi
237 + vecho "Fetching file ${FILE} ..."
238 + # already set DISTDIR=
239 + eval "${FETCHCOMMAND}" ${opts}
240 + [ -s "${FILE}" ]
241 }
242
243 check_file_digest() {
244 @@ -104,15 +128,14 @@
245 local file="$2"
246 local r=1
247
248 - echo "Checking digest..."
249 + vecho "Checking digest ..."
250
251 if type -P md5sum > /dev/null; then
252 md5sum -c $digest && r=0
253 elif type -P md5 > /dev/null; then
254 [ "$(md5 -q $file)" == "$(cut -d \ -f 1 \"$digest\")" ] && r=0
255 else
256 - echo "Error: Cannot check digest"
257 - echo "No suitable md5/md5sum binaries found"
258 + eecho "cannot check digest: no suitable md5/md5sum binaries found"
259 fi
260
261 return "${r}"
262 @@ -125,13 +148,12 @@
263
264 if [ ${WEBSYNC_VERIFY_SIGNATURE} != 0 ]; then
265
266 - echo "Checking signature..."
267 + vecho "Checking signature ..."
268
269 if type -p gpg > /dev/null; then
270 gpg --homedir "${PORTAGE_GPG_DIR}" --verify "$signature" "$file" && r=0
271 else
272 - echo "Error: Cannot check signature"
273 - echo "gpg binary not found"
274 + eecho "cannot check signature: gpg binary not found"
275 fi
276 else
277 r=0
278 @@ -143,27 +165,22 @@
279 get_snapshot_timestamp() {
280 local file="$1"
281
282 - tar --to-stdout -xf "${file}" portage/metadata/timestamp.x | cut -f 1 -d " "
283 + do_tar "${file}" --to-stdout -xf - portage/metadata/timestamp.x | cut -f 1 -d " "
284 }
285
286 sync_local() {
287 local file="$1"
288
289 - echo "Syncing local tree..."
290 + vecho "Syncing local tree ..."
291
292 if type -P tarsync &> /dev/null; then
293 - if [ "${do_verbose}" != 0 ] ; then
294 - tarsync_verbose=-v
295 - fi
296 - if ! tarsync $tarsync_verbose -s 1 -o portage -g portage -e /distfiles -e /packages -e /local "${file}" "${PORTDIR}"; then
297 - echo "Error: tarsync failed; tarball is corrupt? (${file})"
298 + if ! tarsync $(vvecho -v) -s 1 -o portage -g portage -e /distfiles -e /packages -e /local "${file}" "${PORTDIR}"; then
299 + eecho "tarsync failed; tarball is corrupt? (${file})"
300 return 1
301 fi
302 else
303 - echo "Note: tarsync was not found, you may consider emerge it..."
304 -
305 - if ! tar jxf "${file}"; then
306 - echo "Error: tar failed to extract the image. tarball is corrupt? (${file})"
307 + if ! do_tar "${file}" xf -; then
308 + eecho "tar failed to extract the image. tarball is corrupt? (${file})"
309 rm -fr portage
310 return 1
311 fi
312 @@ -179,12 +196,12 @@
313 --exclude='/local' ${PORTAGE_RSYNC_EXTRA_OPTS} . "${PORTDIR%%/}"
314 cd ..
315
316 - echo "Cleaning up..."
317 + vecho "Cleaning up ..."
318 rm -fr portage
319 fi
320
321 if hasq metadata-transfer ${FEATURES} ; then
322 - echo "Updating cache..."
323 + vecho "Updating cache ..."
324 emerge --metadata
325 fi
326 [ -x "${EPREFIX}"/etc/portage/bin/post_sync ] && "${EPREFIX}"/etc/portage/bin/post_sync
327 @@ -197,76 +214,92 @@
328
329 local r=1
330
331 - local file="portage-${date}.tar.bz2"
332 - local digest="${file}.md5sum"
333 - local signature="${file}.gpgsig"
334 + local base_file="portage-${date}.tar"
335
336 local have_files=0
337 local mirror
338
339 - echo "Trying to retrieve ${date} snapshot..."
340 + local compressions=""
341 + type lzcat > /dev/null && compressions="${compressions} lzma"
342 + type bzcat > /dev/null && compressions="${compressions} bz2"
343 + type zcat > /dev/null && compressions="${compressions} gz"
344 + if [[ -z ${compressions} ]] ; then
345 + eecho "unable to locate any decompressors (lzcat or bzcat or zcat)"
346 + exit 1
347 + fi
348
349 for mirror in ${GENTOO_MIRRORS} ; do
350
351 - [ -s "${file}" -a -s "${digest}" -a -s "${signature}" ] && \
352 - check_file_digest "${digest}" "${file}" && \
353 - check_file_signature "${signature}" "${file}" && \
354 - have_files=1
355 + vecho "Trying to retrieve ${date} snapshot from ${mirror} ..."
356
357 - [ ${have_files} == 0 ] && \
358 - fetch_file "${mirror}/snapshots/${digest}" "${digest}" && \
359 - fetch_file "${mirror}/snapshots/${signature}" "${signature}" && \
360 - fetch_file "${mirror}/snapshots/${file}" "${file}" && \
361 - check_file_digest "${digest}" "${file}" && \
362 - check_file_signature "${signature}" "${file}" && \
363 - have_files=1
364 + for compression in ${compressions} ; do
365 + local file="portage-${date}.tar.${compression}"
366 + local digest="${file}.md5sum"
367 + local signature="${file}.gpgsig"
368
369 - #
370 - # If timestamp is invalid
371 - # we want to try and retieve
372 - # from a different mirror
373 - #
374 - if [ ${have_files} != 0 ]; then
375 + if [ -s "${file}" -a -s "${digest}" -a -s "${signature}" ] ; then
376 + check_file_digest "${digest}" "${file}" && \
377 + check_file_signature "${signature}" "${file}" && \
378 + have_files=1
379 + fi
380
381 - echo "Getting snapshot timetasmp..."
382 - local snapshot_timestamp=$(get_snapshot_timestamp "${file}")
383 + if [ ${have_files} -eq 0 ] ; then
384 + fetch_file "${mirror}/snapshots/${digest}" "${digest}" && \
385 + fetch_file "${mirror}/snapshots/${signature}" "${signature}" && \
386 + fetch_file "${mirror}/snapshots/${file}" "${file}" && \
387 + check_file_digest "${digest}" "${file}" && \
388 + check_file_signature "${signature}" "${file}" && \
389 + have_files=1
390 + fi
391
392 - if [ ${ignore_timestamp} == 0 ]; then
393 - if [ ${snapshot_timestamp} -lt $(get_portage_timestamp) ]; then
394 - echo "Warning: Portage is newer than snapshot"
395 - have_files=0
396 + #
397 + # If timestamp is invalid
398 + # we want to try and retrieve
399 + # from a different mirror
400 + #
401 + if [ ${have_files} -eq 1 ]; then
402 +
403 + vecho "Getting snapshot timetasmp ..."
404 + local snapshot_timestamp=$(get_snapshot_timestamp "${file}")
405 +
406 + if [ ${ignore_timestamp} == 0 ]; then
407 + if [ ${snapshot_timestamp} -lt $(get_portage_timestamp) ]; then
408 + wecho "portage is newer than snapshot"
409 + have_files=0
410 + fi
411 + else
412 + local utc_seconds=$(get_utc_second_from_string "${date}")
413 +
414 + #
415 + # Check that this snapshot
416 + # is what it claims to be ...
417 + #
418 + if [ ${snapshot_timestamp} -lt ${utc_seconds} ] || \
419 + [ ${snapshot_timestamp} -gt $((${utc_seconds}+ 2*86400)) ]; then
420 +
421 + wecho "snapshot timestamp is not in acceptable period"
422 + have_files=0
423 + fi
424 fi
425 + fi
426 +
427 + if [ ${have_files} -eq 1 ]; then
428 + break
429 else
430 - local utc_seconds=$(get_utc_second_from_string "${date}")
431 -
432 #
433 - # Check that this snapshot
434 - # is what it claims to be...
435 + # Remove files and use a different mirror
436 #
437 - if [ ${snapshot_timestamp} -lt ${utc_seconds} ] || \
438 - [ ${snapshot_timestamp} -gt $((${utc_seconds}+ 2*86400)) ]; then
439 -
440 - echo "Warning: Snapshot timestamp is not in acceptable period."
441 - have_files=0
442 - fi
443 + rm -f "${file}" "${digest}" "${signature}"
444 fi
445 - fi
446 + done
447
448 - if [ ${have_files} != 0 ]; then
449 - break;
450 - else
451 - #
452 - # Remove files and use
453 - # a diffeernt mirror
454 - #
455 - rm -f "${file}" "${digest}" "${signature}"
456 - fi
457 + [ ${have_files} -eq 1 ] && break
458 done
459
460 - if [ ${have_files} != 0 ]; then
461 + if [ ${have_files} -eq 1 ]; then
462 sync_local "${file}" && r=0
463 else
464 - echo "Warning: ${date} snapshot was not found."
465 + vecho "${date} snapshot was not found"
466 fi
467
468 rm -f "${file}" "${digest}" "${signature}"
469 @@ -277,7 +310,7 @@
470 local attempts=-1
471 local r=1
472
473 - echo "Fetching most recent snapshot..."
474 + vecho "Fetching most recent snapshot ..."
475
476 while (( ${attempts} < 40 )) ; do
477 local day
478 @@ -295,10 +328,9 @@
479 utc_midnight=$(get_date_part $(expr ${utc_attempt} - ${utc_attempt} % 86400) "%s")
480
481 if [ ${utc_midnight} -lt $(($(get_portage_timestamp)-86400)) ]; then
482 - echo "Note: Portage content is newer than available snapshots"
483 - echo "use --revert option to overide."
484 + wecho "portage content is newer than available snapshots (use --revert option to overide)"
485 r=0
486 - break;
487 + break
488 fi
489
490 if do_snapshot 0 "${year}${month}${day}"; then
491 @@ -310,42 +342,50 @@
492 return "${r}"
493 }
494
495 +usage() {
496 + cat <<-EOF
497 + Usage: $0 [options]
498 +
499 + Options:
500 + --revert=yyyymmdd Revert to snapshot
501 + -q, --quiet Only output errors
502 + -v, --verbose Enable verbose output
503 + -x, --debug Enable debug output
504 + -h, --help This help screen (duh!)
505 + EOF
506 + if [[ -n $* ]] ; then
507 + printf "\nError: %s\n" "$*" 1>&2
508 + exit 1
509 + else
510 + exit 0
511 + fi
512 +}
513 +
514 main() {
515 local arg
516 - local do_revert=0
517 local revert_date
518
519 [ ! -d "${DISTDIR}" ] && mkdir -p "${DISTDIR}"
520 cd "${DISTDIR}"
521
522 - for arg in $*; do
523 + for arg in "$@" ; do
524 local v=${arg#*=}
525 case ${arg} in
526 - --help)
527 - echo "usage: $0 [options]"
528 - echo " --verbose (-v) - verbose"
529 - echo " --revert=yyyymmdd - revert to snapshot"
530 - exit 0
531 - ;;
532 - --verbose|-v)
533 - do_verbose=1
534 - ;;
535 - --revert=*)
536 - do_revert=1
537 - revert_date=${v}
538 - ;;
539 - *)
540 - echo "Error: Invalid arguments"
541 - exit 1
542 - ;;
543 + -h|--help) usage ;;
544 + -q|--quiet) PORTAGE_QUIET=1 ;;
545 + -v|--verbose) do_verbose=1 ;;
546 + -x|--debug) do_debug=1 ;;
547 + --revert=*) revert_date=${v} ;;
548 + *) usage "Invalid option '${arg}'" ;;
549 esac
550 done
551 + [[ ${do_debug} -eq 1 ]] && set -x
552
553 - if [ ${do_revert} != 0 ]; then
554 + if [[ -n ${revert_date} ]] ; then
555 do_snapshot 1 "${revert_date}"
556 else
557 do_latest_snapshot
558 fi
559 }
560
561 -main $*
562 +main "$@"
563
564 Modified: main/branches/prefix/bin/misc-functions.sh
565 ===================================================================
566 --- main/branches/prefix/bin/misc-functions.sh 2008-02-23 23:37:49 UTC (rev 9374)
567 +++ main/branches/prefix/bin/misc-functions.sh 2008-02-23 23:46:25 UTC (rev 9375)
568 @@ -592,7 +592,7 @@
569 #note not space-safe
570 for i in $(find "${ED}" -type f \( -perm -4000 -o -perm -2000 \) ); do
571 if [ -s "${sfconf}" ]; then
572 - suid="$(grep "^${i/${D}}$" "${sfconf}")"
573 + suid="$(grep "^/${i#${D}}$" "${sfconf}")"
574 if [ "${suid}" = "${i/${D}}" ]; then
575 vecho "- ${i/${D}} is an approved suid file"
576 else
577
578 Modified: main/branches/prefix/bin/repoman
579 ===================================================================
580 --- main/branches/prefix/bin/repoman 2008-02-23 23:37:49 UTC (rev 9374)
581 +++ main/branches/prefix/bin/repoman 2008-02-23 23:46:25 UTC (rev 9375)
582 @@ -7,6 +7,7 @@
583 # Then, check to make sure deps are satisfiable (to avoid "can't find match for" problems)
584 # that last one is tricky because multiple profiles need to be checked.
585
586 +import commands
587 import codecs
588 import errno
589 import formatter
590 @@ -491,6 +492,9 @@
591 for x in range(0, repolevel - 1):
592 repodir = os.path.dirname(repodir)
593
594 +def caterror(mycat):
595 + err(mycat+" is not an official category. Skipping QA checks in this directory.\nPlease ensure that you add "+catdir+" to "+repodir+"/profiles/categories\nif it is a new category.")
596 +
597 # retreive local USE list
598 luselist={}
599 try:
600 @@ -506,7 +510,7 @@
601 if os.path.exists(filename):
602 try:
603 f = open(filename)
604 - parse_use_local_desc(f, luselist)
605 + utilities.parse_use_local_desc(f, luselist)
606 f.close()
607 except (IOError, OSError, ParseError), e:
608 logging.exception("Couldn't read from '%s'" % filename, e)
609 @@ -546,8 +550,34 @@
610 kwlist.update(portage.grabfile(
611 os.path.join(portdir_overlay, "profiles", "arch.list")))
612
613 -scanlist = utilities.FindPackagesToScan(repoman_settings, startdir, reposplit)
614 +scanlist=[]
615 +if repolevel==2:
616 + #we are inside a category directory
617 + catdir=reposplit[-1]
618 + if catdir not in repoman_settings.categories:
619 + caterror(catdir)
620 + mydirlist=os.listdir(startdir)
621 + for x in mydirlist:
622 + if x == "CVS" or x.startswith("."):
623 + continue
624 + if os.path.isdir(startdir+"/"+x):
625 + scanlist.append(catdir+"/"+x)
626 +elif repolevel==1:
627 + for x in repoman_settings.categories:
628 + if not os.path.isdir(startdir+"/"+x):
629 + continue
630 + for y in os.listdir(startdir+"/"+x):
631 + if y == "CVS" or y.startswith("."):
632 + continue
633 + if os.path.isdir(startdir+"/"+x+"/"+y):
634 + scanlist.append(x+"/"+y)
635 +elif repolevel==3:
636 + catdir = reposplit[-2]
637 + if catdir not in repoman_settings.categories:
638 + caterror(catdir)
639 + scanlist.append(catdir+"/"+reposplit[-1])
640 scanlist.sort()
641 +
642 logging.debug("Found the following packages to scan:\n%s" % '\n'.join(scanlist))
643
644 profiles={}
645
646 Modified: main/branches/prefix/pym/_emerge/__init__.py
647 ===================================================================
648 --- main/branches/prefix/pym/_emerge/__init__.py 2008-02-23 23:37:49 UTC (rev 9374)
649 +++ main/branches/prefix/pym/_emerge/__init__.py 2008-02-23 23:46:25 UTC (rev 9375)
650 @@ -1456,6 +1456,7 @@
651 self._reinstall_nodes = {}
652 self.mydbapi = {}
653 self.trees = {}
654 + self._trees_orig = trees
655 self.roots = {}
656 # Contains a filtered view of preferred packages that are selected
657 # from available repositories.
658 @@ -2150,6 +2151,35 @@
659 cpv=mykey, metadata=metadata, onlydeps=onlydeps)
660 args.append(PackageArg(arg=x, package=pkg,
661 root_config=root_config))
662 + elif x.startswith(os.path.sep):
663 + if not x.startswith(myroot):
664 + portage.writemsg(("\n\n!!! '%s' does not start with" + \
665 + " $ROOT.\n") % x, noiselevel=-1)
666 + return 0, []
667 + relative_path = x[len(myroot):]
668 + vartree = self._trees_orig[myroot]["vartree"]
669 + owner_cpv = None
670 + for cpv in vardb.cpv_all():
671 + self.spinner.update()
672 + cat, pf = portage.catsplit(cpv)
673 + if portage.dblink(cat, pf, myroot,
674 + pkgsettings, vartree=vartree).isowner(
675 + relative_path, myroot):
676 + owner_cpv = cpv
677 + break
678 + if owner_cpv is None:
679 + portage.writemsg(("\n\n!!! '%s' is not claimed " + \
680 + "by any package.\n") % x, noiselevel=-1)
681 + return 0, []
682 + slot = vardb.aux_get(owner_cpv, ["SLOT"])[0]
683 + if not slot:
684 + # portage now masks packages with missing slot, but it's
685 + # possible that one was installed by an older version
686 + atom = portage.cpv_getkey(owner_cpv)
687 + else:
688 + atom = "%s:%s" % (portage.cpv_getkey(owner_cpv), slot)
689 + args.append(AtomArg(arg=atom, atom=atom,
690 + root_config=root_config))
691 else:
692 if x in ("system", "world"):
693 x = SETPREFIX + x
694 @@ -2520,6 +2550,8 @@
695 portdb = self.roots[root].trees["porttree"].dbapi
696 dbs = self._filtered_trees[root]["dbs"]
697 for db, pkg_type, built, installed, db_keys in dbs:
698 + if installed:
699 + continue
700 match = db.match
701 if hasattr(db, "xmatch"):
702 cpv_list = db.xmatch("match-all", atom)
703 @@ -3352,7 +3384,7 @@
704 # have no parents and thus can not be part of a cycle.
705 while True:
706 root_nodes = mygraph.root_nodes(
707 - ignore_priority=DepPriority.SOFT)
708 + ignore_priority=DepPriority.MEDIUM_SOFT)
709 if not root_nodes:
710 break
711 for node in root_nodes:
712
713 Modified: main/branches/prefix/pym/portage/dbapi/porttree.py
714 ===================================================================
715 --- main/branches/prefix/pym/portage/dbapi/porttree.py 2008-02-23 23:37:49 UTC (rev 9374)
716 +++ main/branches/prefix/pym/portage/dbapi/porttree.py 2008-02-23 23:46:25 UTC (rev 9375)
717 @@ -27,9 +27,6 @@
718 class portdbapi(dbapi):
719 """this tree will scan a portage directory located at root (passed to init)"""
720 portdbapi_instances = []
721 - _non_category_dirs = ["distfiles", "eclass", "licenses",
722 - "local", "metadata", "packages", "profiles", "scripts"]
723 - _non_category_dirs = re.compile(r'^(%s)$' % "|".join(_non_category_dirs))
724 def __init__(self, porttree_root, mysettings=None):
725 portdbapi.portdbapi_instances.append(self)
726
727 @@ -39,6 +36,7 @@
728 else:
729 from portage import settings
730 self.mysettings = config(clone=settings)
731 + self._categories = set(self.mysettings.categories)
732 # This is strictly for use in aux_get() doebuild calls when metadata
733 # is generated by the depend phase. It's safest to use a clone for
734 # this purpose because doebuild makes many changes to the config
735 @@ -486,11 +484,8 @@
736 def cp_all(self):
737 "returns a list of all keys in our tree"
738 d = {}
739 - for oroot in self.porttrees:
740 - for x in listdir(oroot, EmptyOnError=1, ignorecvs=1, dirsonly=1):
741 - if not self._category_re.match(x) or \
742 - self._non_category_dirs.match(x):
743 - continue
744 + for x in self.mysettings.categories:
745 + for oroot in self.porttrees:
746 for y in listdir(oroot+"/"+x, EmptyOnError=1, ignorecvs=1, dirsonly=1):
747 if not self._pkg_dir_name_re.match(y) or \
748 y == "CVS":
749 @@ -520,7 +515,7 @@
750 self.xcache["match-all"][mycp] = cachelist
751 return cachelist[:]
752 mysplit = mycp.split("/")
753 - invalid_category = not self._category_re.match(mysplit[0])
754 + invalid_category = mysplit[0] not in self._categories
755 d={}
756 if mytree:
757 mytrees = [mytree]
758
759 Modified: main/branches/prefix/pym/portage/dbapi/vartree.py
760 ===================================================================
761 --- main/branches/prefix/pym/portage/dbapi/vartree.py 2008-02-23 23:37:49 UTC (rev 9374)
762 +++ main/branches/prefix/pym/portage/dbapi/vartree.py 2008-02-23 23:46:25 UTC (rev 9375)
763 @@ -63,8 +63,10 @@
764 """ Store the registry data to file. No need to call this if autocommit
765 was enabled.
766 """
767 - cPickle.dump(self._data, open(self._filename, "w"))
768 -
769 + f = atomic_ofstream(self._filename)
770 + cPickle.dump(self._data, f)
771 + f.close()
772 +
773 def register(self, cpv, slot, counter, paths):
774 """ Register new objects in the registry. If there is a record with the
775 same packagename (internally derived from cpv) and slot it is
776 @@ -160,7 +162,8 @@
777 else:
778 obj_dict[lib].append(mysplit[0])
779 mapfile = open(self._filename, "w")
780 - for lib in obj_dict:
781 + for lib in sorted(obj_dict):
782 + obj_dict[lib].sort()
783 mapfile.write(lib+" "+",".join(obj_dict[lib])+"\n")
784 mapfile.close()
785
786 @@ -909,6 +912,7 @@
787 self._installed_instance = None
788 self.contentscache = None
789 self._contents_inodes = None
790 + self._contents_basenames = None
791
792 def lockdb(self):
793 if self._lock_vdb:
794 @@ -1510,6 +1514,16 @@
795 if pkgfiles and destfile in pkgfiles:
796 return True
797 if pkgfiles:
798 + basename = os.path.basename(destfile)
799 + if self._contents_basenames is None:
800 + self._contents_basenames = set(
801 + os.path.basename(x) for x in pkgfiles)
802 + if basename not in self._contents_basenames:
803 + # This is a shortcut that, in most cases, allows us to
804 + # eliminate this package as an owner without the need
805 + # to examine inode numbers of parent directories.
806 + return False
807 +
808 # Use stat rather than lstat since we want to follow
809 # any symlinks to the real parent directory.
810 parent_path = os.path.dirname(destfile)
811 @@ -1545,7 +1559,6 @@
812 p_path_list = self._contents_inodes.get(
813 (parent_stat.st_dev, parent_stat.st_ino))
814 if p_path_list:
815 - basename = os.path.basename(destfile)
816 for p_path in p_path_list:
817 x = os.path.join(p_path, basename)
818 if x in pkgfiles:
819 @@ -2101,6 +2114,7 @@
820 for dblnk in others_in_slot:
821 dblnk.contentscache = None
822 dblnk._contents_inodes = None
823 + dblnk._contents_basenames = None
824
825 # If portage is reinstalling itself, remove the old
826 # version now since we want to use the temporary
827
828 Modified: main/branches/prefix/pym/portage/sets/__init__.py
829 ===================================================================
830 --- main/branches/prefix/pym/portage/sets/__init__.py 2008-02-23 23:37:49 UTC (rev 9374)
831 +++ main/branches/prefix/pym/portage/sets/__init__.py 2008-02-23 23:46:25 UTC (rev 9375)
832 @@ -9,6 +9,16 @@
833
834 SETPREFIX = "@"
835
836 +def get_boolean(options, name, default):
837 + if not name in options:
838 + return default
839 + elif options[name].lower() in ("1", "yes", "on", "true"):
840 + return True
841 + elif options[name].lower() in ("0", "no", "off", "false"):
842 + return False
843 + else:
844 + raise SetConfigError("invalid value '%s' for option '%s'" % (options[name], name))
845 +
846 class SetConfigError(Exception):
847 pass
848
849
850 Modified: main/branches/prefix/pym/portage/sets/dbapi.py
851 ===================================================================
852 --- main/branches/prefix/pym/portage/sets/dbapi.py 2008-02-23 23:37:49 UTC (rev 9374)
853 +++ main/branches/prefix/pym/portage/sets/dbapi.py 2008-02-23 23:46:25 UTC (rev 9375)
854 @@ -4,7 +4,7 @@
855
856 from portage.versions import catsplit, catpkgsplit
857 from portage.sets.base import PackageSet
858 -from portage.sets import SetConfigError
859 +from portage.sets import SetConfigError, get_boolean
860 from portage.dbapi.vartree import dblink
861 from portage.util import grabfile
862
863 @@ -65,10 +65,7 @@
864 _builderGetRepository = classmethod(_builderGetRepository)
865
866 def _builderGetVisible(cls, options):
867 - visible = options.get("only_visible", "true").lower()
868 - if visible not in ["1", "0", "yes", "no", "true", "false", "on", "off"]:
869 - raise SetConfigError("invalid value for only_visible: %s" % visible)
870 - return bool(visible in ["1", "yes", "true", "on"])
871 + return get_boolean(options, "only_visible", True)
872 _builderGetVisible = classmethod(_builderGetVisible)
873
874 def singleBuilder(cls, options, settings, trees):
875 @@ -176,9 +173,6 @@
876 self._setAtoms(self.mapPathsToAtoms(consumers))
877
878 def singleBuilder(cls, options, settings, trees):
879 - if options.get("debug", "true").lower() in ["true", "on", "1", "yes"]:
880 - debug = True
881 - else:
882 - debug = False
883 + debug = get_boolean(options, "debug", False)
884 return MissingLibraryConsumerSet(trees["vartree"].dbapi, debug=debug)
885 singleBuilder = classmethod(singleBuilder)
886
887 Modified: main/branches/prefix/pym/portage/sets/files.py
888 ===================================================================
889 --- main/branches/prefix/pym/portage/sets/files.py 2008-02-23 23:37:49 UTC (rev 9374)
890 +++ main/branches/prefix/pym/portage/sets/files.py 2008-02-23 23:46:25 UTC (rev 9375)
891 @@ -10,7 +10,7 @@
892 from portage.locks import lockfile, unlockfile
893 from portage import portage_gid
894 from portage.sets.base import PackageSet, EditablePackageSet
895 -from portage.sets import SetConfigError, SETPREFIX
896 +from portage.sets import SetConfigError, SETPREFIX, get_boolean
897 from portage.env.loaders import ItemFileLoader, KeyListFileLoader
898 from portage.env.validators import ValidAtomValidator
899 from portage import dep_getkey, cpv_getkey
900 @@ -20,12 +20,17 @@
901 class StaticFileSet(EditablePackageSet):
902 _operations = ["merge", "unmerge"]
903
904 - def __init__(self, filename):
905 + def __init__(self, filename, greedy=False, dbapi=None):
906 super(StaticFileSet, self).__init__()
907 self._filename = filename
908 self._mtime = None
909 self.description = "Package set loaded from file %s" % self._filename
910 self.loader = ItemFileLoader(self._filename, self._validate)
911 + if greedy and not dbapi:
912 + self.errors.append("%s configured as greedy set, but no dbapi instance passed in constructor" % self._filename)
913 + greedy = False
914 + self.greedy = greedy
915 + self.dbapi = dbapi
916
917 metadata = grabfile(self._filename + ".metadata")
918 key = None
919 @@ -69,13 +74,26 @@
920 raise
921 del e
922 data = {}
923 - self._setAtoms(data.keys())
924 + if self.greedy:
925 + atoms = []
926 + for a in data.keys():
927 + matches = self.dbapi.match(a)
928 + for cpv in matches:
929 + atoms.append("%s:%s" % (cpv_getkey(cpv),
930 + self.dbapi.aux_get(cpv, ["SLOT"])[0]))
931 + # In addition to any installed slots, also try to pull
932 + # in the latest new slot that may be available.
933 + atoms.append(a)
934 + else:
935 + atoms = data.keys()
936 + self._setAtoms(atoms)
937 self._mtime = mtime
938
939 def singleBuilder(self, options, settings, trees):
940 if not "filename" in options:
941 raise SetConfigError("no filename specified")
942 - return ConfigFileSet(options[filename])
943 + greedy = get_boolean(options, "greedy", True)
944 + return StaticFileSet(options["filename"], greedy=greedy, dbapi=trees["vartree"].dbapi)
945 singleBuilder = classmethod(singleBuilder)
946
947 def multiBuilder(self, options, settings, trees):
948 @@ -84,13 +102,14 @@
949 name_pattern = options.get("name_pattern", "sets/$name")
950 if not "$name" in name_pattern and not "${name}" in name_pattern:
951 raise SetConfigError("name_pattern doesn't include $name placeholder")
952 + greedy = get_boolean(options, "greedy", True)
953 if os.path.isdir(directory):
954 for filename in os.listdir(directory):
955 if filename.endswith(".metadata"):
956 continue
957 myname = name_pattern.replace("$name", filename)
958 myname = myname.replace("${name}", filename)
959 - rValue[myname] = StaticFileSet(os.path.join(directory, filename))
960 + rValue[myname] = StaticFileSet(os.path.join(directory, filename), greedy=greedy, dbapi=trees["vartree"].dbapi)
961 return rValue
962 multiBuilder = classmethod(multiBuilder)
963
964 @@ -108,7 +127,7 @@
965 def singleBuilder(self, options, settings, trees):
966 if not "filename" in options:
967 raise SetConfigError("no filename specified")
968 - return ConfigFileSet(options[filename])
969 + return ConfigFileSet(options["filename"])
970 singleBuilder = classmethod(singleBuilder)
971
972 def multiBuilder(self, options, settings, trees):
973
974 Modified: main/branches/prefix/pym/portage/sets/security.py
975 ===================================================================
976 --- main/branches/prefix/pym/portage/sets/security.py 2008-02-23 23:37:49 UTC (rev 9374)
977 +++ main/branches/prefix/pym/portage/sets/security.py 2008-02-23 23:46:25 UTC (rev 9375)
978 @@ -7,6 +7,7 @@
979 from portage.util import grabfile, write_atomic
980 from portage.sets.base import PackageSet
981 from portage.versions import catpkgsplit, pkgcmp
982 +from portage.sets import get_boolean
983
984 __all__ = ["SecuritySet", "NewGlsaSet", "NewAffectedSet", "AffectedSet"]
985
986 @@ -69,11 +70,7 @@
987 myglsa.inject()
988
989 def singleBuilder(cls, options, settings, trees):
990 - if "use_emerge_resoler" in options \
991 - and options.get("use_emerge_resolver").lower() in ["1", "yes", "true", "on"]:
992 - least_change = False
993 - else:
994 - least_change = True
995 + least_change = not get_boolean(options, "use_emerge_resolver", False)
996 return cls(settings, trees["vartree"].dbapi, trees["porttree"].dbapi, least_change=least_change)
997 singleBuilder = classmethod(singleBuilder)
998
999
1000 Modified: main/branches/prefix/pym/portage/util.py
1001 ===================================================================
1002 --- main/branches/prefix/pym/portage/util.py 2008-02-23 23:37:49 UTC (rev 9374)
1003 +++ main/branches/prefix/pym/portage/util.py 2008-02-23 23:46:25 UTC (rev 9375)
1004 @@ -169,7 +169,7 @@
1005 if final_dict is None:
1006 final_dict = {}
1007 for y in mydict.keys():
1008 - if mydict[y]:
1009 + if True:
1010 if final_dict.has_key(y) and (incremental or (y in incrementals)):
1011 final_dict[y] += " "+mydict[y][:]
1012 else:
1013
1014 Modified: main/branches/prefix/pym/repoman/utilities.py
1015 ===================================================================
1016 --- main/branches/prefix/pym/repoman/utilities.py 2008-02-23 23:37:49 UTC (rev 9374)
1017 +++ main/branches/prefix/pym/repoman/utilities.py 2008-02-23 23:46:25 UTC (rev 9375)
1018 @@ -317,12 +317,12 @@
1019
1020 location = normalize_path(location)
1021
1022 - path_ids = set()
1023 + path_ids = {}
1024 p = location
1025 s = None
1026 while True:
1027 s = os.stat(p)
1028 - path_ids.add((s.st_dev, s.st_ino))
1029 + path_ids[(s.st_dev, s.st_ino)] = p
1030 if p == "/":
1031 break
1032 p = os.path.dirname(p)
1033 @@ -335,10 +335,12 @@
1034 s = os.stat(overlay)
1035 except OSError:
1036 continue
1037 - overlay_id = (s.st_dev, s.st_ino)
1038 + overlay = path_ids.get((s.st_dev, s.st_ino))
1039 + if overlay is None:
1040 + continue
1041 if overlay[-1] != "/":
1042 overlay += "/"
1043 - if overlay_id in path_ids:
1044 + if True:
1045 portdir_overlay = overlay
1046 subdir = location[len(overlay):]
1047 if subdir and subdir[-1] != "/":
1048
1049 --
1050 gentoo-commits@l.g.o mailing list