Re: [gentoo-dev] orphaned files on system?

[Daniel Armyr]

Hm, gave it a run. The code looks pretty neat. I havn't studied it 
enough to know exactly what it does, but it seem like it has some real 
nifty features. Unfortunately It seems awfully slow and heavy on the 
system. My first run crashed X, I think. The second hung up when caching 
vanilla-sources. Third time worked though, and I like the brevity of the 
list it produces. A check on the used processor time says it is about 4 
times slower than a simple sh script that was presented a while back 
which does no filtering of the output. It simply prints all files on the 
system ot mentioned in /var/db/pkg. Either way, looks like a good 
program, but I think it can be worth the attempt to try to find ways to 
make it faster.
//Daniel Armyr

leon j. breedt wrote:

>hi,
>
>i use the attached script to scan for unpackaged files on my filesystem,
>and found quite a few in /etc, /usr/lib, /usr/X11R6 as well as the
>expected places. most of them were symlinks, the intention being fairly
>obvious (like the NVIDIA OpenGL stuff).
>
>but i was hoping someone could explain why files like /etc/make.conf, /etc/csh.env,
>/etc/env.d/05gcc and /usr/include/awk/acconfig.h didn't belong to any package.
>
>i run the script with:
>
>$ ./gtfilelint -v -C gtfilelint.conf -o orphans.list
>
>use -h to see available params. multiple -v increases verbosity.
>
>if you have a system with lots of packages, its going to take some time,
>as it caches all the /var/db/pkg/**/CONTENTS entries in a Berkeley hashdb
>for quick lookups, then runs /usr/bin/find on /, and compares results. exclusions
>to the find output are made by adding python re module regexes to
>gtfilelint.conf.
>
>if you run it as user, you may get some error output from find about permissions.
>you will want to specify a config file, otherwise you'll get a lot of stuff
>you probably don't care about.
>
>hope someone finds this useful
>
>leon
>
>
>
>------------------------------------------------------------------------
>
>#!/usr/bin/env python
>#
># Finds files on Gentoo Linux systems that do not belong
># to any installed package.
>#
># Released under the GNU GPL.
>#
># (C) Copyright 2003 Leon J. Breedt
>#
># $Id$
>
>import dbhash
>import getopt
>import os
>import os.path
>import re
>import string
>import sys
>
>TRUE = 1
>FALSE = 0
>version = '0.1.1'
>configfile = '/etc/gtfilelint.conf'
>dbdir = '/var/db/pkg'
>cachefile = '/tmp/gtfilelint.db'
>outputfile = None
>warnmissing = FALSE
>findcmd = 'find / -print'
>exclusions = []
>verbosity = 0
>cachedb = None
>
>def verb(msg, level=1):
>    if verbosity >= level:
>        sys.stderr.write('-- %s\n' % msg)
>
>def vverb(msg):
>    verb(msg, 2)
>
>def info(msg):
>    sys.stderr.write('>> %s\n' % msg)
>
>def error(msg):
>    sys.stderr.write('error: %s\n' % msg)
>    sys.exit(1)
>
>def warn(msg):
>    sys.stderr.write('warning: %s\n' % msg)
>
>def usage():
>    print 'usage: %s [options]' % sys.argv[0]
>    print 'options:'
>    print '-h|--help       display this message'
>    print '-V|--version    print program version and exit'
>    print '-v|--verbose    print verbose messages about what is being done'
>    print '-d|--dbdir      directory containing package database (default: %s)' % dbdir
>    print '-c|--cachefile  file to place temporary cache in (default: %s)' % cachefile
>    print '-C|--configfile configuration file (default: %s)' % configfile
>    print '-o|--outputfile file to print orphan list to (default: stdout)'
>    print '--warnmissing   warn if files declared in CONTENTS don\'t exist'
>
>def parse_cmdline():
>    global verbosity, dbdir, cachefile, configfile, outputfile, warnmissing
>    opts, args = getopt.getopt(sys.argv[1:], "hVvd:c:C:o:", ["help", "version", "verbose", "dbdir=", "cachefile=", "configfile=", "outputfile=", "warnmissing"])
>    for opt, arg in opts:
>        if opt in ("-h", "--help"):
>            usage()
>            sys.exit(0)
>        if opt in ("-V", "--version"):
>            print version
>            sys.exit(0)
>        if opt in ("-v", "--verbose"):
>            verbosity = verbosity + 1
>        if opt in ("-d", "--dbdir"):
>            dbdir = arg
>        if opt in ("-c", "--cachefile"):
>            cachefile = arg
>        if opt in ("-C", "--configfile"):
>            configfile = arg
>        if opt in ("-o", "--outputfile"):
>            outputfile = arg
>        if opt == "--warnmissing":
>            warnmissing = TRUE
>
>def parse_config():
>    if not os.path.exists(configfile) or not os.access(configfile, os.R_OK):
>        warn('missing configfile "%s"' % configfile)
>        return
>    fp = open(configfile, 'r')
>    for line in fp.readlines():
>        line = string.strip(line)
>        if len(line) == 0:
>            continue
>        if line[0] == '#':
>            continue
>        exclusions.append(re.compile(line))
>        verb('adding "%s" to list of exclusion regular expressions' % line)
>    fp.close()
>
>def cache_package_files(package, packagepath):
>    verb('caching contents of "%s"' % package)
>    fp = open(packagepath + '/CONTENTS')
>    lineno = 0
>    for line in fp.readlines():
>        lineno = lineno + 1
>        line = string.strip(line)
>        if len(line) == 0:
>            continue
>        key = None
>        m = re.match(r"^dir (\S.*)$", line)
>        if m:
>            key = m.group(1)
>            m = None
>        else:
>            m = re.match(r"^obj (\S.*) (\S+) (\d+)\s*$", line)
>        if m:
>            key = m.group(1)
>            m = None
>        m = re.match(r"^sym (\S.*) -> .*$", line)
>        if m:
>            key = m.group(1)
>        if key != None:
>            if not os.path.exists(key) and warnmissing:
>                warn('%s: "%s" does not exist on filesystem, ignoring' % (package, key))
>            vverb('caching "%s"' % key)
>            cachedb[key] = ''
>        else:
>            vverb('key is None for "%s" CONTENTS line %d' % (package, lineno))
>    fp.close()
>
>def scan_group_packages(group, grouppath):
>    packages = os.listdir(grouppath)
>    packages.sort()
>    verb('found %d packages in group "%s"' % (len(packages), group))
>    for package in packages:
>        packagepath = grouppath + '/' + package
>        cache_package_files(package, packagepath)
>
>def create_system_filelist():
>    info('scanning all files on system')
>    sout = os.popen(findcmd, 'r')
>    verb('reading paths from "%s"' % findcmd)
>    paths = sout.readlines()
>    orphans = 0
>    rptfp = None
>    for path in paths:
>        path = string.strip(path)
>        if len(path) == 0:
>            continue
>        if path[0] != '/':
>            warn('ignoring relative path "%s"' % path)
>            continue
>        matched = FALSE
>        for exre in exclusions:
>            if exre.match(path):
>                matched = TRUE
>                break
>        if matched:
>            vverb('"%s" matched exclusion regex, ignoring' % path)
>            continue
>        if not cachedb.has_key(path):
>            if orphans == 0:
>                if outputfile:
>                    info('writing orphaned file list [%s]' % outputfile)
>                    rptfp = open(outputfile, 'w+')
>                else:
>                    info('orphaned files:')
>                    rptfp = sys.stdout
>                rptfp.flush()
>            orphans = orphans + 1
>            rptfp.write('%s\n' % path)
>            rptfp.flush()
>    if rptfp:
>        rptfp.close()
>    sout.close()
>    if orphans > 0:
>        info('%d orphaned file(s) found' % orphans)
>    else:
>        info('no orphaned files on system')
>
># Main
>try:
>    parse_cmdline()
>    parse_config()
>    info('creating packaged files cache [%s]' % cachefile)
>    cachedb = dbhash.open(cachefile, 'n')
>    try:
>        groups = os.listdir(dbdir)
>        groups.sort()
>        for group in groups:
>            grouppath = dbdir + '/' + group
>            scan_group_packages(group, grouppath)
>        create_system_filelist()
>    finally:
>        if cachedb:
>            cachedb.close()
>            os.unlink(cachefile)
>except KeyboardInterrupt: 1
>except:
>    raise
>
>
>------------------------------------------------------------------------
>
># we don't really care about these dynamic paths
>^/var/log/.*
>^/var/db/.*
>^/var/spool/.*
>^/var/tmp/.*
>^/var/lib/.*
>^/var/cache/.*
>^/var/run/.*
>
># / is not owned by any package
>^/$
>
># don't care about root's config files
>^/root/.*
>
># /usr/local is typically just user compiled stuff,
># don't care about it -- this list from baselayout
>^/usr/local/bin/.*
>^/usr/local/doc$
>^/usr/local/lib/.*
>^/usr/local/man$
>^/usr/local/src/.*
>^/usr/local/sbin/.*
>^/usr/local/games/.*
>^/usr/local/share/doc/.*
>^/usr/local/share/man/.*
>^/usr/local/share/.*
>
># what is /lib/dev-state? dunno...but the dir is in
># baselayout, even if the files arent
>^/lib/dev-state/.*
>
># devices aren't that important to us...the packaged
># files will not be visible anyway due to devfs
>^/dev/.*
>
># anyone packaging anything into /tmp should be shot
>^/tmp/.*
>
># portage tree we don't care about either
>^/usr/portage$
>^/usr/portage/.*
>
># mountpoints shouldn't have package files installed in them
>^/mnt/.*
>
># system filesystems should be ignored
>^/proc/.*
>^/sys/.*
>
># USER CUSTOMIZATIONS
>^/data
>^/data/.*
>^/cdrom.*
>^/windata
>^/windata/.*
>
>


--
gentoo-dev@g.o mailing list