Re: [gentoo-dev] "Updating Portage Cache" optimizations

[ross b girshick]

On Tue, 24 Jun 2003, Seemant Kulleen wrote:

> no I meant rsync on the local filesystem

Ah, I see what you mean. Here's a patch to do that. The first time it 
runs, it takes my machine ~ 3 minutes (compared to 4.5 in past) and after 
that usually <1 minute (30 seconds if there aren't many updates to to the 
tree).

If people are interested in testing this out and/or providing feedback 
please do. I've been using it all day without anything breaking (i wish 
you luck too) ;).

Ross Girshick

--- emerge.orig 2003-06-24 15:51:10.000000000 -0400
+++ emerge      2003-06-24 17:40:26.000000000 -0400
@@ -1656,31 +1656,55 @@
                sys.exit(1)
        if os.path.exists(myportdir+"/metadata/cache"):
                print "\n>>> Updating Portage cache...  ",
-               os.umask(0002)
-               if os.path.exists(portage.dbcachedir):
-                       portage.spawn("rm -Rf "+portage.dbcachedir,free=1)
-               try:
-                       os.mkdir(portage.dbcachedir)
-                       os.chown(portage.dbcachedir, os.getuid(), portage.portage_gid)
-                       os.chmod(portage.dbcachedir, 06775)
-                       os.umask(002)
-               except:
-                       pass
-               mynodes=portage.portdb.cp_all()
+               sys.stdout.flush()
+               # We shouldn't have to worry about this because when portage is imported dbcachedir is created if it's missing
+               if not os.path.exists(portage.dbcachedir):
+                       print "!!! Cache Directory " + portage.dbcachedir + " does not exist. Re-running emerge should fix this"
+                       sys.exit(1)
+               # XXX If we don't --delete, then we don't have to regenerate the cache files...what danger does this create?
+               # maybe it's sufficient to use --delete only every N syncs??? XXX
+               #update_cache_command = "/usr/bin/rsync -rlptD --delete --delete-after "...
+               update_cache_command = "/usr/bin/rsync -rlptD " + myportdir + "/metadata/cache/ " + portage.dbcachedir
+               exitcode = portage.spawn(update_cache_command, free=1)
+               # print update_cache_command
+               if (exitcode > 0):
+                       ## more error info might be good
+                       print "!!! local rsync error (cache update failed): " + exitcode + "\n"
+                       sys.exit(1)
+               mynodes = portage.portdb.cp_all()
                for x in mynodes:
-                       myxsplit=x.split("/")
-                       if not os.path.exists(portage.dbcachedir+"/"+myxsplit[0]):
-                               os.mkdir(portage.dbcachedir+"/"+myxsplit[0])
-                               os.chown(portage.dbcachedir+"/"+myxsplit[0], os.getuid(), portage.portage_gid)
-                               os.chmod(portage.dbcachedir+"/"+myxsplit[0], 06775)
-                       mymatches=portage.portdb.xmatch("match-all",x)
+                       mymatches = portage.portdb.xmatch("match-all", x)
                        for y in mymatches:
                                update_spinner()
+                               mydbkey = portage.dbcachedir+y
+                               myebuild, in_overlay = portage.portdb.findname2(y)
+                               myebuild_mtime = os.stat(myebuild)[ST_MTIME]
                                try:
-                                       ignored=portage.portdb.aux_get(y,[],metacachedir=myportdir+"/metadata/cache")
-                               except:
-                                       pass
-               portage.spawn("chmod -R g+rw "+portage.dbcachedir, free=1)
+                                       mydbkeystat = os.stat(mydbkey)
+                                       if mydbkeystat[ST_SIZE] == 0 or myebuild_mtime != mydbkeystat[ST_MTIME]:
+                                               doregen = 1
+                                       else:
+                                               doregen = 0
+                               except OSError:
+                                       doregen = 1
+
+                               if doregen:
+                                       ##print "doregen " + mydbkey + "\n"
+                                       try:
+                                               os.unlink(mydbkey)
+                                       except:
+                                               pass
+                                       # regenerate the dep cache file using doebuild interface
+                                       if portage.doebuild(myebuild, "depend", "/"):
+                                               #depend returned non-zero exit code...
+                                               sys.stderr.write(str(red("\nemerge sync:")+" (0) Error in "+y+" ebuild.\n"
+                                               "               Check for syntax error or corruption in the ebuild. (--debug)\n\n"))
+                                       try:
+                                               os.utime(mydbkey, (myebuild_mtime, myebuild_mtime))
+                                       except (IOError, OSError):
+                                               sys.stderr.write(str(red("\nemerge sync:")+" (1) Error in "+y+" ebuild.\n"
+                                               "               Check for syntax error or corruption in the ebuild. (--debug)\n\n"))
+               ##portage.spawn("chmod -R g+rw "+portage.dbcachedir, free=1)
                sys.stdout.write("\b\b  ...done!\n\n")
                sys.stdout.flush()



--
gentoo-dev@g.o mailing list