1 |
Author: zmedico |
2 |
Date: 2009-04-30 06:51:30 +0000 (Thu, 30 Apr 2009) |
3 |
New Revision: 13439 |
4 |
|
5 |
Added: |
6 |
main/branches/2.1.6/bin/egencache |
7 |
Log: |
8 |
Bug #261377 - Add a new 'egencache' tool to generate metadata cache for |
9 |
distribution. It only have the most basic functionality now, and more |
10 |
features will be added later. |
11 |
|
12 |
Usage: egencache [options] --update [atom] ... |
13 |
|
14 |
Options: |
15 |
-h, --help show this help message and exit |
16 |
--update update metadata/cache/ (generate as necessary) |
17 |
--cache-dir=CACHE_DIR |
18 |
location of the metadata cache |
19 |
--config-root=CONFIG_ROOT |
20 |
location of portage config files |
21 |
--jobs=JOBS max ebuild processes to spawn |
22 |
--load-average=LOAD_AVERAGE |
23 |
max load allowed when spawning multiple jobs |
24 |
(trunk r13260) |
25 |
|
26 |
Copied: main/branches/2.1.6/bin/egencache (from rev 13260, main/trunk/bin/egencache) |
27 |
=================================================================== |
28 |
--- main/branches/2.1.6/bin/egencache (rev 0) |
29 |
+++ main/branches/2.1.6/bin/egencache 2009-04-30 06:51:30 UTC (rev 13439) |
30 |
@@ -0,0 +1,205 @@ |
31 |
+#!/usr/bin/python |
32 |
+# Copyright 2009 Gentoo Foundation |
33 |
+# Distributed under the terms of the GNU General Public License v2 |
34 |
+# $Id$ |
35 |
+ |
36 |
+import sys |
37 |
+# This block ensures that ^C interrupts are handled quietly. |
38 |
+try: |
39 |
+ import signal |
40 |
+ |
41 |
+ def exithandler(signum,frame): |
42 |
+ signal.signal(signal.SIGINT, signal.SIG_IGN) |
43 |
+ signal.signal(signal.SIGTERM, signal.SIG_IGN) |
44 |
+ sys.exit(1) |
45 |
+ |
46 |
+ signal.signal(signal.SIGINT, exithandler) |
47 |
+ signal.signal(signal.SIGTERM, exithandler) |
48 |
+ |
49 |
+except KeyboardInterrupt: |
50 |
+ sys.exit(1) |
51 |
+ |
52 |
+import logging |
53 |
+import optparse |
54 |
+import os |
55 |
+import portage |
56 |
+import _emerge |
57 |
+from portage.cache.cache_errors import CacheError |
58 |
+from portage.util import writemsg_level |
59 |
+ |
60 |
+def parse_args(args): |
61 |
+ usage = "egencache [options] --update [atom] ..." |
62 |
+ parser = optparse.OptionParser(usage=usage) |
63 |
+ parser.add_option("--update", |
64 |
+ action="store_true", |
65 |
+ help="update metadata/cache/ (generate as necessary)") |
66 |
+ parser.add_option("--cache-dir", |
67 |
+ help="location of the metadata cache", |
68 |
+ dest="cache_dir") |
69 |
+ parser.add_option("--config-root", |
70 |
+ help="location of portage config files", |
71 |
+ dest="config_root") |
72 |
+ parser.add_option("--jobs", |
73 |
+ action="store", |
74 |
+ help="max ebuild processes to spawn") |
75 |
+ parser.add_option("--load-average", |
76 |
+ action="store", |
77 |
+ help="max load allowed when spawning multiple jobs", |
78 |
+ dest="load_average") |
79 |
+ options, args = parser.parse_args(args) |
80 |
+ |
81 |
+ if not options.update: |
82 |
+ parser.error('No action specified (--update ' + \ |
83 |
+ 'is the only available action)') |
84 |
+ |
85 |
+ if options.config_root is not None and \ |
86 |
+ not os.path.isdir(options.config_root): |
87 |
+ parser.error("Not a directory: --config-root='%s'" % \ |
88 |
+ (options.config_root,)) |
89 |
+ |
90 |
+ if options.cache_dir is not None and not os.path.isdir(options.cache_dir): |
91 |
+ parser.error("Not a directory: --cache-dir='%s'" % \ |
92 |
+ (options.cache_dir,)) |
93 |
+ |
94 |
+ for atom in args: |
95 |
+ try: |
96 |
+ atom = portage.dep.Atom(atom) |
97 |
+ except portage.exception.InvalidAtom: |
98 |
+ parser.error('Invalid atom: %s' % (atom,)) |
99 |
+ |
100 |
+ if str(atom) != atom.cp: |
101 |
+ parser.error('Atom is too specific: %s' % (atom,)) |
102 |
+ |
103 |
+ return options, args |
104 |
+ |
105 |
+class GenCache(object): |
106 |
+ def __init__(self, portdb, cp_iter=None, max_jobs=None, max_load=None): |
107 |
+ self._portdb = portdb |
108 |
+ # We can globally cleanse stale cache only if we |
109 |
+ # iterate over every single cp. |
110 |
+ self._global_cleanse = cp_iter is None |
111 |
+ if cp_iter is not None: |
112 |
+ self._cp_set = set(cp_iter) |
113 |
+ cp_iter = iter(self._cp_set) |
114 |
+ else: |
115 |
+ self._cp_set = None |
116 |
+ self._regen = _emerge.MetadataRegen(portdb, cp_iter=cp_iter, |
117 |
+ consumer=self._metadata_callback, |
118 |
+ max_jobs=max_jobs, max_load=max_load) |
119 |
+ self.returncode = os.EX_OK |
120 |
+ metadbmodule = portdb.mysettings.load_best_module("portdbapi.metadbmodule") |
121 |
+ self._trg_cache = metadbmodule(portdb.porttree_root, |
122 |
+ "metadata/cache", portage.auxdbkeys[:]) |
123 |
+ self._existing_nodes = set() |
124 |
+ |
125 |
+ def _metadata_callback(self, cpv, ebuild_path, repo_path, metadata): |
126 |
+ self._existing_nodes.add(cpv) |
127 |
+ if metadata is not None: |
128 |
+ # TODO: Implement a workaround for bug 139134 here. The cache |
129 |
+ # should be able to optionally raise an exception in order to |
130 |
+ # indicate any mtime + size collisions that will prevent rsync |
131 |
+ # from detecting changes. These exceptions will be handled by |
132 |
+ # bumping the mtime on the ebuild (and the corresponding cache |
133 |
+ # entry). |
134 |
+ if metadata.get('EAPI') == '0': |
135 |
+ del metadata['EAPI'] |
136 |
+ try: |
137 |
+ self._trg_cache[cpv] = metadata |
138 |
+ except CacheError, ce: |
139 |
+ writemsg_level( |
140 |
+ "%s writing target: %s\n" % (cpv, ce), |
141 |
+ level=logging.ERROR, noiselevel=-1) |
142 |
+ |
143 |
+ def run(self): |
144 |
+ self._regen.run() |
145 |
+ self.returncode |= self._regen.returncode |
146 |
+ |
147 |
+ trg_cache = self._trg_cache |
148 |
+ dead_nodes = None |
149 |
+ if self._global_cleanse: |
150 |
+ try: |
151 |
+ dead_nodes = set(trg_cache.iterkeys()) |
152 |
+ except CacheError, ce: |
153 |
+ self.returncode |= 1 |
154 |
+ writemsg_level( |
155 |
+ "Error listing cache entries for " + \ |
156 |
+ "'%s/metadata/cache': %s, continuing...\n" % \ |
157 |
+ (self._portdb.porttree_root, ce), |
158 |
+ level=logging.ERROR, noiselevel=-1) |
159 |
+ |
160 |
+ else: |
161 |
+ cp_set = self._cp_set |
162 |
+ cpv_getkey = portage.cpv_getkey |
163 |
+ try: |
164 |
+ dead_nodes = set(cpv for cpv in \ |
165 |
+ trg_cache.iterkeys() \ |
166 |
+ if cpv_getkey(cpv) in cp_set) |
167 |
+ except CacheError, ce: |
168 |
+ self.returncode |= 1 |
169 |
+ writemsg_level( |
170 |
+ "Error listing cache entries for " + \ |
171 |
+ "'%s/metadata/cache': %s, continuing...\n" % \ |
172 |
+ (self._portdb.porttree_root, ce), |
173 |
+ level=logging.ERROR, noiselevel=-1) |
174 |
+ |
175 |
+ if dead_nodes: |
176 |
+ dead_nodes.difference_update(self._existing_nodes) |
177 |
+ for k in dead_nodes: |
178 |
+ try: |
179 |
+ del trg_cache[k] |
180 |
+ except KeyError: |
181 |
+ pass |
182 |
+ except CacheError: |
183 |
+ self.returncode |= 1 |
184 |
+ |
185 |
+ if not trg_cache.autocommits: |
186 |
+ try: |
187 |
+ trg_cache.commit() |
188 |
+ except CacheError, ce: |
189 |
+ self.returncode |= 1 |
190 |
+ writemsg_level( |
191 |
+ "committing target: %s\n" % (ce,), |
192 |
+ level=logging.ERROR, noiselevel=-1) |
193 |
+ |
194 |
+def egencache_main(args): |
195 |
+ options, args = parse_args(args) |
196 |
+ |
197 |
+ config_root = options.config_root |
198 |
+ if config_root is None: |
199 |
+ config_root = '/' |
200 |
+ |
201 |
+ # The calling environment is ignored, so the program is |
202 |
+ # completely controlled by commandline arguments. |
203 |
+ env = {} |
204 |
+ |
205 |
+ # TODO: Implement --repo for choosing a repo. |
206 |
+ env['PORTDIR_OVERLAY'] = '' |
207 |
+ |
208 |
+ if options.cache_dir is not None: |
209 |
+ env['PORTAGE_DEPCACHEDIR'] = options.cache_dir |
210 |
+ |
211 |
+ settings = portage.config(config_root=config_root, |
212 |
+ target_root='/', env=env) |
213 |
+ |
214 |
+ if 'metadata-transfer' not in settings.features: |
215 |
+ writemsg_level("ecachegen: error: " + \ |
216 |
+ "FEATURES=metadata-transfer is not enabled\n", |
217 |
+ level=logging.ERROR, noiselevel=-1) |
218 |
+ return 1 |
219 |
+ |
220 |
+ portdb = portage.portdbapi(settings["PORTDIR"], mysettings=settings) |
221 |
+ |
222 |
+ cp_iter = None |
223 |
+ if args: |
224 |
+ cp_iter = iter(args) |
225 |
+ |
226 |
+ gen_cache = GenCache(portdb, cp_iter=cp_iter, |
227 |
+ max_jobs=options.jobs, |
228 |
+ max_load=options.load_average) |
229 |
+ gen_cache.run() |
230 |
+ return gen_cache.returncode |
231 |
+ |
232 |
+if __name__ == "__main__": |
233 |
+ portage._disable_legacy_globals() |
234 |
+ portage.util.noiselimit = -1 |
235 |
+ sys.exit(egencache_main(sys.argv[1:])) |