1 |
Author: zmedico |
2 |
Date: 2008-07-03 06:06:34 +0000 (Thu, 03 Jul 2008) |
3 |
New Revision: 10902 |
4 |
|
5 |
Modified: |
6 |
main/trunk/pym/_emerge/__init__.py |
7 |
main/trunk/pym/portage/__init__.py |
8 |
Log: |
9 |
Implement asynchronous binary package extraction, so that the scheduler |
10 |
can run while a package is extracting in the background. |
11 |
|
12 |
|
13 |
Modified: main/trunk/pym/_emerge/__init__.py |
14 |
=================================================================== |
15 |
--- main/trunk/pym/_emerge/__init__.py 2008-07-03 04:24:42 UTC (rev 10901) |
16 |
+++ main/trunk/pym/_emerge/__init__.py 2008-07-03 06:06:34 UTC (rev 10902) |
17 |
@@ -24,6 +24,7 @@ |
18 |
import fcntl |
19 |
import select |
20 |
import shlex |
21 |
+import shutil |
22 |
import urlparse |
23 |
import weakref |
24 |
import gc |
25 |
@@ -1565,7 +1566,7 @@ |
26 |
|
27 |
kwargs["fd_pipes"] = fd_pipes |
28 |
kwargs["returnpid"] = True |
29 |
- del kwargs["logfile"] |
30 |
+ kwargs.pop("logfile", None) |
31 |
|
32 |
retval = portage.process.spawn(self.args, **kwargs) |
33 |
|
34 |
@@ -2104,6 +2105,8 @@ |
35 |
pkg_count = self.pkg_count |
36 |
scheduler = self.scheduler |
37 |
settings = self.settings |
38 |
+ settings.setcpv(pkg) |
39 |
+ debug = settings.get("PORTAGE_DEBUG") == "1" |
40 |
|
41 |
# The prefetcher has already completed or it |
42 |
# could be running now. If it's running now, |
43 |
@@ -2167,16 +2170,122 @@ |
44 |
"portage", pkg.category, pkg.pf) |
45 |
build_dir = EbuildBuildDir(dir_path=dir_path, |
46 |
pkg=pkg, settings=settings) |
47 |
+ image_dir = os.path.join(dir_path, "image") |
48 |
+ infloc = os.path.join(dir_path, "build-info") |
49 |
|
50 |
+ fd_pipes = { |
51 |
+ 0 : sys.stdin.fileno(), |
52 |
+ 1 : sys.stdout.fileno(), |
53 |
+ 2 : sys.stderr.fileno(), |
54 |
+ } |
55 |
+ |
56 |
try: |
57 |
build_dir.lock() |
58 |
- merge = BinpkgMerge(find_blockers=find_blockers, |
59 |
- ldpath_mtimes=ldpath_mtimes, pkg=pkg, pretend=opts.pretend, |
60 |
- pkg_path=pkg_path, settings=settings) |
61 |
+ |
62 |
+ root_config = self.pkg.root_config |
63 |
+ ebuild_path = os.path.join(infloc, pkg.pf + ".ebuild") |
64 |
+ cleanup = 1 |
65 |
+ tree = "bintree" |
66 |
+ mydbapi = root_config.trees[tree].dbapi |
67 |
+ |
68 |
+ retval = portage.doebuild(ebuild_path, "clean", |
69 |
+ root_config.root, settings, debug, cleanup=cleanup, |
70 |
+ mydbapi=mydbapi, tree=tree) |
71 |
+ if retval != os.EX_OK: |
72 |
+ return retval |
73 |
+ |
74 |
+ try: |
75 |
+ shutil.rmtree(dir_path) |
76 |
+ except (IOError, OSError), e: |
77 |
+ if e.errno != errno.ENOENT: |
78 |
+ raise |
79 |
+ del e |
80 |
+ |
81 |
+ # This initializes PORTAGE_LOG_FILE. |
82 |
+ portage.prepare_build_dirs(root_config.root, settings, cleanup) |
83 |
+ |
84 |
+ dir_mode = 0755 |
85 |
+ for mydir in (dir_path, image_dir, infloc): |
86 |
+ portage.util.ensure_dirs(mydir, uid=portage.data.portage_uid, |
87 |
+ gid=portage.data.portage_gid, mode=dir_mode) |
88 |
+ |
89 |
+ portage.writemsg_stdout(">>> Extracting info\n") |
90 |
+ |
91 |
+ pkg_xpak = portage.xpak.tbz2(pkg_path) |
92 |
+ check_missing_metadata = ("CATEGORY", "PF") |
93 |
+ missing_metadata = set() |
94 |
+ for k in check_missing_metadata: |
95 |
+ v = pkg_xpak.getfile(k) |
96 |
+ if not v: |
97 |
+ missing_metadata.add(k) |
98 |
+ |
99 |
+ pkg_xpak.unpackinfo(infloc) |
100 |
+ for k in missing_metadata: |
101 |
+ if k == "CATEGORY": |
102 |
+ v = pkg.category |
103 |
+ elif k == "PF": |
104 |
+ v = pkg.pf |
105 |
+ else: |
106 |
+ continue |
107 |
+ |
108 |
+ f = open(os.path.join(infloc, k), 'wb') |
109 |
+ try: |
110 |
+ f.write(v + "\n") |
111 |
+ finally: |
112 |
+ f.close() |
113 |
+ |
114 |
+ # Store the md5sum in the vdb. |
115 |
+ f = open(os.path.join(infloc, "BINPKGMD5"), "w") |
116 |
+ try: |
117 |
+ f.write(str(portage.checksum.perform_md5(pkg_path)) + "\n") |
118 |
+ finally: |
119 |
+ f.close() |
120 |
+ |
121 |
+ # This gives bashrc users an opportunity to do various things |
122 |
+ # such as remove binary packages after they're installed. |
123 |
+ settings["PORTAGE_BINPKG_FILE"] = pkg_path |
124 |
+ settings.backup_changes("PORTAGE_BINPKG_FILE") |
125 |
+ |
126 |
+ phase = "setup" |
127 |
+ ebuild_phase = EbuildPhase(fd_pipes=fd_pipes, |
128 |
+ pkg=pkg, phase=phase, register=scheduler.register, |
129 |
+ settings=settings, unregister=scheduler.unregister) |
130 |
+ |
131 |
+ ebuild_phase.start() |
132 |
+ retval = None |
133 |
+ while retval is None: |
134 |
+ scheduler.schedule() |
135 |
+ retval = ebuild_phase.poll() |
136 |
+ |
137 |
+ if retval != os.EX_OK: |
138 |
+ return retval |
139 |
+ |
140 |
+ extractor = BinpkgExtractorAsync(image_dir=image_dir, |
141 |
+ pkg=pkg, pkg_path=pkg_path, register=scheduler.register, |
142 |
+ unregister=scheduler.unregister) |
143 |
+ portage.writemsg_stdout(">>> Extracting %s\n" % pkg.cpv) |
144 |
+ extractor.start() |
145 |
+ retval = None |
146 |
+ while retval is None: |
147 |
+ scheduler.schedule() |
148 |
+ retval = extractor.poll() |
149 |
+ |
150 |
+ if retval != os.EX_OK: |
151 |
+ writemsg("!!! Error Extracting '%s'\n" % pkg_path, |
152 |
+ noiselevel=-1) |
153 |
+ return retval |
154 |
+ |
155 |
+ merge = EbuildMerge( |
156 |
+ find_blockers=find_blockers, |
157 |
+ ldpath_mtimes=ldpath_mtimes, |
158 |
+ pkg=pkg, settings=settings) |
159 |
+ |
160 |
retval = merge.execute() |
161 |
if retval != os.EX_OK: |
162 |
return retval |
163 |
+ |
164 |
finally: |
165 |
+ settings.pop("PORTAGE_BINPKG_FILE", None) |
166 |
build_dir.unlock() |
167 |
return os.EX_OK |
168 |
|
169 |
@@ -2328,6 +2437,21 @@ |
170 |
self._lock_obj = None |
171 |
self.locked = False |
172 |
|
173 |
+class BinpkgExtractorAsync(SpawnProcess): |
174 |
+ |
175 |
+ __slots__ = ("image_dir", "pkg", "pkg_path") |
176 |
+ |
177 |
+ _shell_binary = portage.const.BASH_BINARY |
178 |
+ |
179 |
+ def start(self): |
180 |
+ self.args = [self._shell_binary, "-c", |
181 |
+ "bzip2 -dqc -- %s | tar -xp -C %s -f -" % \ |
182 |
+ (portage._shell_quote(self.pkg_path), |
183 |
+ portage._shell_quote(self.image_dir))] |
184 |
+ |
185 |
+ self.env = self.pkg.root_config.settings.environ() |
186 |
+ SpawnProcess.start(self) |
187 |
+ |
188 |
class BinpkgMerge(Task): |
189 |
|
190 |
__slots__ = ("find_blockers", "ldpath_mtimes", |
191 |
|
192 |
Modified: main/trunk/pym/portage/__init__.py |
193 |
=================================================================== |
194 |
--- main/trunk/pym/portage/__init__.py 2008-07-03 04:24:42 UTC (rev 10901) |
195 |
+++ main/trunk/pym/portage/__init__.py 2008-07-03 06:06:34 UTC (rev 10902) |
196 |
@@ -4874,6 +4874,7 @@ |
197 |
noauto = "noauto" in features |
198 |
from portage.data import secpass |
199 |
|
200 |
+ clean_phases = ("clean", "cleanrm") |
201 |
validcommands = ["help","clean","prerm","postrm","cleanrm","preinst","postinst", |
202 |
"config","info","setup","depend","fetch","digest", |
203 |
"unpack","compile","test","install","rpm","qmerge","merge", |
204 |
@@ -4890,7 +4891,7 @@ |
205 |
writemsg("\n", noiselevel=-1) |
206 |
return 1 |
207 |
|
208 |
- if not os.path.exists(myebuild): |
209 |
+ if mydo not in clean_phases and not os.path.exists(myebuild): |
210 |
writemsg("!!! doebuild: %s not found for %s\n" % (myebuild, mydo), |
211 |
noiselevel=-1) |
212 |
return 1 |
213 |
@@ -4987,7 +4988,6 @@ |
214 |
doebuild_environment(myebuild, mydo, myroot, mysettings, debug, |
215 |
use_cache, mydbapi) |
216 |
|
217 |
- clean_phases = ("clean", "cleanrm") |
218 |
if mydo in clean_phases: |
219 |
retval = spawn(_shell_quote(ebuild_sh_binary) + " clean", |
220 |
mysettings, debug=debug, free=1, logfile=None) |
221 |
|
222 |
-- |
223 |
gentoo-commits@l.g.o mailing list |