1 |
commit: 5b29d4a88f492f6890d0574d0addefb9e6a13271 |
2 |
Author: Matt Turner <mattst88 <AT> gentoo <DOT> org> |
3 |
AuthorDate: Fri Apr 17 23:31:52 2020 +0000 |
4 |
Commit: Matt Turner <mattst88 <AT> gentoo <DOT> org> |
5 |
CommitDate: Thu Apr 30 23:04:34 2020 +0000 |
6 |
URL: https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=5b29d4a8 |
7 |
|
8 |
catalyst: Make and use squashfs snapshots |
9 |
|
10 |
There were a number of problems with catalyst's snapshot system. It was |
11 |
built around using the build system's portdir and had no control over |
12 |
what was in that portdir or when it was updated. |
13 |
|
14 |
As a result, when a stage build failed, it was difficult to tell what |
15 |
the snapshot consistet of precisely or whether it contained a particular |
16 |
recent fix. |
17 |
|
18 |
With snapcache disabled, ebuild repo snapshots were tar'd and compressed |
19 |
and then unpacked into the stage chroot which is an unnecessarily |
20 |
expensive process. Moreover, a porttree has more than 100k small files, |
21 |
which are stored extremely inefficiently on most file systems—a whole |
22 |
porttree is usually around 700M on disk. Just removing all of those |
23 |
files during the cleaning stage is an expensive operation. |
24 |
|
25 |
Instead, we make a compressed squashfs image and mount it in the build |
26 |
chroot. The porttree has many duplicate files, and squashfs deduplicates |
27 |
the files and then compresses, so the result is very efficiently packed: |
28 |
~38M with gzip -9 compression. |
29 |
|
30 |
The snapshot target has been modified to generate a squashfs image from |
31 |
a bare ebuild git repo. Piping git-archive to tar2sqfs generates the |
32 |
squashfs image in less than 10 seconds on a modern system. The git repo |
33 |
is fetched with --depth=1 to minize bandwidth and disk usage, and git gc |
34 |
is run after fetch to minimize disk usage. Storage requirements for the |
35 |
git ebuild repo with metadata are ~70M. |
36 |
|
37 |
The squashfs snapshot is stored in /var/tmp/catalyst/snapshots/ by |
38 |
default with a name <repo_name>-<git sha1>.sqfs. With this convention, |
39 |
we know the exact point in history that the snapshot was taken. The |
40 |
catalyst-auto script can use the sha1 to get a deterministic timestamp, |
41 |
so that it is independent on when `catalyst -s` was run, but is instead |
42 |
the timestamp of the commit date of the repo's git SHA1. |
43 |
|
44 |
Signed-off-by: Matt Turner <mattst88 <AT> gentoo.org> |
45 |
|
46 |
README | 3 +- |
47 |
catalyst/base/stagebase.py | 88 ++++---------------- |
48 |
catalyst/base/targetbase.py | 16 +++- |
49 |
catalyst/defaults.py | 2 +- |
50 |
catalyst/main.py | 28 +++---- |
51 |
catalyst/targets/embedded.py | 1 - |
52 |
catalyst/targets/livecd_stage1.py | 1 - |
53 |
catalyst/targets/livecd_stage2.py | 1 - |
54 |
catalyst/targets/netboot.py | 1 - |
55 |
catalyst/targets/snapshot.py | 165 +++++++++++++++++++------------------- |
56 |
catalyst/targets/stage4.py | 1 - |
57 |
doc/catalyst-config.5.txt | 13 +-- |
58 |
doc/catalyst-spec.5.txt | 4 +- |
59 |
13 files changed, 139 insertions(+), 185 deletions(-) |
60 |
|
61 |
diff --git a/README b/README |
62 |
index 1a039fca..1cceb63e 100644 |
63 |
--- a/README |
64 |
+++ b/README |
65 |
@@ -18,8 +18,9 @@ Requirements |
66 |
======================= |
67 |
|
68 |
- Python 3.6 or greater |
69 |
-- An ebuild repository snapshot (or an ebuild tree to create one) |
70 |
- A generic stage3 tarball for your architecture |
71 |
+- A squashfs ebuild repository snapshot |
72 |
+ - Or an ebuild git repo with sys-fs/squashfs-tools-ng and dev-vcs/git |
73 |
|
74 |
What is catalyst? |
75 |
======================== |
76 |
|
77 |
diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py |
78 |
index 9aecf013..41da97b3 100644 |
79 |
--- a/catalyst/base/stagebase.py |
80 |
+++ b/catalyst/base/stagebase.py |
81 |
@@ -35,7 +35,7 @@ class StageBase(TargetBase, ClearBase, GenBase): |
82 |
self.required_values |= frozenset([ |
83 |
"profile", |
84 |
"rel_type", |
85 |
- "snapshot", |
86 |
+ "snapshot_treeish", |
87 |
"source_subpath", |
88 |
"subarch", |
89 |
"target", |
90 |
@@ -149,7 +149,7 @@ class StageBase(TargetBase, ClearBase, GenBase): |
91 |
self.set_source_subpath() |
92 |
|
93 |
# Set paths |
94 |
- self.set_snapshot_path() |
95 |
+ self.set_snapshot() |
96 |
self.set_root_path() |
97 |
self.set_source_path() |
98 |
self.set_chroot_path() |
99 |
@@ -191,9 +191,8 @@ class StageBase(TargetBase, ClearBase, GenBase): |
100 |
# Setup our mount points. |
101 |
self.mount = MOUNT_DEFAULTS.copy() |
102 |
|
103 |
- # Always unpack snapshot tarball |
104 |
- self.mount['portdir']['enable'] = False |
105 |
- |
106 |
+ self.mount['portdir']['source'] = self.snapshot |
107 |
+ self.mount['portdir']['target'] = self.settings['repo_basedir'] + '/' + self.settings['repo_name'] |
108 |
self.mount['distdir']['source'] = self.settings['distdir'] |
109 |
self.mount["distdir"]['target'] = self.settings['target_distdir'] |
110 |
|
111 |
@@ -435,21 +434,11 @@ class StageBase(TargetBase, ClearBase, GenBase): |
112 |
self.settings["destpath"] = normpath(self.settings["chroot_path"]) |
113 |
|
114 |
def set_cleanables(self): |
115 |
- self.settings["cleanables"] = ["/etc/resolv.conf", "/var/tmp/*", "/tmp/*", |
116 |
- self.settings["repo_basedir"] + "/" + |
117 |
- self.settings["repo_name"]] |
118 |
- |
119 |
- def set_snapshot_path(self): |
120 |
- self.settings["snapshot_path"] = file_check( |
121 |
- normpath(self.settings["storedir"] + |
122 |
- "/snapshots/" + self.settings["snapshot_name"] + |
123 |
- self.settings["snapshot"]), |
124 |
- self.accepted_extensions, |
125 |
- self.settings["source_matching"] == "strict" |
126 |
- ) |
127 |
- log.info('SNAPSHOT_PATH set to: %s', self.settings['snapshot_path']) |
128 |
- self.settings["snapshot_path_hash"] = \ |
129 |
- self.generate_hash(self.settings["snapshot_path"], "sha1") |
130 |
+ self.settings['cleanables'] = [ |
131 |
+ "/etc/resolv.conf", |
132 |
+ "/var/tmp/*", |
133 |
+ "/tmp/*", |
134 |
+ ] |
135 |
|
136 |
def set_chroot_path(self): |
137 |
""" |
138 |
@@ -485,7 +474,7 @@ class StageBase(TargetBase, ClearBase, GenBase): |
139 |
"ISO volume ID must not exceed 32 characters.") |
140 |
else: |
141 |
self.settings["iso_volume_id"] = "catalyst " + \ |
142 |
- self.settings["snapshot"] |
143 |
+ self.settings['snapshot_treeish'] |
144 |
|
145 |
def set_default_action_sequence(self): |
146 |
""" Default action sequence for run method. |
147 |
@@ -502,7 +491,6 @@ class StageBase(TargetBase, ClearBase, GenBase): |
148 |
"""Set basic stage1, 2, 3 action sequences""" |
149 |
self.settings['action_sequence'] = [ |
150 |
"unpack", |
151 |
- "unpack_snapshot", |
152 |
"setup_confdir", |
153 |
"portage_overlay", |
154 |
"bind", |
155 |
@@ -810,50 +798,6 @@ class StageBase(TargetBase, ClearBase, GenBase): |
156 |
log.notice( |
157 |
'Resume: Valid resume point detected, skipping seed unpack operation...') |
158 |
|
159 |
- def unpack_snapshot(self): |
160 |
- unpack = True |
161 |
- snapshot_hash = self.resume.get("unpack_repo") |
162 |
- |
163 |
- unpack_errmsg = "Error unpacking snapshot using mode %(mode)s" |
164 |
- |
165 |
- unpack_info = self.decompressor.create_infodict( |
166 |
- source=self.settings["snapshot_path"], |
167 |
- arch=self.settings["compressor_arch"], |
168 |
- other_options=self.settings["compressor_options"], |
169 |
- ) |
170 |
- |
171 |
- target_portdir = normpath(self.settings["chroot_path"] + |
172 |
- self.settings["repo_basedir"] + "/" + self.settings["repo_name"]) |
173 |
- log.info('%s', self.settings['chroot_path']) |
174 |
- log.info('unpack_snapshot(), target_portdir = %s', target_portdir) |
175 |
- cleanup_msg = \ |
176 |
- 'Cleaning up existing portage tree (this can take a long time)...' |
177 |
- unpack_info['destination'] = normpath( |
178 |
- self.settings["chroot_path"] + self.settings["repo_basedir"]) |
179 |
- unpack_info['mode'] = self.decompressor.determine_mode( |
180 |
- unpack_info['source']) |
181 |
- |
182 |
- if "autoresume" in self.settings["options"] \ |
183 |
- and os.path.exists(target_portdir) \ |
184 |
- and self.resume.is_enabled("unpack_repo") \ |
185 |
- and self.settings["snapshot_path_hash"] == snapshot_hash: |
186 |
- log.notice( |
187 |
- 'Valid Resume point detected, skipping unpack of portage tree...') |
188 |
- unpack = False |
189 |
- |
190 |
- if unpack: |
191 |
- if os.path.exists(target_portdir): |
192 |
- log.info('%s', cleanup_msg) |
193 |
- clear_dir(target_portdir) |
194 |
- |
195 |
- log.notice('Unpacking portage tree (this can take a long time) ...') |
196 |
- if not self.decompressor.extract(unpack_info): |
197 |
- log.error('%s', unpack_errmsg % unpack_info) |
198 |
- |
199 |
- log.info('Setting snapshot autoresume point') |
200 |
- self.resume.enable("unpack_repo", |
201 |
- data=self.settings["snapshot_path_hash"]) |
202 |
- |
203 |
def config_profile_link(self): |
204 |
log.info('Configuring profile link...') |
205 |
make_profile = Path(self.settings['chroot_path'] + self.settings['port_conf'], |
206 |
@@ -929,14 +873,16 @@ class StageBase(TargetBase, ClearBase, GenBase): |
207 |
_cmd = ['mount', '-t', 'tmpfs', '-o', 'noexec,nosuid,nodev', |
208 |
'shm', target] |
209 |
else: |
210 |
- _cmd = ['mount', '--bind', source, target] |
211 |
+ _cmd = ['mount', source, target] |
212 |
|
213 |
source = Path(self.mount[x]['source']) |
214 |
+ if source.suffix != '.sqfs': |
215 |
+ _cmd.insert(1, '--bind') |
216 |
|
217 |
- # We may need to create the source of the bind mount. E.g., in the |
218 |
- # case of an empty package cache we must create the directory that |
219 |
- # the binary packages will be stored into. |
220 |
- source.mkdir(mode=0o755, exist_ok=True) |
221 |
+ # We may need to create the source of the bind mount. E.g., in the |
222 |
+ # case of an empty package cache we must create the directory that |
223 |
+ # the binary packages will be stored into. |
224 |
+ source.mkdir(mode=0o755, exist_ok=True) |
225 |
|
226 |
Path(target).mkdir(mode=0o755, parents=True, exist_ok=True) |
227 |
|
228 |
|
229 |
diff --git a/catalyst/base/targetbase.py b/catalyst/base/targetbase.py |
230 |
index fa15ec11..5bcea920 100644 |
231 |
--- a/catalyst/base/targetbase.py |
232 |
+++ b/catalyst/base/targetbase.py |
233 |
@@ -1,8 +1,9 @@ |
234 |
import os |
235 |
|
236 |
from abc import ABC, abstractmethod |
237 |
+from pathlib import Path |
238 |
|
239 |
-from catalyst.support import addl_arg_parse |
240 |
+from catalyst.support import addl_arg_parse, CatalystError |
241 |
|
242 |
|
243 |
class TargetBase(ABC): |
244 |
@@ -18,6 +19,19 @@ class TargetBase(ABC): |
245 |
'PATH': '/bin:/sbin:/usr/bin:/usr/sbin', |
246 |
'TERM': os.getenv('TERM', 'dumb'), |
247 |
} |
248 |
+ self.snapshot = None |
249 |
+ |
250 |
+ def set_snapshot(self, treeish=None): |
251 |
+ # Make snapshots directory |
252 |
+ snapshot_dir = Path(self.settings['storedir'], 'snapshots') |
253 |
+ snapshot_dir.mkdir(mode=0o755, exist_ok=True) |
254 |
+ |
255 |
+ repo_name = self.settings['repo_name'] |
256 |
+ if treeish is None: |
257 |
+ treeish = self.settings['snapshot_treeish'] |
258 |
+ |
259 |
+ self.snapshot = Path(snapshot_dir, |
260 |
+ f'{repo_name}-{treeish}.sqfs') |
261 |
|
262 |
@property |
263 |
@classmethod |
264 |
|
265 |
diff --git a/catalyst/defaults.py b/catalyst/defaults.py |
266 |
index 787a13cc..33f06d34 100644 |
267 |
--- a/catalyst/defaults.py |
268 |
+++ b/catalyst/defaults.py |
269 |
@@ -68,8 +68,8 @@ confdefaults = { |
270 |
"PythonDir": "./catalyst", |
271 |
"repo_basedir": "/var/db/repos", |
272 |
"repo_name": "gentoo", |
273 |
+ "repos": "%(storedir)s/repos", |
274 |
"sharedir": "/usr/share/catalyst", |
275 |
- "snapshot_name": "%(repo_name)s-", |
276 |
"shdir": "%(sharedir)s/targets", |
277 |
"source_matching": "strict", |
278 |
"storedir": "/var/tmp/catalyst", |
279 |
|
280 |
diff --git a/catalyst/main.py b/catalyst/main.py |
281 |
index 8ded4bd1..4ca1aa5b 100644 |
282 |
--- a/catalyst/main.py |
283 |
+++ b/catalyst/main.py |
284 |
@@ -3,6 +3,7 @@ import datetime |
285 |
import hashlib |
286 |
import os |
287 |
import sys |
288 |
+import textwrap |
289 |
|
290 |
from snakeoil.process import namespaces |
291 |
|
292 |
@@ -63,7 +64,7 @@ def parse_config(config_files): |
293 |
log.info(option_messages[opt]) |
294 |
|
295 |
for key in ["digests", "envscript", "var_tmpfs_portage", "port_logdir", |
296 |
- "local_overlay"]: |
297 |
+ "local_overlay", "repos"]: |
298 |
if key in myconf: |
299 |
conf_values[key] = myconf[key] |
300 |
|
301 |
@@ -121,16 +122,15 @@ class FilePath(): |
302 |
|
303 |
def get_parser(): |
304 |
"""Return an argument parser""" |
305 |
- epilog = """Usage examples: |
306 |
+ epilog = textwrap.dedent("""\ |
307 |
+ Usage examples: |
308 |
|
309 |
-Using the commandline option (-C, --cli) to build a Portage snapshot: |
310 |
-$ catalyst -C target=snapshot version_stamp=my_date |
311 |
+ Using the snapshot option to make a snapshot of the ebuild repo: |
312 |
+ $ catalyst --snapshot <git-treeish> |
313 |
|
314 |
-Using the snapshot option (-s, --snapshot) to build a release snapshot: |
315 |
-$ catalyst -s 20071121 |
316 |
- |
317 |
-Using the specfile option (-f, --file) to build a stage target: |
318 |
-$ catalyst -f stage1-specfile.spec""" |
319 |
+ Using the specfile option (-f, --file) to build a stage target: |
320 |
+ $ catalyst -f stage1-specfile.spec |
321 |
+ """) |
322 |
|
323 |
parser = argparse.ArgumentParser( |
324 |
epilog=epilog, formatter_class=argparse.RawDescriptionHelpFormatter) |
325 |
@@ -200,8 +200,8 @@ $ catalyst -f stage1-specfile.spec""" |
326 |
group.add_argument('-f', '--file', |
327 |
type=FilePath(), |
328 |
help='read specfile') |
329 |
- group.add_argument('-s', '--snapshot', |
330 |
- help='generate a release snapshot') |
331 |
+ group.add_argument('-s', '--snapshot', type=str, |
332 |
+ help='Make an ebuild repo snapshot') |
333 |
group.add_argument('-C', '--cli', |
334 |
default=[], nargs=argparse.REMAINDER, |
335 |
help='catalyst commandline (MUST BE LAST OPTION)') |
336 |
@@ -298,7 +298,7 @@ def _main(parser, opts): |
337 |
|
338 |
if opts.snapshot: |
339 |
mycmdline.append('target=snapshot') |
340 |
- mycmdline.append('version_stamp=' + opts.snapshot) |
341 |
+ mycmdline.append('snapshot_treeish=' + opts.snapshot) |
342 |
|
343 |
conf_values['DEBUG'] = opts.debug |
344 |
conf_values['VERBOSE'] = opts.debug or opts.verbose |
345 |
@@ -348,8 +348,8 @@ def _main(parser, opts): |
346 |
if digests - valid_digests: |
347 |
raise CatalystError('These are not valid digest entries:\n%s\n' |
348 |
'Valid digest entries:\n%s' % |
349 |
- ', '.join(sorted(digests - valid_digests)), |
350 |
- ', '.join(sorted(valid_digests))) |
351 |
+ (', '.join(sorted(digests - valid_digests)), |
352 |
+ ', '.join(sorted(valid_digests)))) |
353 |
|
354 |
addlargs = {} |
355 |
|
356 |
|
357 |
diff --git a/catalyst/targets/embedded.py b/catalyst/targets/embedded.py |
358 |
index 189eb722..aa23f5b3 100644 |
359 |
--- a/catalyst/targets/embedded.py |
360 |
+++ b/catalyst/targets/embedded.py |
361 |
@@ -43,7 +43,6 @@ class embedded(StageBase): |
362 |
self.settings['action_sequence'] = [ |
363 |
"dir_setup", |
364 |
"unpack", |
365 |
- "unpack_snapshot", |
366 |
"config_profile_link", |
367 |
"setup_confdir", |
368 |
"portage_overlay", |
369 |
|
370 |
diff --git a/catalyst/targets/livecd_stage1.py b/catalyst/targets/livecd_stage1.py |
371 |
index 727d95f2..f0b6be8b 100644 |
372 |
--- a/catalyst/targets/livecd_stage1.py |
373 |
+++ b/catalyst/targets/livecd_stage1.py |
374 |
@@ -25,7 +25,6 @@ class livecd_stage1(StageBase): |
375 |
def set_action_sequence(self): |
376 |
self.settings['action_sequence'] = [ |
377 |
"unpack", |
378 |
- "unpack_snapshot", |
379 |
"config_profile_link", |
380 |
"setup_confdir", |
381 |
"portage_overlay", |
382 |
|
383 |
diff --git a/catalyst/targets/livecd_stage2.py b/catalyst/targets/livecd_stage2.py |
384 |
index 09de22fb..22450645 100644 |
385 |
--- a/catalyst/targets/livecd_stage2.py |
386 |
+++ b/catalyst/targets/livecd_stage2.py |
387 |
@@ -90,7 +90,6 @@ class livecd_stage2(StageBase): |
388 |
def set_action_sequence(self): |
389 |
self.settings['action_sequence'] = [ |
390 |
"unpack", |
391 |
- "unpack_snapshot", |
392 |
"config_profile_link", |
393 |
"setup_confdir", |
394 |
"portage_overlay", |
395 |
|
396 |
diff --git a/catalyst/targets/netboot.py b/catalyst/targets/netboot.py |
397 |
index 7c37bad5..5620e0d3 100644 |
398 |
--- a/catalyst/targets/netboot.py |
399 |
+++ b/catalyst/targets/netboot.py |
400 |
@@ -162,7 +162,6 @@ class netboot(StageBase): |
401 |
def set_action_sequence(self): |
402 |
self.settings['action_sequence'] = [ |
403 |
"unpack", |
404 |
- "unpack_snapshot", |
405 |
"config_profile_link", |
406 |
"setup_confdir", |
407 |
"portage_overlay", |
408 |
|
409 |
diff --git a/catalyst/targets/snapshot.py b/catalyst/targets/snapshot.py |
410 |
index 16563f14..b6c72c51 100644 |
411 |
--- a/catalyst/targets/snapshot.py |
412 |
+++ b/catalyst/targets/snapshot.py |
413 |
@@ -2,106 +2,103 @@ |
414 |
Snapshot target |
415 |
""" |
416 |
|
417 |
-from DeComp.compress import CompressMap |
418 |
+import subprocess |
419 |
+import sys |
420 |
+ |
421 |
+from pathlib import Path |
422 |
|
423 |
from catalyst import log |
424 |
-from catalyst.support import normpath, cmd |
425 |
from catalyst.base.targetbase import TargetBase |
426 |
-from catalyst.base.genbase import GenBase |
427 |
-from catalyst.fileops import (clear_dir, ensure_dirs) |
428 |
- |
429 |
+from catalyst.lock import write_lock |
430 |
+from catalyst.support import command |
431 |
|
432 |
-class snapshot(TargetBase, GenBase): |
433 |
+class snapshot(TargetBase): |
434 |
""" |
435 |
Builder class for snapshots. |
436 |
""" |
437 |
required_values = frozenset([ |
438 |
- "target", |
439 |
- "version_stamp", |
440 |
+ 'target', |
441 |
]) |
442 |
valid_values = required_values | frozenset([ |
443 |
- "compression_mode", |
444 |
+ 'snapshot_treeish', |
445 |
]) |
446 |
|
447 |
def __init__(self, myspec, addlargs): |
448 |
TargetBase.__init__(self, myspec, addlargs) |
449 |
- GenBase.__init__(self, myspec) |
450 |
|
451 |
- self.settings["target_subpath"] = "repos" |
452 |
- st = self.settings["storedir"] |
453 |
- self.settings["snapshot_path"] = normpath(st + "/snapshots/" |
454 |
- + self.settings["snapshot_name"] |
455 |
- + self.settings["version_stamp"]) |
456 |
- self.settings["tmp_path"] = normpath( |
457 |
- st+"/tmp/"+self.settings["target_subpath"]) |
458 |
+ self.git = command('git') |
459 |
+ self.ebuild_repo = Path(self.settings['repos'], |
460 |
+ self.settings['repo_name']).with_suffix('.git') |
461 |
+ self.gitdir = str(self.ebuild_repo) |
462 |
|
463 |
- def setup(self): |
464 |
- x = normpath(self.settings["storedir"]+"/snapshots") |
465 |
- ensure_dirs(x) |
466 |
+ def update_ebuild_repo(self) -> str: |
467 |
+ repouri = 'https://anongit.gentoo.org/git/repo/sync/gentoo.git' |
468 |
+ |
469 |
+ if self.ebuild_repo.is_dir(): |
470 |
+ git_cmds = [ |
471 |
+ [self.git, '-C', self.gitdir, 'fetch', '--quiet', '--depth=1'], |
472 |
+ [self.git, '-C', self.gitdir, 'update-ref', 'HEAD', 'FETCH_HEAD'], |
473 |
+ [self.git, '-C', self.gitdir, 'gc', '--quiet'], |
474 |
+ ] |
475 |
+ else: |
476 |
+ git_cmds = [ |
477 |
+ [self.git, 'clone', '--quiet', '--depth=1', '--bare', |
478 |
+ '-c', 'gc.reflogExpire=0', |
479 |
+ '-c', 'gc.reflogExpireUnreachable=0', |
480 |
+ '-c', 'gc.rerereresolved=0', |
481 |
+ '-c', 'gc.rerereunresolved=0', |
482 |
+ '-c', 'gc.pruneExpire=now', |
483 |
+ '--branch=stable', |
484 |
+ repouri, self.gitdir], |
485 |
+ ] |
486 |
+ |
487 |
+ for cmd in git_cmds: |
488 |
+ log.notice('>>> ' + ' '.join(cmd)) |
489 |
+ subprocess.run(cmd, |
490 |
+ encoding='utf-8', |
491 |
+ close_fds=False) |
492 |
+ |
493 |
+ sp = subprocess.run([self.git, '-C', self.gitdir, 'rev-parse', 'stable'], |
494 |
+ stdout=subprocess.PIPE, |
495 |
+ encoding='utf-8', |
496 |
+ close_fds=False) |
497 |
+ return sp.stdout.rstrip() |
498 |
|
499 |
def run(self): |
500 |
- if "purgeonly" in self.settings["options"]: |
501 |
- self.purge() |
502 |
- return True |
503 |
- |
504 |
- if "purge" in self.settings["options"]: |
505 |
- self.purge() |
506 |
- |
507 |
- success = True |
508 |
- self.setup() |
509 |
- log.notice('Creating %s tree snapshot %s from %s ...', |
510 |
- self.settings["repo_name"], self.settings['version_stamp'], |
511 |
- self.settings['portdir']) |
512 |
- |
513 |
- mytmp = self.settings["tmp_path"] |
514 |
- ensure_dirs(mytmp) |
515 |
- |
516 |
- cmd(['rsync', '-a', '--no-o', '--no-g', '--delete', |
517 |
- '--exclude=/packages/', |
518 |
- '--exclude=/distfiles/', |
519 |
- '--exclude=/local/', |
520 |
- '--exclude=CVS/', |
521 |
- '--exclude=.svn', |
522 |
- '--exclude=.git/', |
523 |
- '--filter=H_**/files/digest-*', |
524 |
- self.settings['portdir'] + '/', |
525 |
- mytmp + '/' + self.settings['repo_name'] + '/'], |
526 |
- env=self.env) |
527 |
- |
528 |
- log.notice('Compressing %s snapshot tarball ...', |
529 |
- self.settings["repo_name"]) |
530 |
- compressor = CompressMap(self.settings["compress_definitions"], |
531 |
- env=self.env, default_mode=self.settings['compression_mode'], |
532 |
- comp_prog=self.settings["comp_prog"]) |
533 |
- infodict = compressor.create_infodict( |
534 |
- source=self.settings["repo_name"], |
535 |
- destination=self.settings["snapshot_path"], |
536 |
- basedir=mytmp, |
537 |
- filename=self.settings["snapshot_path"], |
538 |
- mode=self.settings["compression_mode"], |
539 |
- auto_extension=True |
540 |
- ) |
541 |
- if not compressor.compress(infodict): |
542 |
- success = False |
543 |
- log.error('Snapshot compression failure') |
544 |
+ if self.settings['snapshot_treeish'] == 'stable': |
545 |
+ treeish = self.update_ebuild_repo() |
546 |
+ else: |
547 |
+ treeish = self.settings['snapshot_treeish'] |
548 |
+ |
549 |
+ self.set_snapshot(treeish) |
550 |
+ |
551 |
+ git_cmd = [self.git, '-C', self.gitdir, 'archive', '--format=tar', |
552 |
+ treeish] |
553 |
+ tar2sqfs_cmd = [command('tar2sqfs'), str(self.snapshot), '-q', '-f', |
554 |
+ '-j1', '-c', 'gzip'] |
555 |
+ |
556 |
+ log.notice('Creating %s tree snapshot %s from %s', |
557 |
+ self.settings['repo_name'], treeish, self.gitdir) |
558 |
+ log.notice('>>> ' + ' '.join([*git_cmd, '|'])) |
559 |
+ log.notice(' ' + ' '.join(tar2sqfs_cmd)) |
560 |
+ |
561 |
+ lockfile = self.snapshot.with_suffix('.lock') |
562 |
+ with write_lock(lockfile): |
563 |
+ git = subprocess.Popen(git_cmd, |
564 |
+ stdout=subprocess.PIPE, |
565 |
+ stderr=sys.stderr, |
566 |
+ close_fds=False) |
567 |
+ tar2sqfs = subprocess.Popen(tar2sqfs_cmd, |
568 |
+ stdin=git.stdout, |
569 |
+ stdout=sys.stdout, |
570 |
+ stderr=sys.stderr, |
571 |
+ close_fds=False) |
572 |
+ git.stdout.close() |
573 |
+ git.wait() |
574 |
+ tar2sqfs.wait() |
575 |
+ |
576 |
+ if tar2sqfs.returncode == 0: |
577 |
+ log.notice('Wrote snapshot to %s', self.snapshot) |
578 |
else: |
579 |
- filename = '.'.join([self.settings["snapshot_path"], |
580 |
- compressor.extension(self.settings["compression_mode"])]) |
581 |
- log.notice('Snapshot successfully written to %s', filename) |
582 |
- self.gen_contents_file(filename) |
583 |
- self.gen_digest_file(filename) |
584 |
- if "keepwork" not in self.settings["options"]: |
585 |
- self.cleanup() |
586 |
- if success: |
587 |
- log.info('snapshot: complete!') |
588 |
- return success |
589 |
- |
590 |
- def kill_chroot_pids(self): |
591 |
- pass |
592 |
- |
593 |
- def cleanup(self): |
594 |
- log.info('Cleaning up ...') |
595 |
- self.purge() |
596 |
- |
597 |
- def purge(self): |
598 |
- clear_dir(self.settings['tmp_path']) |
599 |
+ log.error('Failed to create snapshot') |
600 |
+ return tar2sqfs.returncode == 0 |
601 |
|
602 |
diff --git a/catalyst/targets/stage4.py b/catalyst/targets/stage4.py |
603 |
index a3de2cae..17719f0e 100644 |
604 |
--- a/catalyst/targets/stage4.py |
605 |
+++ b/catalyst/targets/stage4.py |
606 |
@@ -38,7 +38,6 @@ class stage4(StageBase): |
607 |
def set_action_sequence(self): |
608 |
self.settings['action_sequence'] = [ |
609 |
"unpack", |
610 |
- "unpack_snapshot", |
611 |
"config_profile_link", |
612 |
"setup_confdir", |
613 |
"portage_overlay", |
614 |
|
615 |
diff --git a/doc/catalyst-config.5.txt b/doc/catalyst-config.5.txt |
616 |
index 44c905d7..925934ad 100644 |
617 |
--- a/doc/catalyst-config.5.txt |
618 |
+++ b/doc/catalyst-config.5.txt |
619 |
@@ -111,17 +111,18 @@ Defaults to the host's DISTDIR. |
620 |
Source Gentoo tree location (primary repo). `/var/db/repos/gentoo/` should work for most |
621 |
default installations. |
622 |
|
623 |
+*repos*:: |
624 |
+The directory in which git repositories exist for use by the snapshot target. |
625 |
+Defaults to `${storedir}/repos`. |
626 |
+ |
627 |
*repo_basedir*:: |
628 |
The target repository directory to contain the primary repo (gentoo repo) and |
629 |
any overlays. The default location is `/var/db/repos`. |
630 |
|
631 |
*repo_name*:: |
632 |
-The name of the main repository (ie: gentoo). This has had a directory name |
633 |
-of `portage` in the past. But it has an internal name of `gentoo`, which is |
634 |
-what its directory name should be. This name is used in the snapshot name |
635 |
-generated and also the directory name of the repository created with the |
636 |
-snapshot target. The new general rule is that the directory name and its |
637 |
-internal repo_name value should be the same. |
638 |
+The name of the main repository (e.g. gentoo). The git repository at |
639 |
+`${repos}/${repo_name}.git` will be used to produce the portdir sqfs |
640 |
+snapshot. |
641 |
|
642 |
*target_distdir*:: |
643 |
This is the target distfiles directory location for the stage being created. |
644 |
|
645 |
diff --git a/doc/catalyst-spec.5.txt b/doc/catalyst-spec.5.txt |
646 |
index 58f0a9f0..f87bd69e 100644 |
647 |
--- a/doc/catalyst-spec.5.txt |
648 |
+++ b/doc/catalyst-spec.5.txt |
649 |
@@ -62,9 +62,9 @@ allowing multiple concurrent builds. Usually, `default` will suffice. |
650 |
*profile*:: |
651 |
This is the system profile to be used by catalyst to build this target |
652 |
(example: `default/linux/x86/10.0/`). It is specified as a relative |
653 |
-path from `profiles` in your portage snapshot |
654 |
+path from `profiles` in your portdir snapshot |
655 |
|
656 |
-*snapshot*:: |
657 |
+*snapshot_treeish*:: |
658 |
This specifies which snapshot to use for building this target |
659 |
(example: `2006.1`). |