1 |
commit: 4e908d9934f988c01782b7d4fa8cd9bfa8853efa |
2 |
Author: Matt Turner <mattst88 <AT> gentoo <DOT> org> |
3 |
AuthorDate: Thu Apr 9 03:37:16 2020 +0000 |
4 |
Commit: Matt Turner <mattst88 <AT> gentoo <DOT> org> |
5 |
CommitDate: Thu Apr 23 18:20:35 2020 +0000 |
6 |
URL: https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=4e908d99 |
7 |
|
8 |
catalyst: Remove snapcache support |
9 |
|
10 |
Among the problems with snapcache were broken locking, and still needing |
11 |
to pack and unpack snapshot tarballs. In a few commits, I'm going to |
12 |
replace it with something significantly better. |
13 |
|
14 |
Signed-off-by: Matt Turner <mattst88 <AT> gentoo.org> |
15 |
|
16 |
catalyst/base/stagebase.py | 103 +++++++++------------------------------------ |
17 |
catalyst/defaults.py | 3 -- |
18 |
doc/catalyst-config.5.txt | 10 +---- |
19 |
etc/catalyst.conf | 5 +-- |
20 |
4 files changed, 24 insertions(+), 97 deletions(-) |
21 |
|
22 |
diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py |
23 |
index 1c0d98c6..275c07eb 100644 |
24 |
--- a/catalyst/base/stagebase.py |
25 |
+++ b/catalyst/base/stagebase.py |
26 |
@@ -152,7 +152,6 @@ class StageBase(TargetBase, ClearBase, GenBase): |
27 |
self.set_snapshot_path() |
28 |
self.set_root_path() |
29 |
self.set_source_path() |
30 |
- self.set_snapcache_path() |
31 |
self.set_chroot_path() |
32 |
self.set_autoresume_path() |
33 |
self.set_dest_path() |
34 |
@@ -192,20 +191,11 @@ class StageBase(TargetBase, ClearBase, GenBase): |
35 |
# Setup our mount points. |
36 |
self.mount = MOUNT_DEFAULTS.copy() |
37 |
|
38 |
- self.mount['portdir']['source'] = self.settings['portdir'] |
39 |
- self.mount['portdir']['target'] = normpath(self.settings["repo_basedir"] |
40 |
- + "/" + |
41 |
- self.settings["repo_name"]) |
42 |
- self.mount['distdir']['source'] = self.settings['distdir'] |
43 |
- self.mount["distdir"]['target'] = self.settings["target_distdir"] |
44 |
+ # Always unpack snapshot tarball |
45 |
+ self.mount['portdir']['enable'] = False |
46 |
|
47 |
- if "snapcache" not in self.settings["options"]: |
48 |
- self.mount['portdir']['enable'] = False |
49 |
- else: |
50 |
- self.mount['portdir']['source'] = normpath("/".join([ |
51 |
- self.settings['snapshot_cache_path'], |
52 |
- self.settings['repo_name'] |
53 |
- ])) |
54 |
+ self.mount['distdir']['source'] = self.settings['distdir'] |
55 |
+ self.mount["distdir"]['target'] = self.settings['target_distdir'] |
56 |
|
57 |
# Configure any user specified options (either in catalyst.conf or on |
58 |
# the command line). |
59 |
@@ -461,19 +451,6 @@ class StageBase(TargetBase, ClearBase, GenBase): |
60 |
self.settings["snapshot_path_hash"] = \ |
61 |
self.generate_hash(self.settings["snapshot_path"], "sha1") |
62 |
|
63 |
- def set_snapcache_path(self): |
64 |
- self.settings["snapshot_cache_path"] = \ |
65 |
- normpath(pjoin(self.settings["snapshot_cache"], |
66 |
- self.settings["snapshot"])) |
67 |
- if "snapcache" in self.settings["options"]: |
68 |
- self.settings["snapshot_cache_path"] = \ |
69 |
- normpath(pjoin(self.settings["snapshot_cache"], |
70 |
- self.settings["snapshot"])) |
71 |
- self.snapcache_lock = \ |
72 |
- LockDir(self.settings["snapshot_cache_path"]) |
73 |
- log.info('Setting snapshot cache to %s', |
74 |
- self.settings['snapshot_cache_path']) |
75 |
- |
76 |
def set_chroot_path(self): |
77 |
""" |
78 |
NOTE: the trailing slash has been removed |
79 |
@@ -841,7 +818,6 @@ class StageBase(TargetBase, ClearBase, GenBase): |
80 |
|
81 |
unpack_info = self.decompressor.create_infodict( |
82 |
source=self.settings["snapshot_path"], |
83 |
- destination=self.settings["snapshot_cache_path"], |
84 |
arch=self.settings["compressor_arch"], |
85 |
other_options=self.settings["compressor_options"], |
86 |
) |
87 |
@@ -850,41 +826,22 @@ class StageBase(TargetBase, ClearBase, GenBase): |
88 |
self.settings["repo_basedir"] + "/" + self.settings["repo_name"]) |
89 |
log.info('%s', self.settings['chroot_path']) |
90 |
log.info('unpack_snapshot(), target_portdir = %s', target_portdir) |
91 |
- if "snapcache" in self.settings["options"]: |
92 |
- snapshot_cache_hash_path = pjoin( |
93 |
- self.settings['snapshot_cache_path'], 'catalyst-hash') |
94 |
- snapshot_cache_hash = fileutils.readfile( |
95 |
- snapshot_cache_hash_path, True) |
96 |
- unpack_info['mode'] = self.decompressor.determine_mode( |
97 |
- unpack_info['source']) |
98 |
- |
99 |
- cleanup_msg = "Cleaning up invalid snapshot cache at \n\t" + \ |
100 |
- self.settings["snapshot_cache_path"] + \ |
101 |
- " (this can take a long time)..." |
102 |
- |
103 |
- if self.settings["snapshot_path_hash"] == snapshot_cache_hash: |
104 |
- log.info( |
105 |
- 'Valid snapshot cache, skipping unpack of portage tree...') |
106 |
- unpack = False |
107 |
- else: |
108 |
- cleanup_msg = \ |
109 |
- 'Cleaning up existing portage tree (this can take a long time)...' |
110 |
- unpack_info['destination'] = normpath( |
111 |
- self.settings["chroot_path"] + self.settings["repo_basedir"]) |
112 |
- unpack_info['mode'] = self.decompressor.determine_mode( |
113 |
- unpack_info['source']) |
114 |
- |
115 |
- if "autoresume" in self.settings["options"] \ |
116 |
- and os.path.exists(target_portdir) \ |
117 |
- and self.resume.is_enabled("unpack_repo") \ |
118 |
- and self.settings["snapshot_path_hash"] == snapshot_hash: |
119 |
- log.notice( |
120 |
- 'Valid Resume point detected, skipping unpack of portage tree...') |
121 |
- unpack = False |
122 |
+ cleanup_msg = \ |
123 |
+ 'Cleaning up existing portage tree (this can take a long time)...' |
124 |
+ unpack_info['destination'] = normpath( |
125 |
+ self.settings["chroot_path"] + self.settings["repo_basedir"]) |
126 |
+ unpack_info['mode'] = self.decompressor.determine_mode( |
127 |
+ unpack_info['source']) |
128 |
+ |
129 |
+ if "autoresume" in self.settings["options"] \ |
130 |
+ and os.path.exists(target_portdir) \ |
131 |
+ and self.resume.is_enabled("unpack_repo") \ |
132 |
+ and self.settings["snapshot_path_hash"] == snapshot_hash: |
133 |
+ log.notice( |
134 |
+ 'Valid Resume point detected, skipping unpack of portage tree...') |
135 |
+ unpack = False |
136 |
|
137 |
if unpack: |
138 |
- if "snapcache" in self.settings["options"]: |
139 |
- self.snapcache_lock.write_lock() |
140 |
if os.path.exists(target_portdir): |
141 |
log.info('%s', cleanup_msg) |
142 |
clear_dir(target_portdir) |
143 |
@@ -893,16 +850,9 @@ class StageBase(TargetBase, ClearBase, GenBase): |
144 |
if not self.decompressor.extract(unpack_info): |
145 |
log.error('%s', unpack_errmsg % unpack_info) |
146 |
|
147 |
- if "snapcache" in self.settings["options"]: |
148 |
- with open(snapshot_cache_hash_path, 'w') as myf: |
149 |
- myf.write(self.settings["snapshot_path_hash"]) |
150 |
- else: |
151 |
- log.info('Setting snapshot autoresume point') |
152 |
- self.resume.enable("unpack_repo", |
153 |
- data=self.settings["snapshot_path_hash"]) |
154 |
- |
155 |
- if "snapcache" in self.settings["options"]: |
156 |
- self.snapcache_lock.unlock() |
157 |
+ log.info('Setting snapshot autoresume point') |
158 |
+ self.resume.enable("unpack_repo", |
159 |
+ data=self.settings["snapshot_path_hash"]) |
160 |
|
161 |
def config_profile_link(self): |
162 |
log.info('Configuring profile link...') |
163 |
@@ -967,9 +917,6 @@ class StageBase(TargetBase, ClearBase, GenBase): |
164 |
|
165 |
src = self.mount[x]['source'] |
166 |
log.debug('bind(); src = %s', src) |
167 |
- if "snapcache" in self.settings["options"] and x == "portdir": |
168 |
- self.snapcache_lock.read_lock() |
169 |
- _cmd = None |
170 |
if src == "maybe_tmpfs": |
171 |
if "var_tmpfs_portage" in self.settings: |
172 |
_cmd = ['mount', '-t', 'tmpfs', |
173 |
@@ -1018,14 +965,6 @@ class StageBase(TargetBase, ClearBase, GenBase): |
174 |
ouch = 1 |
175 |
log.warning("Couldn't umount bind mount: %s", target) |
176 |
|
177 |
- if "snapcache" in self.settings["options"] and x == "/var/db/repos/gentoo": |
178 |
- try: |
179 |
- # It's possible the snapshot lock object isn't created yet. |
180 |
- # This is because mount safety check calls unbind before the |
181 |
- # target is fully initialized |
182 |
- self.snapcache_lock.unlock() |
183 |
- except Exception: |
184 |
- pass |
185 |
if ouch: |
186 |
# if any bind mounts really failed, then we need to raise |
187 |
# this to potentially prevent an upcoming bash stage cleanup script |
188 |
|
189 |
diff --git a/catalyst/defaults.py b/catalyst/defaults.py |
190 |
index 77133cb1..787a13cc 100644 |
191 |
--- a/catalyst/defaults.py |
192 |
+++ b/catalyst/defaults.py |
193 |
@@ -45,7 +45,6 @@ valid_config_file_values.extend([ |
194 |
"distcc", |
195 |
"envscript", |
196 |
"options", |
197 |
- "snapshot_cache", |
198 |
"VERBOSE", |
199 |
]) |
200 |
|
201 |
@@ -70,7 +69,6 @@ confdefaults = { |
202 |
"repo_basedir": "/var/db/repos", |
203 |
"repo_name": "gentoo", |
204 |
"sharedir": "/usr/share/catalyst", |
205 |
- "snapshot_cache": "/var/tmp/catalyst/snapshot_cache", |
206 |
"snapshot_name": "%(repo_name)s-", |
207 |
"shdir": "%(sharedir)s/targets", |
208 |
"source_matching": "strict", |
209 |
@@ -162,5 +160,4 @@ option_messages = { |
210 |
"pkgcache": "Package cache support enabled.", |
211 |
"purge": "Purge support enabled.", |
212 |
"seedcache": "Seed cache support enabled.", |
213 |
- "snapcache": "Snapshot cache support enabled.", |
214 |
} |
215 |
|
216 |
diff --git a/doc/catalyst-config.5.txt b/doc/catalyst-config.5.txt |
217 |
index a0cd0490..44c905d7 100644 |
218 |
--- a/doc/catalyst-config.5.txt |
219 |
+++ b/doc/catalyst-config.5.txt |
220 |
@@ -27,7 +27,7 @@ and empty lines are interpreted as comments. For example: |
221 |
|
222 |
digests="auto" |
223 |
envscript="/etc/catalyst/catalystrc" |
224 |
-options="autoresume bindist kerncache pkgcache seedcache snapcache" |
225 |
+options="autoresume bindist kerncache pkgcache seedcache" |
226 |
|
227 |
# source repo settings |
228 |
portdir="/var/db/repos/gentoo" |
229 |
@@ -56,7 +56,7 @@ export FOO="bar" |
230 |
|
231 |
**options*:: |
232 |
Set different build-time options (example: `autoresume bindist |
233 |
-kerncache pkgcache seedcache snapcache`). Supported values: |
234 |
+kerncache pkgcache seedcache`). Supported values: |
235 |
+ |
236 |
-- |
237 |
autoresume:: |
238 |
@@ -95,12 +95,6 @@ seedcache:: |
239 |
Use the build output of a previous target if it exists to speed up the |
240 |
creation of a new stage. This avoids unpacking the seed tarball. |
241 |
|
242 |
-snapcache:: |
243 |
-Cache the snapshot so that it can be bind-mounted into the chroot. |
244 |
-WARNING: moving parts of the portage tree from within fsscript *will* |
245 |
-break your cache. The cache is unlinked before any empty or rm |
246 |
-processing. |
247 |
- |
248 |
versioned_cache:: |
249 |
Name the cache directories (packagecache, kerncache) based on the version of a |
250 |
spec file. |
251 |
|
252 |
diff --git a/etc/catalyst.conf b/etc/catalyst.conf |
253 |
index 6d530a39..6a499038 100644 |
254 |
--- a/etc/catalyst.conf |
255 |
+++ b/etc/catalyst.conf |
256 |
@@ -42,9 +42,6 @@ envscript="/etc/catalyst/catalystrc" |
257 |
# (used on installcd-stage2 and stage4 targets) |
258 |
# seedcache = use the build output of a previous target if it exists to speed up |
259 |
# the copy |
260 |
-# snapcache = cache the snapshot so that it can be bind-mounted into the chroot. |
261 |
-# WARNING: moving parts of the portage tree from within fsscript *will* break |
262 |
-# your cache. The cache is unlinked before any empty or rm processing, though. |
263 |
# sticky-config = enables the code that will keep any internal 'catalyst_use' flags |
264 |
# added to the USE= for building the stage. These are usually added for legal |
265 |
# or specific needs in building the the early stage. Mostly it is the |
266 |
@@ -53,7 +50,7 @@ envscript="/etc/catalyst/catalystrc" |
267 |
# files or directories. |
268 |
# |
269 |
# (These options can be used together) |
270 |
-options="autoresume bindist kerncache pkgcache seedcache snapcache" |
271 |
+options="autoresume bindist kerncache pkgcache seedcache" |
272 |
|
273 |
# Source portdir specifies the source portage tree used by the snapshot target. |
274 |
portdir="/var/db/repos/gentoo" |