Gentoo Archives: gentoo-commits

From: Matt Turner <mattst88@g.o>
To: gentoo-commits@l.g.o
Subject: [gentoo-commits] proj/catalyst:master commit in: etc/, doc/, catalyst/base/, catalyst/
Date: Thu, 30 Apr 2020 22:56:27
Message-Id: 1587666035.4e908d9934f988c01782b7d4fa8cd9bfa8853efa.mattst88@gentoo
1 commit: 4e908d9934f988c01782b7d4fa8cd9bfa8853efa
2 Author: Matt Turner <mattst88 <AT> gentoo <DOT> org>
3 AuthorDate: Thu Apr 9 03:37:16 2020 +0000
4 Commit: Matt Turner <mattst88 <AT> gentoo <DOT> org>
5 CommitDate: Thu Apr 23 18:20:35 2020 +0000
6 URL: https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=4e908d99
7
8 catalyst: Remove snapcache support
9
10 Among the problems with snapcache were broken locking, and still needing
11 to pack and unpack snapshot tarballs. In a few commits, I'm going to
12 replace it with something significantly better.
13
14 Signed-off-by: Matt Turner <mattst88 <AT> gentoo.org>
15
16 catalyst/base/stagebase.py | 103 +++++++++------------------------------------
17 catalyst/defaults.py | 3 --
18 doc/catalyst-config.5.txt | 10 +----
19 etc/catalyst.conf | 5 +--
20 4 files changed, 24 insertions(+), 97 deletions(-)
21
22 diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py
23 index 1c0d98c6..275c07eb 100644
24 --- a/catalyst/base/stagebase.py
25 +++ b/catalyst/base/stagebase.py
26 @@ -152,7 +152,6 @@ class StageBase(TargetBase, ClearBase, GenBase):
27 self.set_snapshot_path()
28 self.set_root_path()
29 self.set_source_path()
30 - self.set_snapcache_path()
31 self.set_chroot_path()
32 self.set_autoresume_path()
33 self.set_dest_path()
34 @@ -192,20 +191,11 @@ class StageBase(TargetBase, ClearBase, GenBase):
35 # Setup our mount points.
36 self.mount = MOUNT_DEFAULTS.copy()
37
38 - self.mount['portdir']['source'] = self.settings['portdir']
39 - self.mount['portdir']['target'] = normpath(self.settings["repo_basedir"]
40 - + "/" +
41 - self.settings["repo_name"])
42 - self.mount['distdir']['source'] = self.settings['distdir']
43 - self.mount["distdir"]['target'] = self.settings["target_distdir"]
44 + # Always unpack snapshot tarball
45 + self.mount['portdir']['enable'] = False
46
47 - if "snapcache" not in self.settings["options"]:
48 - self.mount['portdir']['enable'] = False
49 - else:
50 - self.mount['portdir']['source'] = normpath("/".join([
51 - self.settings['snapshot_cache_path'],
52 - self.settings['repo_name']
53 - ]))
54 + self.mount['distdir']['source'] = self.settings['distdir']
55 + self.mount["distdir"]['target'] = self.settings['target_distdir']
56
57 # Configure any user specified options (either in catalyst.conf or on
58 # the command line).
59 @@ -461,19 +451,6 @@ class StageBase(TargetBase, ClearBase, GenBase):
60 self.settings["snapshot_path_hash"] = \
61 self.generate_hash(self.settings["snapshot_path"], "sha1")
62
63 - def set_snapcache_path(self):
64 - self.settings["snapshot_cache_path"] = \
65 - normpath(pjoin(self.settings["snapshot_cache"],
66 - self.settings["snapshot"]))
67 - if "snapcache" in self.settings["options"]:
68 - self.settings["snapshot_cache_path"] = \
69 - normpath(pjoin(self.settings["snapshot_cache"],
70 - self.settings["snapshot"]))
71 - self.snapcache_lock = \
72 - LockDir(self.settings["snapshot_cache_path"])
73 - log.info('Setting snapshot cache to %s',
74 - self.settings['snapshot_cache_path'])
75 -
76 def set_chroot_path(self):
77 """
78 NOTE: the trailing slash has been removed
79 @@ -841,7 +818,6 @@ class StageBase(TargetBase, ClearBase, GenBase):
80
81 unpack_info = self.decompressor.create_infodict(
82 source=self.settings["snapshot_path"],
83 - destination=self.settings["snapshot_cache_path"],
84 arch=self.settings["compressor_arch"],
85 other_options=self.settings["compressor_options"],
86 )
87 @@ -850,41 +826,22 @@ class StageBase(TargetBase, ClearBase, GenBase):
88 self.settings["repo_basedir"] + "/" + self.settings["repo_name"])
89 log.info('%s', self.settings['chroot_path'])
90 log.info('unpack_snapshot(), target_portdir = %s', target_portdir)
91 - if "snapcache" in self.settings["options"]:
92 - snapshot_cache_hash_path = pjoin(
93 - self.settings['snapshot_cache_path'], 'catalyst-hash')
94 - snapshot_cache_hash = fileutils.readfile(
95 - snapshot_cache_hash_path, True)
96 - unpack_info['mode'] = self.decompressor.determine_mode(
97 - unpack_info['source'])
98 -
99 - cleanup_msg = "Cleaning up invalid snapshot cache at \n\t" + \
100 - self.settings["snapshot_cache_path"] + \
101 - " (this can take a long time)..."
102 -
103 - if self.settings["snapshot_path_hash"] == snapshot_cache_hash:
104 - log.info(
105 - 'Valid snapshot cache, skipping unpack of portage tree...')
106 - unpack = False
107 - else:
108 - cleanup_msg = \
109 - 'Cleaning up existing portage tree (this can take a long time)...'
110 - unpack_info['destination'] = normpath(
111 - self.settings["chroot_path"] + self.settings["repo_basedir"])
112 - unpack_info['mode'] = self.decompressor.determine_mode(
113 - unpack_info['source'])
114 -
115 - if "autoresume" in self.settings["options"] \
116 - and os.path.exists(target_portdir) \
117 - and self.resume.is_enabled("unpack_repo") \
118 - and self.settings["snapshot_path_hash"] == snapshot_hash:
119 - log.notice(
120 - 'Valid Resume point detected, skipping unpack of portage tree...')
121 - unpack = False
122 + cleanup_msg = \
123 + 'Cleaning up existing portage tree (this can take a long time)...'
124 + unpack_info['destination'] = normpath(
125 + self.settings["chroot_path"] + self.settings["repo_basedir"])
126 + unpack_info['mode'] = self.decompressor.determine_mode(
127 + unpack_info['source'])
128 +
129 + if "autoresume" in self.settings["options"] \
130 + and os.path.exists(target_portdir) \
131 + and self.resume.is_enabled("unpack_repo") \
132 + and self.settings["snapshot_path_hash"] == snapshot_hash:
133 + log.notice(
134 + 'Valid Resume point detected, skipping unpack of portage tree...')
135 + unpack = False
136
137 if unpack:
138 - if "snapcache" in self.settings["options"]:
139 - self.snapcache_lock.write_lock()
140 if os.path.exists(target_portdir):
141 log.info('%s', cleanup_msg)
142 clear_dir(target_portdir)
143 @@ -893,16 +850,9 @@ class StageBase(TargetBase, ClearBase, GenBase):
144 if not self.decompressor.extract(unpack_info):
145 log.error('%s', unpack_errmsg % unpack_info)
146
147 - if "snapcache" in self.settings["options"]:
148 - with open(snapshot_cache_hash_path, 'w') as myf:
149 - myf.write(self.settings["snapshot_path_hash"])
150 - else:
151 - log.info('Setting snapshot autoresume point')
152 - self.resume.enable("unpack_repo",
153 - data=self.settings["snapshot_path_hash"])
154 -
155 - if "snapcache" in self.settings["options"]:
156 - self.snapcache_lock.unlock()
157 + log.info('Setting snapshot autoresume point')
158 + self.resume.enable("unpack_repo",
159 + data=self.settings["snapshot_path_hash"])
160
161 def config_profile_link(self):
162 log.info('Configuring profile link...')
163 @@ -967,9 +917,6 @@ class StageBase(TargetBase, ClearBase, GenBase):
164
165 src = self.mount[x]['source']
166 log.debug('bind(); src = %s', src)
167 - if "snapcache" in self.settings["options"] and x == "portdir":
168 - self.snapcache_lock.read_lock()
169 - _cmd = None
170 if src == "maybe_tmpfs":
171 if "var_tmpfs_portage" in self.settings:
172 _cmd = ['mount', '-t', 'tmpfs',
173 @@ -1018,14 +965,6 @@ class StageBase(TargetBase, ClearBase, GenBase):
174 ouch = 1
175 log.warning("Couldn't umount bind mount: %s", target)
176
177 - if "snapcache" in self.settings["options"] and x == "/var/db/repos/gentoo":
178 - try:
179 - # It's possible the snapshot lock object isn't created yet.
180 - # This is because mount safety check calls unbind before the
181 - # target is fully initialized
182 - self.snapcache_lock.unlock()
183 - except Exception:
184 - pass
185 if ouch:
186 # if any bind mounts really failed, then we need to raise
187 # this to potentially prevent an upcoming bash stage cleanup script
188
189 diff --git a/catalyst/defaults.py b/catalyst/defaults.py
190 index 77133cb1..787a13cc 100644
191 --- a/catalyst/defaults.py
192 +++ b/catalyst/defaults.py
193 @@ -45,7 +45,6 @@ valid_config_file_values.extend([
194 "distcc",
195 "envscript",
196 "options",
197 - "snapshot_cache",
198 "VERBOSE",
199 ])
200
201 @@ -70,7 +69,6 @@ confdefaults = {
202 "repo_basedir": "/var/db/repos",
203 "repo_name": "gentoo",
204 "sharedir": "/usr/share/catalyst",
205 - "snapshot_cache": "/var/tmp/catalyst/snapshot_cache",
206 "snapshot_name": "%(repo_name)s-",
207 "shdir": "%(sharedir)s/targets",
208 "source_matching": "strict",
209 @@ -162,5 +160,4 @@ option_messages = {
210 "pkgcache": "Package cache support enabled.",
211 "purge": "Purge support enabled.",
212 "seedcache": "Seed cache support enabled.",
213 - "snapcache": "Snapshot cache support enabled.",
214 }
215
216 diff --git a/doc/catalyst-config.5.txt b/doc/catalyst-config.5.txt
217 index a0cd0490..44c905d7 100644
218 --- a/doc/catalyst-config.5.txt
219 +++ b/doc/catalyst-config.5.txt
220 @@ -27,7 +27,7 @@ and empty lines are interpreted as comments. For example:
221
222 digests="auto"
223 envscript="/etc/catalyst/catalystrc"
224 -options="autoresume bindist kerncache pkgcache seedcache snapcache"
225 +options="autoresume bindist kerncache pkgcache seedcache"
226
227 # source repo settings
228 portdir="/var/db/repos/gentoo"
229 @@ -56,7 +56,7 @@ export FOO="bar"
230
231 **options*::
232 Set different build-time options (example: `autoresume bindist
233 -kerncache pkgcache seedcache snapcache`). Supported values:
234 +kerncache pkgcache seedcache`). Supported values:
235 +
236 --
237 autoresume::
238 @@ -95,12 +95,6 @@ seedcache::
239 Use the build output of a previous target if it exists to speed up the
240 creation of a new stage. This avoids unpacking the seed tarball.
241
242 -snapcache::
243 -Cache the snapshot so that it can be bind-mounted into the chroot.
244 -WARNING: moving parts of the portage tree from within fsscript *will*
245 -break your cache. The cache is unlinked before any empty or rm
246 -processing.
247 -
248 versioned_cache::
249 Name the cache directories (packagecache, kerncache) based on the version of a
250 spec file.
251
252 diff --git a/etc/catalyst.conf b/etc/catalyst.conf
253 index 6d530a39..6a499038 100644
254 --- a/etc/catalyst.conf
255 +++ b/etc/catalyst.conf
256 @@ -42,9 +42,6 @@ envscript="/etc/catalyst/catalystrc"
257 # (used on installcd-stage2 and stage4 targets)
258 # seedcache = use the build output of a previous target if it exists to speed up
259 # the copy
260 -# snapcache = cache the snapshot so that it can be bind-mounted into the chroot.
261 -# WARNING: moving parts of the portage tree from within fsscript *will* break
262 -# your cache. The cache is unlinked before any empty or rm processing, though.
263 # sticky-config = enables the code that will keep any internal 'catalyst_use' flags
264 # added to the USE= for building the stage. These are usually added for legal
265 # or specific needs in building the the early stage. Mostly it is the
266 @@ -53,7 +50,7 @@ envscript="/etc/catalyst/catalystrc"
267 # files or directories.
268 #
269 # (These options can be used together)
270 -options="autoresume bindist kerncache pkgcache seedcache snapcache"
271 +options="autoresume bindist kerncache pkgcache seedcache"
272
273 # Source portdir specifies the source portage tree used by the snapshot target.
274 portdir="/var/db/repos/gentoo"