1 |
commit: fd04c5fb1619f86381b5d5e6ff66b20fa3967c43 |
2 |
Author: Zac Medico <zmedico <AT> gentoo <DOT> org> |
3 |
AuthorDate: Wed Feb 24 19:56:38 2021 +0000 |
4 |
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org> |
5 |
CommitDate: Sat Feb 27 07:43:23 2021 +0000 |
6 |
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=fd04c5fb |
7 |
|
8 |
emirrordist: add --content-db option required for content-hash layout (bug 756778) |
9 |
|
10 |
Add a --content-db option which is required for the content-hash |
11 |
layout because its file listings return content digests instead of |
12 |
distfile names. |
13 |
|
14 |
The content db serves to translate content digests to distfiles |
15 |
names, and distfiles names to content digests. All keys have one or |
16 |
more prefixes separated by colons. For a digest key, the first |
17 |
prefix is "digest" and the second prefix is the hash algorithm name. |
18 |
For a filename key, the prefix is "filename". |
19 |
|
20 |
The value associated with a digest key is a set of file names. The |
21 |
value associated with a distfile key is a set of content revisions. |
22 |
Each content revision is expressed as a dictionary of digests which |
23 |
is suitable for construction of a DistfileName instance. |
24 |
|
25 |
A given content digest will translate to multiple distfile names if |
26 |
multiple associations have been created via the content db add |
27 |
method. The relationship between a content digest and a distfile |
28 |
name is similar to the relationship between an inode and a hardlink. |
29 |
|
30 |
Bug: https://bugs.gentoo.org/756778 |
31 |
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org> |
32 |
|
33 |
lib/portage/_emirrordist/Config.py | 6 + |
34 |
lib/portage/_emirrordist/ContentDB.py | 196 +++++++++++++++++++++++++++ |
35 |
lib/portage/_emirrordist/DeletionIterator.py | 25 +++- |
36 |
lib/portage/_emirrordist/DeletionTask.py | 8 ++ |
37 |
lib/portage/_emirrordist/FetchTask.py | 5 +- |
38 |
lib/portage/_emirrordist/main.py | 15 +- |
39 |
lib/portage/package/ebuild/fetch.py | 8 +- |
40 |
lib/portage/tests/ebuild/test_fetch.py | 148 ++++++++++++++++++++ |
41 |
man/emirrordist.1 | 6 +- |
42 |
9 files changed, 407 insertions(+), 10 deletions(-) |
43 |
|
44 |
diff --git a/lib/portage/_emirrordist/Config.py b/lib/portage/_emirrordist/Config.py |
45 |
index 1c7a27d66..a4b75809f 100644 |
46 |
--- a/lib/portage/_emirrordist/Config.py |
47 |
+++ b/lib/portage/_emirrordist/Config.py |
48 |
@@ -10,6 +10,7 @@ import time |
49 |
from portage import os |
50 |
from portage.package.ebuild.fetch import MirrorLayoutConfig |
51 |
from portage.util import grabdict, grablines |
52 |
+from .ContentDB import ContentDB |
53 |
|
54 |
class Config: |
55 |
def __init__(self, options, portdb, event_loop): |
56 |
@@ -65,6 +66,11 @@ class Config: |
57 |
self.distfiles_db = self._open_shelve( |
58 |
options.distfiles_db, 'distfiles') |
59 |
|
60 |
+ self.content_db = None |
61 |
+ if getattr(options, 'content_db', None) is not None: |
62 |
+ self.content_db = ContentDB(self._open_shelve( |
63 |
+ options.content_db, 'content')) |
64 |
+ |
65 |
self.deletion_db = None |
66 |
if getattr(options, 'deletion_db', None) is not None: |
67 |
self.deletion_db = self._open_shelve( |
68 |
|
69 |
diff --git a/lib/portage/_emirrordist/ContentDB.py b/lib/portage/_emirrordist/ContentDB.py |
70 |
new file mode 100644 |
71 |
index 000000000..d9ce3cc45 |
72 |
--- /dev/null |
73 |
+++ b/lib/portage/_emirrordist/ContentDB.py |
74 |
@@ -0,0 +1,196 @@ |
75 |
+# Copyright 2021 Gentoo Authors |
76 |
+# Distributed under the terms of the GNU General Public License v2 |
77 |
+ |
78 |
+import logging |
79 |
+import operator |
80 |
+import shelve |
81 |
+import typing |
82 |
+ |
83 |
+from portage.package.ebuild.fetch import DistfileName |
84 |
+ |
85 |
+ |
86 |
+class ContentDB: |
87 |
+ """ |
88 |
+ The content db serves to translate content digests to distfiles |
89 |
+ names, and distfiles names to content digests. All keys have one or |
90 |
+ more prefixes separated by colons. For a digest key, the first |
91 |
+ prefix is "digest" and the second prefix is the hash algorithm name. |
92 |
+ For a filename key, the prefix is "filename". |
93 |
+ |
94 |
+ The value associated with a digest key is a set of file names. The |
95 |
+ value associated with a distfile key is a set of content revisions. |
96 |
+ Each content revision is expressed as a dictionary of digests which |
97 |
+ is suitable for construction of a DistfileName instance. |
98 |
+ """ |
99 |
+ |
100 |
+ def __init__(self, shelve_instance: shelve.Shelf): |
101 |
+ self._shelve = shelve_instance |
102 |
+ |
103 |
+ def add(self, filename: DistfileName): |
104 |
+ """ |
105 |
+ Add file name and digests, creating a new content revision, or |
106 |
+ incrementing the reference count to an identical content revision |
107 |
+ if one exists. If the file name had previous content revisions, |
108 |
+ then they continue to exist independently of the new one. |
109 |
+ |
110 |
+ @param filename: file name with digests attribute |
111 |
+ """ |
112 |
+ distfile_str = str(filename) |
113 |
+ distfile_key = "filename:{}".format(distfile_str) |
114 |
+ for k, v in filename.digests.items(): |
115 |
+ if k != "size": |
116 |
+ digest_key = "digest:{}:{}".format(k.upper(), v.lower()) |
117 |
+ try: |
118 |
+ digest_files = self._shelve[digest_key] |
119 |
+ except KeyError: |
120 |
+ digest_files = set() |
121 |
+ digest_files.add(distfile_str) |
122 |
+ self._shelve[digest_key] = digest_files |
123 |
+ try: |
124 |
+ content_revisions = self._shelve[distfile_key] |
125 |
+ except KeyError: |
126 |
+ content_revisions = set() |
127 |
+ |
128 |
+ revision_key = tuple( |
129 |
+ sorted( |
130 |
+ ( |
131 |
+ (algo.upper(), filename.digests[algo.upper()].lower()) |
132 |
+ for algo in filename.digests |
133 |
+ if algo != "size" |
134 |
+ ), |
135 |
+ key=operator.itemgetter(0), |
136 |
+ ) |
137 |
+ ) |
138 |
+ content_revisions.add(revision_key) |
139 |
+ self._shelve[distfile_key] = content_revisions |
140 |
+ |
141 |
+ def remove(self, filename: DistfileName): |
142 |
+ """ |
143 |
+ Remove a file name and digests from the database. If identical |
144 |
+ content is still referenced by one or more other file names, |
145 |
+ then those references are preserved (like removing one of many |
146 |
+ hardlinks). Also, this file name may reference other content |
147 |
+ revisions with different digests, and those content revisions |
148 |
+ will remain as well. |
149 |
+ |
150 |
+ @param filename: file name with digests attribute |
151 |
+ """ |
152 |
+ distfile_key = "filename:{}".format(filename) |
153 |
+ try: |
154 |
+ content_revisions = self._shelve[distfile_key] |
155 |
+ except KeyError: |
156 |
+ pass |
157 |
+ else: |
158 |
+ remaining = set() |
159 |
+ for revision_key in content_revisions: |
160 |
+ if not any(digest_item in revision_key for digest_item in filename.digests.items()): |
161 |
+ remaining.add(revision_key) |
162 |
+ continue |
163 |
+ for k, v in revision_key: |
164 |
+ digest_key = "digest:{}:{}".format(k, v) |
165 |
+ try: |
166 |
+ digest_files = self._shelve[digest_key] |
167 |
+ except KeyError: |
168 |
+ digest_files = set() |
169 |
+ |
170 |
+ try: |
171 |
+ digest_files.remove(filename) |
172 |
+ except KeyError: |
173 |
+ pass |
174 |
+ |
175 |
+ if digest_files: |
176 |
+ self._shelve[digest_key] = digest_files |
177 |
+ else: |
178 |
+ try: |
179 |
+ del self._shelve[digest_key] |
180 |
+ except KeyError: |
181 |
+ pass |
182 |
+ |
183 |
+ if remaining: |
184 |
+ logging.debug(("drop '%s' revision(s) from content db") % filename) |
185 |
+ self._shelve[distfile_key] = remaining |
186 |
+ else: |
187 |
+ logging.debug(("drop '%s' from content db") % filename) |
188 |
+ try: |
189 |
+ del self._shelve[distfile_key] |
190 |
+ except KeyError: |
191 |
+ pass |
192 |
+ |
193 |
+ def get_filenames_translate( |
194 |
+ self, filename: typing.Union[str, DistfileName] |
195 |
+ ) -> typing.Generator[DistfileName, None, None]: |
196 |
+ """ |
197 |
+ Translate distfiles content digests to zero or more distfile names. |
198 |
+ If filename is already a distfile name, then it will pass |
199 |
+ through unchanged. |
200 |
+ |
201 |
+ A given content digest will translate to multiple distfile names if |
202 |
+ multiple associations have been created via the add method. The |
203 |
+ relationship between a content digest and a distfile name is similar |
204 |
+ to the relationship between an inode and a hardlink. |
205 |
+ |
206 |
+ @param filename: A filename listed by layout get_filenames |
207 |
+ """ |
208 |
+ if not isinstance(filename, DistfileName): |
209 |
+ filename = DistfileName(filename) |
210 |
+ |
211 |
+ # Match content digests with zero or more content revisions. |
212 |
+ matched_revisions = {} |
213 |
+ |
214 |
+ for k, v in filename.digests.items(): |
215 |
+ digest_item = (k.upper(), v.lower()) |
216 |
+ digest_key = "digest:{}:{}".format(*digest_item) |
217 |
+ try: |
218 |
+ digest_files = self._shelve[digest_key] |
219 |
+ except KeyError: |
220 |
+ continue |
221 |
+ |
222 |
+ for distfile_str in digest_files: |
223 |
+ matched_revisions.setdefault(distfile_str, set()) |
224 |
+ try: |
225 |
+ content_revisions = self._shelve["filename:{}".format(distfile_str)] |
226 |
+ except KeyError: |
227 |
+ pass |
228 |
+ else: |
229 |
+ for revision_key in content_revisions: |
230 |
+ if ( |
231 |
+ digest_item in revision_key |
232 |
+ and revision_key not in matched_revisions[distfile_str] |
233 |
+ ): |
234 |
+ matched_revisions[distfile_str].add(revision_key) |
235 |
+ yield DistfileName(distfile_str, digests=dict(revision_key)) |
236 |
+ |
237 |
+ if not any(matched_revisions.values()): |
238 |
+ # Since filename matched zero content revisions, allow |
239 |
+ # it to pass through unchanged (on the path toward deletion). |
240 |
+ yield filename |
241 |
+ |
242 |
+ def __len__(self): |
243 |
+ return len(self._shelve) |
244 |
+ |
245 |
+ def __contains__(self, k): |
246 |
+ return k in self._shelve |
247 |
+ |
248 |
+ def __iter__(self): |
249 |
+ return self._shelve.__iter__() |
250 |
+ |
251 |
+ def items(self): |
252 |
+ return self._shelve.items() |
253 |
+ |
254 |
+ def __setitem__(self, k, v): |
255 |
+ self._shelve[k] = v |
256 |
+ |
257 |
+ def __getitem__(self, k): |
258 |
+ return self._shelve[k] |
259 |
+ |
260 |
+ def __delitem__(self, k): |
261 |
+ del self._shelve[k] |
262 |
+ |
263 |
+ def get(self, k, *args): |
264 |
+ return self._shelve.get(k, *args) |
265 |
+ |
266 |
+ def close(self): |
267 |
+ self._shelve.close() |
268 |
+ |
269 |
+ def clear(self): |
270 |
+ self._shelve.clear() |
271 |
|
272 |
diff --git a/lib/portage/_emirrordist/DeletionIterator.py b/lib/portage/_emirrordist/DeletionIterator.py |
273 |
index 08985ed6c..ab4309f9a 100644 |
274 |
--- a/lib/portage/_emirrordist/DeletionIterator.py |
275 |
+++ b/lib/portage/_emirrordist/DeletionIterator.py |
276 |
@@ -1,10 +1,12 @@ |
277 |
-# Copyright 2013-2019 Gentoo Authors |
278 |
+# Copyright 2013-2021 Gentoo Authors |
279 |
# Distributed under the terms of the GNU General Public License v2 |
280 |
|
281 |
+import itertools |
282 |
import logging |
283 |
import stat |
284 |
|
285 |
from portage import os |
286 |
+from portage.package.ebuild.fetch import DistfileName |
287 |
from .DeletionTask import DeletionTask |
288 |
|
289 |
class DeletionIterator: |
290 |
@@ -21,8 +23,25 @@ class DeletionIterator: |
291 |
deletion_delay = self._config.options.deletion_delay |
292 |
start_time = self._config.start_time |
293 |
distfiles_set = set() |
294 |
- for layout in self._config.layouts: |
295 |
- distfiles_set.update(layout.get_filenames(distdir)) |
296 |
+ distfiles_set.update( |
297 |
+ ( |
298 |
+ filename |
299 |
+ if isinstance(filename, DistfileName) |
300 |
+ else DistfileName(filename) |
301 |
+ for filename in itertools.chain.from_iterable( |
302 |
+ layout.get_filenames(distdir) for layout in self._config.layouts |
303 |
+ ) |
304 |
+ ) |
305 |
+ if self._config.content_db is None |
306 |
+ else itertools.chain.from_iterable( |
307 |
+ ( |
308 |
+ self._config.content_db.get_filenames_translate(filename) |
309 |
+ for filename in itertools.chain.from_iterable( |
310 |
+ layout.get_filenames(distdir) for layout in self._config.layouts |
311 |
+ ) |
312 |
+ ) |
313 |
+ ) |
314 |
+ ) |
315 |
for filename in distfiles_set: |
316 |
# require at least one successful stat() |
317 |
exceptions = [] |
318 |
|
319 |
diff --git a/lib/portage/_emirrordist/DeletionTask.py b/lib/portage/_emirrordist/DeletionTask.py |
320 |
index 5eb01d840..73493c5a1 100644 |
321 |
--- a/lib/portage/_emirrordist/DeletionTask.py |
322 |
+++ b/lib/portage/_emirrordist/DeletionTask.py |
323 |
@@ -5,6 +5,7 @@ import errno |
324 |
import logging |
325 |
|
326 |
from portage import os |
327 |
+from portage.package.ebuild.fetch import ContentHashLayout |
328 |
from portage.util._async.FileCopier import FileCopier |
329 |
from _emerge.CompositeTask import CompositeTask |
330 |
|
331 |
@@ -99,6 +100,10 @@ class DeletionTask(CompositeTask): |
332 |
def _delete_links(self): |
333 |
success = True |
334 |
for layout in self.config.layouts: |
335 |
+ if isinstance(layout, ContentHashLayout) and not self.distfile.digests: |
336 |
+ logging.debug(("_delete_links: '%s' has " |
337 |
+ "no digests") % self.distfile) |
338 |
+ continue |
339 |
distfile_path = os.path.join( |
340 |
self.config.options.distfiles, |
341 |
layout.get_path(self.distfile)) |
342 |
@@ -134,6 +139,9 @@ class DeletionTask(CompositeTask): |
343 |
logging.debug(("drop '%s' from " |
344 |
"distfiles db") % self.distfile) |
345 |
|
346 |
+ if self.config.content_db is not None: |
347 |
+ self.config.content_db.remove(self.distfile) |
348 |
+ |
349 |
if self.config.deletion_db is not None: |
350 |
try: |
351 |
del self.config.deletion_db[self.distfile] |
352 |
|
353 |
diff --git a/lib/portage/_emirrordist/FetchTask.py b/lib/portage/_emirrordist/FetchTask.py |
354 |
index 997762082..5a48f91cd 100644 |
355 |
--- a/lib/portage/_emirrordist/FetchTask.py |
356 |
+++ b/lib/portage/_emirrordist/FetchTask.py |
357 |
@@ -1,4 +1,4 @@ |
358 |
-# Copyright 2013-2020 Gentoo Authors |
359 |
+# Copyright 2013-2021 Gentoo Authors |
360 |
# Distributed under the terms of the GNU General Public License v2 |
361 |
|
362 |
import collections |
363 |
@@ -47,6 +47,9 @@ class FetchTask(CompositeTask): |
364 |
# Convert _pkg_str to str in order to prevent pickle problems. |
365 |
self.config.distfiles_db[self.distfile] = str(self.cpv) |
366 |
|
367 |
+ if self.config.content_db is not None: |
368 |
+ self.config.content_db.add(self.distfile) |
369 |
+ |
370 |
if not self._have_needed_digests(): |
371 |
msg = "incomplete digests: %s" % " ".join(self.digests) |
372 |
self.scheduler.output(msg, background=self.background, |
373 |
|
374 |
diff --git a/lib/portage/_emirrordist/main.py b/lib/portage/_emirrordist/main.py |
375 |
index 8d00a05f5..2200ec715 100644 |
376 |
--- a/lib/portage/_emirrordist/main.py |
377 |
+++ b/lib/portage/_emirrordist/main.py |
378 |
@@ -1,4 +1,4 @@ |
379 |
-# Copyright 2013-2020 Gentoo Authors |
380 |
+# Copyright 2013-2021 Gentoo Authors |
381 |
# Distributed under the terms of the GNU General Public License v2 |
382 |
|
383 |
import argparse |
384 |
@@ -7,6 +7,7 @@ import sys |
385 |
|
386 |
import portage |
387 |
from portage import os |
388 |
+from portage.package.ebuild.fetch import ContentHashLayout |
389 |
from portage.util import normalize_path, _recursive_file_list |
390 |
from portage.util._async.run_main_scheduler import run_main_scheduler |
391 |
from portage.util._async.SchedulerInterface import SchedulerInterface |
392 |
@@ -151,6 +152,12 @@ common_options = ( |
393 |
"distfile belongs to", |
394 |
"metavar" : "FILE" |
395 |
}, |
396 |
+ { |
397 |
+ "longopt" : "--content-db", |
398 |
+ "help" : "database file used to map content digests to" |
399 |
+ "distfiles names (required for content-hash layout)", |
400 |
+ "metavar" : "FILE" |
401 |
+ }, |
402 |
{ |
403 |
"longopt" : "--recycle-dir", |
404 |
"help" : "directory for extended retention of files that " |
405 |
@@ -441,6 +448,12 @@ def emirrordist_main(args): |
406 |
if not options.mirror: |
407 |
parser.error('No action specified') |
408 |
|
409 |
+ if options.delete and config.content_db is None: |
410 |
+ for layout in config.layouts: |
411 |
+ if isinstance(layout, ContentHashLayout): |
412 |
+ parser.error("content-hash layout requires " |
413 |
+ "--content-db to be specified") |
414 |
+ |
415 |
returncode = os.EX_OK |
416 |
|
417 |
if options.mirror: |
418 |
|
419 |
diff --git a/lib/portage/package/ebuild/fetch.py b/lib/portage/package/ebuild/fetch.py |
420 |
index a683793f0..73abec595 100644 |
421 |
--- a/lib/portage/package/ebuild/fetch.py |
422 |
+++ b/lib/portage/package/ebuild/fetch.py |
423 |
@@ -365,10 +365,10 @@ class DistfileName(str): |
424 |
In order to prepare for a migration from filename-hash to |
425 |
content-hash layout, all consumers of the layout get_filenames |
426 |
method need to be updated to work with content digests as a |
427 |
- substitute for distfile names. For example, in order to prepare |
428 |
- emirrordist for content-hash, a key-value store needs to be |
429 |
- added as a means to associate distfile names with content |
430 |
- digest values yielded by the content-hash get_filenames |
431 |
+ substitute for distfile names. For example, emirrordist requires |
432 |
+ the --content-db option when working with a content-hash layout, |
433 |
+ which serves as a means to associate distfile names |
434 |
+ with content digest values yielded by the content-hash get_filenames |
435 |
implementation. |
436 |
""" |
437 |
def __new__(cls, s, digests=None): |
438 |
|
439 |
diff --git a/lib/portage/tests/ebuild/test_fetch.py b/lib/portage/tests/ebuild/test_fetch.py |
440 |
index d50a4cbfc..24990e4db 100644 |
441 |
--- a/lib/portage/tests/ebuild/test_fetch.py |
442 |
+++ b/lib/portage/tests/ebuild/test_fetch.py |
443 |
@@ -4,6 +4,7 @@ |
444 |
import functools |
445 |
import io |
446 |
import tempfile |
447 |
+import types |
448 |
|
449 |
import portage |
450 |
from portage import shutil, os |
451 |
@@ -28,6 +29,7 @@ from portage.package.ebuild.fetch import ( |
452 |
FlatLayout, |
453 |
MirrorLayoutConfig, |
454 |
) |
455 |
+from portage._emirrordist.Config import Config as EmirrordistConfig |
456 |
from _emerge.EbuildFetcher import EbuildFetcher |
457 |
from _emerge.Package import Package |
458 |
|
459 |
@@ -172,6 +174,16 @@ class EbuildFetchTestCase(TestCase): |
460 |
with open(os.path.join(settings['DISTDIR'], 'layout.conf'), 'wt') as f: |
461 |
f.write(layout_data) |
462 |
|
463 |
+ if any(isinstance(layout, ContentHashLayout) for layout in layouts): |
464 |
+ content_db = os.path.join(playground.eprefix, 'var/db/emirrordist/content.db') |
465 |
+ os.makedirs(os.path.dirname(content_db), exist_ok=True) |
466 |
+ try: |
467 |
+ os.unlink(content_db) |
468 |
+ except OSError: |
469 |
+ pass |
470 |
+ else: |
471 |
+ content_db = None |
472 |
+ |
473 |
# Demonstrate that fetch preserves a stale file in DISTDIR when no digests are given. |
474 |
foo_uri = {'foo': ('{scheme}://{host}:{port}/distfiles/foo'.format(scheme=scheme, host=host, port=server.server_port),)} |
475 |
foo_path = os.path.join(settings['DISTDIR'], 'foo') |
476 |
@@ -233,9 +245,13 @@ class EbuildFetchTestCase(TestCase): |
477 |
os.path.join(self.bindir, 'emirrordist'), |
478 |
'--distfiles', settings['DISTDIR'], |
479 |
'--config-root', settings['EPREFIX'], |
480 |
+ '--delete', |
481 |
'--repositories-configuration', settings.repositories.config_string(), |
482 |
'--repo', 'test_repo', '--mirror') |
483 |
|
484 |
+ if content_db is not None: |
485 |
+ emirrordist_cmd = emirrordist_cmd + ('--content-db', content_db,) |
486 |
+ |
487 |
env = settings.environ() |
488 |
env['PYTHONPATH'] = ':'.join( |
489 |
filter(None, [PORTAGE_PYM_PATH] + os.environ.get('PYTHONPATH', '').split(':'))) |
490 |
@@ -253,6 +269,19 @@ class EbuildFetchTestCase(TestCase): |
491 |
with open(os.path.join(settings['DISTDIR'], layouts[0].get_path(k)), 'rb') as f: |
492 |
self.assertEqual(f.read(), distfiles[k]) |
493 |
|
494 |
+ if content_db is not None: |
495 |
+ loop.run_until_complete( |
496 |
+ self._test_content_db( |
497 |
+ emirrordist_cmd, |
498 |
+ env, |
499 |
+ layouts, |
500 |
+ content_db, |
501 |
+ distfiles, |
502 |
+ settings, |
503 |
+ portdb, |
504 |
+ ) |
505 |
+ ) |
506 |
+ |
507 |
# Tests only work with one ebuild at a time, so the config |
508 |
# pool only needs a single config instance. |
509 |
class config_pool: |
510 |
@@ -427,6 +456,125 @@ class EbuildFetchTestCase(TestCase): |
511 |
settings.features.remove('skiprocheck') |
512 |
settings.features.add('distlocks') |
513 |
|
514 |
+ async def _test_content_db( |
515 |
+ self, emirrordist_cmd, env, layouts, content_db, distfiles, settings, portdb |
516 |
+ ): |
517 |
+ # Simulate distfile digest change for ContentDB. |
518 |
+ emdisopts = types.SimpleNamespace( |
519 |
+ content_db=content_db, distfiles=settings["DISTDIR"] |
520 |
+ ) |
521 |
+ with EmirrordistConfig( |
522 |
+ emdisopts, portdb, asyncio.get_event_loop() |
523 |
+ ) as emdisconf: |
524 |
+ # Copy revisions from bar to foo. |
525 |
+ for revision_key in emdisconf.content_db["filename:{}".format("bar")]: |
526 |
+ emdisconf.content_db.add( |
527 |
+ DistfileName("foo", digests=dict(revision_key)) |
528 |
+ ) |
529 |
+ |
530 |
+ # Copy revisions from foo to bar. |
531 |
+ for revision_key in emdisconf.content_db["filename:{}".format("foo")]: |
532 |
+ emdisconf.content_db.add( |
533 |
+ DistfileName("bar", digests=dict(revision_key)) |
534 |
+ ) |
535 |
+ |
536 |
+ content_db_state = dict(emdisconf.content_db.items()) |
537 |
+ self.assertEqual(content_db_state, dict(emdisconf.content_db.items())) |
538 |
+ self.assertEqual( |
539 |
+ [ |
540 |
+ k[len("filename:") :] |
541 |
+ for k in content_db_state |
542 |
+ if k.startswith("filename:") |
543 |
+ ], |
544 |
+ ["bar", "foo"], |
545 |
+ ) |
546 |
+ self.assertEqual( |
547 |
+ content_db_state["filename:foo"], content_db_state["filename:bar"] |
548 |
+ ) |
549 |
+ self.assertEqual(len(content_db_state["filename:foo"]), 2) |
550 |
+ |
551 |
+ for k in distfiles: |
552 |
+ try: |
553 |
+ os.unlink(os.path.join(settings["DISTDIR"], k)) |
554 |
+ except OSError: |
555 |
+ pass |
556 |
+ |
557 |
+ proc = await asyncio.create_subprocess_exec(*emirrordist_cmd, env=env) |
558 |
+ self.assertEqual(await proc.wait(), 0) |
559 |
+ |
560 |
+ for k in distfiles: |
561 |
+ with open( |
562 |
+ os.path.join(settings["DISTDIR"], layouts[0].get_path(k)), "rb" |
563 |
+ ) as f: |
564 |
+ self.assertEqual(f.read(), distfiles[k]) |
565 |
+ |
566 |
+ with EmirrordistConfig( |
567 |
+ emdisopts, portdb, asyncio.get_event_loop() |
568 |
+ ) as emdisconf: |
569 |
+ self.assertEqual(content_db_state, dict(emdisconf.content_db.items())) |
570 |
+ |
571 |
+ # Verify that remove works as expected |
572 |
+ filename = [filename for filename in distfiles if filename == "foo"][0] |
573 |
+ self.assertTrue(bool(filename.digests)) |
574 |
+ emdisconf.content_db.remove(filename) |
575 |
+ # foo should still have a content revision corresponding to bar's content. |
576 |
+ self.assertEqual( |
577 |
+ [ |
578 |
+ k[len("filename:") :] |
579 |
+ for k in emdisconf.content_db |
580 |
+ if k.startswith("filename:") |
581 |
+ ], |
582 |
+ ["bar", "foo"], |
583 |
+ ) |
584 |
+ self.assertEqual(len(emdisconf.content_db["filename:foo"]), 1) |
585 |
+ self.assertEqual( |
586 |
+ len( |
587 |
+ [ |
588 |
+ revision_key |
589 |
+ for revision_key in emdisconf.content_db["filename:foo"] |
590 |
+ if not filename.digests_equal( |
591 |
+ DistfileName( |
592 |
+ "foo", |
593 |
+ digests=dict(revision_key), |
594 |
+ ) |
595 |
+ ) |
596 |
+ ] |
597 |
+ ), |
598 |
+ 1, |
599 |
+ ) |
600 |
+ # bar should still have a content revision corresponding to foo's content. |
601 |
+ self.assertEqual(len(emdisconf.content_db["filename:bar"]), 2) |
602 |
+ self.assertEqual( |
603 |
+ len( |
604 |
+ [ |
605 |
+ revision_key |
606 |
+ for revision_key in emdisconf.content_db["filename:bar"] |
607 |
+ if filename.digests_equal( |
608 |
+ DistfileName( |
609 |
+ "bar", |
610 |
+ digests=dict(revision_key), |
611 |
+ ) |
612 |
+ ) |
613 |
+ ] |
614 |
+ ), |
615 |
+ 1, |
616 |
+ ) |
617 |
+ # remove the foo which refers to bar's content |
618 |
+ bar = [filename for filename in distfiles if filename == "bar"][0] |
619 |
+ foo_remaining = DistfileName("foo", digests=bar.digests) |
620 |
+ emdisconf.content_db.remove(foo_remaining) |
621 |
+ self.assertEqual( |
622 |
+ [ |
623 |
+ k[len("filename:") :] |
624 |
+ for k in emdisconf.content_db |
625 |
+ if k.startswith("filename:") |
626 |
+ ], |
627 |
+ ["bar"], |
628 |
+ ) |
629 |
+ self.assertRaises(KeyError, emdisconf.content_db.__getitem__, "filename:foo") |
630 |
+ # bar should still have a content revision corresponding to foo's content. |
631 |
+ self.assertEqual(len(emdisconf.content_db["filename:bar"]), 2) |
632 |
+ |
633 |
def test_flat_layout(self): |
634 |
self.assertTrue(FlatLayout.verify_args(('flat',))) |
635 |
self.assertFalse(FlatLayout.verify_args(('flat', 'extraneous-arg'))) |
636 |
|
637 |
diff --git a/man/emirrordist.1 b/man/emirrordist.1 |
638 |
index 45108ef8c..7ad10dfd0 100644 |
639 |
--- a/man/emirrordist.1 |
640 |
+++ b/man/emirrordist.1 |
641 |
@@ -1,4 +1,4 @@ |
642 |
-.TH "EMIRRORDIST" "1" "Dec 2015" "Portage VERSION" "Portage" |
643 |
+.TH "EMIRRORDIST" "1" "Feb 2021" "Portage VERSION" "Portage" |
644 |
.SH "NAME" |
645 |
emirrordist \- a fetch tool for mirroring of package distfiles |
646 |
.SH SYNOPSIS |
647 |
@@ -66,6 +66,10 @@ reporting purposes. Opened in append mode. |
648 |
Log file for scheduled deletions, with tab\-delimited output, for |
649 |
reporting purposes. Overwritten with each run. |
650 |
.TP |
651 |
+\fB\-\-content\-db\fR=\fIFILE\fR |
652 |
+Database file used to pair content digests with distfiles names |
653 |
+(required fo content\-hash layout). |
654 |
+.TP |
655 |
\fB\-\-delete\fR |
656 |
Enable deletion of unused distfiles. |
657 |
.TP |