Gentoo Archives: gentoo-portage-dev

From: Zac Medico <zmedico@g.o>
To: gentoo-portage-dev@l.g.o
Cc: Zac Medico <zmedico@g.o>
Subject: [gentoo-portage-dev] [PATCH] egencache: add --external-cache-only option (bug 741474)
Date: Sun, 13 Sep 2020 00:56:00
Message-Id: 20200913005336.44758-1-zmedico@gentoo.org
1 The --external-cache-only option is useful for client-side
2 use cases where writing cache files inside the repository itself
3 may interfere with repository verification. This option is
4 currently supported for --update and --update-pkg-desc-index
5 actions, for which consumers of the corresponding cache or
6 index files are already capable of consuming files from the
7 external cache directory (/var/cache/edb/dep).
8
9 Bug: https://bugs.gentoo.org/737470
10 Signed-off-by: Zac Medico <zmedico@g.o>
11 ---
12 bin/egencache | 44 ++++++++++++++++++++++---------------
13 cnf/repo.postsync.d/example | 19 ++++++++--------
14 man/egencache.1 | 7 ++++++
15 3 files changed, 42 insertions(+), 28 deletions(-)
16
17 diff --git a/bin/egencache b/bin/egencache
18 index 532e37f20..4ee63edad 100755
19 --- a/bin/egencache
20 +++ b/bin/egencache
21 @@ -107,6 +107,9 @@ def parse_args(args):
22 common.add_argument("--config-root",
23 help="location of portage config files",
24 dest="portage_configroot")
25 + common.add_argument("--external-cache-only",
26 + action="store_true",
27 + help="Output only to the external cache (not the repository itself)")
28 common.add_argument("--gpg-dir",
29 help="override the PORTAGE_GPG_DIR variable",
30 dest="gpg_dir")
31 @@ -246,7 +249,7 @@ def parse_args(args):
32
33 class GenCache:
34 def __init__(self, portdb, cp_iter=None, max_jobs=None, max_load=None,
35 - rsync=False):
36 + rsync=False, external_cache_only=False):
37 # The caller must set portdb.porttrees in order to constrain
38 # findname, cp_list, and cpv_list to the desired tree.
39 tree = portdb.porttrees[0]
40 @@ -263,18 +266,21 @@ class GenCache:
41 else:
42 self._cp_set = None
43 self._cp_missing = set()
44 - write_auxdb = "metadata-transfer" in portdb.settings.features
45 + write_auxdb = external_cache_only or "metadata-transfer" in portdb.settings.features
46 self._regen = MetadataRegen(portdb, cp_iter=cp_iter,
47 consumer=self._metadata_callback,
48 max_jobs=max_jobs, max_load=max_load,
49 write_auxdb=write_auxdb, main=True)
50 self.returncode = os.EX_OK
51 conf = portdb.repositories.get_repo_for_location(tree)
52 - self._trg_caches = tuple(conf.iter_pregenerated_caches(
53 - self._auxdbkeys, force=True, readonly=False))
54 - if not self._trg_caches:
55 - raise Exception("cache formats '%s' aren't supported" %
56 - (" ".join(conf.cache_formats),))
57 + if external_cache_only:
58 + self._trg_caches = ()
59 + else:
60 + self._trg_caches = tuple(conf.iter_pregenerated_caches(
61 + self._auxdbkeys, force=True, readonly=False))
62 + if not self._trg_caches:
63 + raise Exception("cache formats '%s' aren't supported" %
64 + (" ".join(conf.cache_formats),))
65
66 if rsync:
67 for trg_cache in self._trg_caches:
68 @@ -1092,7 +1098,8 @@ def egencache_main(args):
69 gen_cache = GenCache(portdb, cp_iter=cp_iter,
70 max_jobs=options.jobs,
71 max_load=options.load_average,
72 - rsync=options.rsync)
73 + rsync=options.rsync,
74 + external_cache_only=options.external_cache_only)
75 gen_cache.run()
76 if options.tolerant:
77 ret.append(os.EX_OK)
78 @@ -1100,20 +1107,21 @@ def egencache_main(args):
79 ret.append(gen_cache.returncode)
80
81 if options.update_pkg_desc_index:
82 - if repo_config.writable:
83 + if not options.external_cache_only and repo_config.writable:
84 writable_location = repo_config.location
85 else:
86 writable_location = os.path.join(portdb.depcachedir,
87 repo_config.location.lstrip(os.sep))
88 - msg = [
89 - "WARNING: Repository is not writable: %s" % (
90 - repo_config.location,),
91 - " Using cache directory instead: %s" % (
92 - writable_location,)
93 - ]
94 - msg = "".join(line + '\n' for line in msg)
95 - writemsg_level(msg,
96 - level=logging.WARNING, noiselevel=-1)
97 + if not options.external_cache_only:
98 + msg = [
99 + "WARNING: Repository is not writable: %s" % (
100 + repo_config.location,),
101 + " Using cache directory instead: %s" % (
102 + writable_location,)
103 + ]
104 + msg = "".join(line + '\n' for line in msg)
105 + writemsg_level(msg,
106 + level=logging.WARNING, noiselevel=-1)
107
108 gen_index = GenPkgDescIndex(repo_config, portdb, os.path.join(
109 writable_location, "metadata", "pkg_desc_index"),
110 diff --git a/cnf/repo.postsync.d/example b/cnf/repo.postsync.d/example
111 index d1e385c11..f7c6f5092 100644
112 --- a/cnf/repo.postsync.d/example
113 +++ b/cnf/repo.postsync.d/example
114 @@ -46,17 +46,16 @@ if [ -n "${repository_name}" ]; then
115 fi
116 fi
117
118 - # Regenerate the metadata/pkg_desc_index file if needed. It's not
119 + # Regenerate the metadata/pkg_desc_index file. This is not
120 # needed for https://gitweb.gentoo.org/repo/sync/gentoo.git which
121 - # provides a freshly generated copy.
122 - if [[ ! -e ${repository_path}/metadata/pkg_desc_index || (
123 - -d ${repository_path}/metadata/md5-cache &&
124 - -n $(find "${repository_path}/metadata/md5-cache" -type f -newer "${repository_path}/metadata/pkg_desc_index" -print -quit) ) ]]; then
125 - if ! egencache --update-pkg-desc-index --repo="${repository_name}" ${PORTAGE_VERBOSE+--verbose}
126 - then
127 - echo "!!! egencache failed!"
128 - ret=1
129 - fi
130 + # provides a freshly generated copy. The --external-cache-only
131 + # option causes the metadata/pkg_desc_index file to be written under
132 + # /var/cache/edb/dep instead of the repository itself, so that it
133 + # does not interfere with repository verification.
134 + if ! egencache --update-pkg-desc-index --external-cache-only --repo="${repository_name}" ${PORTAGE_VERBOSE+--verbose}
135 + then
136 + echo "!!! egencache failed!"
137 + ret=1
138 fi
139 fi
140
141 diff --git a/man/egencache.1 b/man/egencache.1
142 index ae7370e21..9826942d2 100644
143 --- a/man/egencache.1
144 +++ b/man/egencache.1
145 @@ -55,6 +55,13 @@ Location of portage config files.
146 .br
147 Defaults to /.
148 .TP
149 +.BR "\-\-external\-cache\-only"
150 +Output to the external cache rather than to the repository itself.
151 +This is useful for client\-side usage, where it may not be possible to
152 +output to the repository itself since that would interfere with
153 +repository verification. This option is supported by the \-\-update and
154 +\-\-update\-pkg\-desc\-index actions.
155 +.TP
156 .BR "\-\-gpg\-dir"
157 Override the PORTAGE_GPG_DIR variable.
158 .TP
159 --
160 2.25.3