1 |
The set of required hashes specify which hashes must be present for |
2 |
a distfile not to be refetched. It makes little sense to hardcode this |
3 |
value, and it is mostly useful for transition periods, so make it |
4 |
configurable via layout.conf and default to all hashes |
5 |
in manifest-hashes. |
6 |
--- |
7 |
pym/portage/_emirrordist/FetchTask.py | 2 +- |
8 |
pym/portage/const.py | 2 +- |
9 |
pym/portage/manifest.py | 26 +++++++++++++------- |
10 |
pym/portage/package/ebuild/digestgen.py | 4 ++-- |
11 |
pym/portage/repository/config.py | 42 ++++++++++++++++++++++++++------- |
12 |
pym/portage/tests/ebuild/test_config.py | 1 + |
13 |
repoman/pym/repoman/repos.py | 14 +++++++---- |
14 |
7 files changed, 64 insertions(+), 27 deletions(-) |
15 |
|
16 |
diff --git a/pym/portage/_emirrordist/FetchTask.py b/pym/portage/_emirrordist/FetchTask.py |
17 |
index 203b8c213..47908cb6b 100644 |
18 |
--- a/pym/portage/_emirrordist/FetchTask.py |
19 |
+++ b/pym/portage/_emirrordist/FetchTask.py |
20 |
@@ -20,7 +20,7 @@ from portage.util._async.PipeLogger import PipeLogger |
21 |
from portage.util._async.PopenProcess import PopenProcess |
22 |
from _emerge.CompositeTask import CompositeTask |
23 |
|
24 |
-default_hash_name = portage.const.MANIFEST2_REQUIRED_HASH |
25 |
+default_hash_name = portage.const.MANIFEST2_HASH_DEFAULT |
26 |
|
27 |
# Use --no-check-certificate since Manifest digests should provide |
28 |
# enough security, and certificates can be self-signed or whatnot. |
29 |
diff --git a/pym/portage/const.py b/pym/portage/const.py |
30 |
index 0af57d0e2..ec877b841 100644 |
31 |
--- a/pym/portage/const.py |
32 |
+++ b/pym/portage/const.py |
33 |
@@ -207,7 +207,7 @@ EAPI = 6 |
34 |
HASHING_BLOCKSIZE = 32768 |
35 |
|
36 |
MANIFEST2_HASH_DEFAULTS = frozenset(["SHA256", "SHA512", "WHIRLPOOL"]) |
37 |
-MANIFEST2_REQUIRED_HASH = "SHA512" |
38 |
+MANIFEST2_HASH_DEFAULT = "SHA512" |
39 |
|
40 |
MANIFEST2_IDENTIFIERS = ("AUX", "MISC", "DIST", "EBUILD") |
41 |
|
42 |
diff --git a/pym/portage/manifest.py b/pym/portage/manifest.py |
43 |
index 36c82690c..4ec20515e 100644 |
44 |
--- a/pym/portage/manifest.py |
45 |
+++ b/pym/portage/manifest.py |
46 |
@@ -26,8 +26,7 @@ from portage import _unicode_encode |
47 |
from portage.exception import DigestException, FileNotFound, \ |
48 |
InvalidDataType, MissingParameter, PermissionDenied, \ |
49 |
PortageException, PortagePackageException |
50 |
-from portage.const import (MANIFEST2_HASH_DEFAULTS, |
51 |
- MANIFEST2_IDENTIFIERS, MANIFEST2_REQUIRED_HASH) |
52 |
+from portage.const import (MANIFEST2_HASH_DEFAULTS, MANIFEST2_IDENTIFIERS) |
53 |
from portage.localization import _ |
54 |
|
55 |
_manifest_re = re.compile( |
56 |
@@ -128,7 +127,7 @@ class Manifest(object): |
57 |
parsers = (parseManifest2,) |
58 |
def __init__(self, pkgdir, distdir=None, fetchlist_dict=None, |
59 |
manifest1_compat=DeprecationWarning, from_scratch=False, thin=False, |
60 |
- allow_missing=False, allow_create=True, hashes=None, |
61 |
+ allow_missing=False, allow_create=True, hashes=None, required_hashes=None, |
62 |
find_invalid_path_char=None, strict_misc_digests=True): |
63 |
""" Create new Manifest instance for package in pkgdir. |
64 |
Do not parse Manifest file if from_scratch == True (only for internal use) |
65 |
@@ -148,15 +147,21 @@ class Manifest(object): |
66 |
self.pkgdir = _unicode_decode(pkgdir).rstrip(os.sep) + os.sep |
67 |
self.fhashdict = {} |
68 |
self.hashes = set() |
69 |
+ self.required_hashes = set() |
70 |
|
71 |
if hashes is None: |
72 |
hashes = MANIFEST2_HASH_DEFAULTS |
73 |
+ if required_hashes is None: |
74 |
+ required_hashes = hashes |
75 |
|
76 |
self.hashes.update(hashes) |
77 |
self.hashes.difference_update(hashname for hashname in \ |
78 |
list(self.hashes) if hashname not in get_valid_checksum_keys()) |
79 |
self.hashes.add("size") |
80 |
- self.hashes.add(MANIFEST2_REQUIRED_HASH) |
81 |
+ |
82 |
+ self.required_hashes.update(required_hashes) |
83 |
+ self.required_hashes.intersection_update(self.hashes) |
84 |
+ |
85 |
for t in MANIFEST2_IDENTIFIERS: |
86 |
self.fhashdict[t] = {} |
87 |
if not from_scratch: |
88 |
@@ -269,9 +274,11 @@ class Manifest(object): |
89 |
def checkIntegrity(self): |
90 |
for t in self.fhashdict: |
91 |
for f in self.fhashdict[t]: |
92 |
- if MANIFEST2_REQUIRED_HASH not in self.fhashdict[t][f]: |
93 |
- raise MissingParameter(_("Missing %s checksum: %s %s") % |
94 |
- (MANIFEST2_REQUIRED_HASH, t, f)) |
95 |
+ diff = self.required_hashes.difference( |
96 |
+ set(self.fhashdict[t][f])) |
97 |
+ if diff: |
98 |
+ raise MissingParameter(_("Missing %s checksum(s): %s %s") % |
99 |
+ (' '.join(diff), t, f)) |
100 |
|
101 |
def write(self, sign=False, force=False): |
102 |
""" Write Manifest instance to disk, optionally signing it. Returns |
103 |
@@ -422,7 +429,7 @@ class Manifest(object): |
104 |
self.fhashdict[ftype][fname] = {} |
105 |
if hashdict != None: |
106 |
self.fhashdict[ftype][fname].update(hashdict) |
107 |
- if not MANIFEST2_REQUIRED_HASH in self.fhashdict[ftype][fname]: |
108 |
+ if self.required_hashes.difference(set(self.fhashdict[ftype][fname])): |
109 |
self.updateFileHashes(ftype, fname, checkExisting=False, ignoreMissing=ignoreMissing) |
110 |
|
111 |
def removeFile(self, ftype, fname): |
112 |
@@ -462,6 +469,7 @@ class Manifest(object): |
113 |
fetchlist_dict=self.fetchlist_dict, from_scratch=True, |
114 |
thin=self.thin, allow_missing=self.allow_missing, |
115 |
allow_create=self.allow_create, hashes=self.hashes, |
116 |
+ required_hashes=self.required_hashes, |
117 |
find_invalid_path_char=self._find_invalid_path_char, |
118 |
strict_misc_digests=self.strict_misc_digests) |
119 |
pn = os.path.basename(self.pkgdir.rstrip(os.path.sep)) |
120 |
@@ -487,7 +495,7 @@ class Manifest(object): |
121 |
requiredDistfiles = distlist.copy() |
122 |
required_hash_types = set() |
123 |
required_hash_types.add("size") |
124 |
- required_hash_types.add(MANIFEST2_REQUIRED_HASH) |
125 |
+ required_hash_types.update(self.required_hashes) |
126 |
for f in distlist: |
127 |
fname = os.path.join(self.distdir, f) |
128 |
mystat = None |
129 |
diff --git a/pym/portage/package/ebuild/digestgen.py b/pym/portage/package/ebuild/digestgen.py |
130 |
index 95d02db9b..40c1b7288 100644 |
131 |
--- a/pym/portage/package/ebuild/digestgen.py |
132 |
+++ b/pym/portage/package/ebuild/digestgen.py |
133 |
@@ -11,7 +11,6 @@ portage.proxy.lazyimport.lazyimport(globals(), |
134 |
) |
135 |
|
136 |
from portage import os |
137 |
-from portage.const import MANIFEST2_REQUIRED_HASH |
138 |
from portage.dbapi.porttree import FetchlistDict |
139 |
from portage.dep import use_reduce |
140 |
from portage.exception import InvalidDependString, FileNotFound, \ |
141 |
@@ -58,6 +57,7 @@ def digestgen(myarchives=None, mysettings=None, myportdb=None): |
142 |
mytree = os.path.realpath(mytree) |
143 |
mf = mysettings.repositories.get_repo_for_location(mytree) |
144 |
|
145 |
+ repo_required_hashes = mf.manifest_required_hashes |
146 |
mf = mf.load_manifest(mysettings["O"], mysettings["DISTDIR"], |
147 |
fetchlist_dict=fetchlist_dict) |
148 |
|
149 |
@@ -72,7 +72,7 @@ def digestgen(myarchives=None, mysettings=None, myportdb=None): |
150 |
# exist before and after the transition. |
151 |
required_hash_types = set() |
152 |
required_hash_types.add("size") |
153 |
- required_hash_types.add(MANIFEST2_REQUIRED_HASH) |
154 |
+ required_hash_types.update(repo_required_hashes) |
155 |
dist_hashes = mf.fhashdict.get("DIST", {}) |
156 |
|
157 |
# To avoid accidental regeneration of digests with the incorrect |
158 |
diff --git a/pym/portage/repository/config.py b/pym/portage/repository/config.py |
159 |
index 3be0e8bda..be31ed3b1 100644 |
160 |
--- a/pym/portage/repository/config.py |
161 |
+++ b/pym/portage/repository/config.py |
162 |
@@ -12,8 +12,7 @@ import re |
163 |
import portage |
164 |
from portage import eclass_cache, os |
165 |
from portage.checksum import get_valid_checksum_keys |
166 |
-from portage.const import (MANIFEST2_REQUIRED_HASH, |
167 |
- PORTAGE_BASE_PATH, REPO_NAME_LOC, USER_CONFIG_PATH) |
168 |
+from portage.const import (PORTAGE_BASE_PATH, REPO_NAME_LOC, USER_CONFIG_PATH) |
169 |
from portage.eapi import eapi_allows_directories_on_profile_level_and_repository_level |
170 |
from portage.env.loaders import KeyValuePairFileLoader |
171 |
from portage.util import (normalize_path, read_corresponding_eapi_file, shlex_split, |
172 |
@@ -86,7 +85,7 @@ class RepoConfig(object): |
173 |
'sync_depth', 'sync_hooks_only_on_change', |
174 |
'sync_type', 'sync_umask', 'sync_uri', 'sync_user', 'thin_manifest', |
175 |
'update_changelog', '_eapis_banned', '_eapis_deprecated', |
176 |
- '_masters_orig', 'module_specific_options', |
177 |
+ '_masters_orig', 'module_specific_options', 'manifest_required_hashes', |
178 |
) |
179 |
|
180 |
def __init__(self, name, repo_opts, local_config=True): |
181 |
@@ -227,6 +226,7 @@ class RepoConfig(object): |
182 |
self.create_manifest = True |
183 |
self.disable_manifest = False |
184 |
self.manifest_hashes = None |
185 |
+ self.manifest_required_hashes = None |
186 |
self.update_changelog = False |
187 |
self.cache_formats = None |
188 |
self.portage1_profiles = True |
189 |
@@ -262,7 +262,7 @@ class RepoConfig(object): |
190 |
for value in ('allow-missing-manifest', |
191 |
'allow-provide-virtual', 'cache-formats', |
192 |
'create-manifest', 'disable-manifest', 'manifest-hashes', |
193 |
- 'profile-formats', |
194 |
+ 'manifest-required-hashes', 'profile-formats', |
195 |
'sign-commit', 'sign-manifest', 'thin-manifest', 'update-changelog'): |
196 |
setattr(self, value.lower().replace("-", "_"), layout_data[value]) |
197 |
|
198 |
@@ -337,6 +337,7 @@ class RepoConfig(object): |
199 |
kwds['allow_missing'] = self.allow_missing_manifest |
200 |
kwds['allow_create'] = self.create_manifest |
201 |
kwds['hashes'] = self.manifest_hashes |
202 |
+ kwds['required_hashes'] = self.manifest_required_hashes |
203 |
kwds['strict_misc_digests'] = self.strict_misc_digests |
204 |
if self.disable_manifest: |
205 |
kwds['from_scratch'] = True |
206 |
@@ -1046,20 +1047,41 @@ def parse_layout_conf(repo_location, repo_name=None): |
207 |
data['cache-formats'] = tuple(cache_formats) |
208 |
|
209 |
manifest_hashes = layout_data.get('manifest-hashes') |
210 |
+ manifest_required_hashes = layout_data.get('manifest-required-hashes') |
211 |
+ |
212 |
+ if manifest_required_hashes is not None and manifest_hashes is None: |
213 |
+ repo_name = _get_repo_name(repo_location, cached=repo_name) |
214 |
+ warnings.warn((_("Repository named '%(repo_name)s' specifies " |
215 |
+ "'manifest-required-hashes' setting without corresponding " |
216 |
+ "'manifest-hashes'. Portage will default it to match " |
217 |
+ "the required set but please add the missing entry " |
218 |
+ "to: %(layout_filename)s") % |
219 |
+ {"repo_name": repo_name or 'unspecified', |
220 |
+ "layout_filename":layout_filename}), |
221 |
+ SyntaxWarning) |
222 |
+ manifest_hashes = manifest_required_hashes |
223 |
+ |
224 |
if manifest_hashes is not None: |
225 |
+ # require all the hashes unless specified otherwise |
226 |
+ if manifest_required_hashes is None: |
227 |
+ manifest_required_hashes = manifest_hashes |
228 |
+ |
229 |
+ manifest_required_hashes = frozenset(manifest_required_hashes.upper().split()) |
230 |
manifest_hashes = frozenset(manifest_hashes.upper().split()) |
231 |
- if MANIFEST2_REQUIRED_HASH not in manifest_hashes: |
232 |
+ missing_required_hashes = manifest_required_hashes.difference( |
233 |
+ manifest_hashes) |
234 |
+ if missing_required_hashes: |
235 |
repo_name = _get_repo_name(repo_location, cached=repo_name) |
236 |
warnings.warn((_("Repository named '%(repo_name)s' has a " |
237 |
"'manifest-hashes' setting that does not contain " |
238 |
- "the '%(hash)s' hash which is required by this " |
239 |
- "portage version. You will have to upgrade portage " |
240 |
+ "the '%(hash)s' hashes which are listed in " |
241 |
+ "'manifest-required-hashes'. Please fix that file " |
242 |
"if you want to generate valid manifests for this " |
243 |
"repository: %(layout_filename)s") % |
244 |
{"repo_name": repo_name or 'unspecified', |
245 |
- "hash":MANIFEST2_REQUIRED_HASH, |
246 |
+ "hash": ' '.join(missing_required_hashes), |
247 |
"layout_filename":layout_filename}), |
248 |
- DeprecationWarning) |
249 |
+ SyntaxWarning) |
250 |
unsupported_hashes = manifest_hashes.difference( |
251 |
get_valid_checksum_keys()) |
252 |
if unsupported_hashes: |
253 |
@@ -1074,7 +1096,9 @@ def parse_layout_conf(repo_location, repo_name=None): |
254 |
"hashes":" ".join(sorted(unsupported_hashes)), |
255 |
"layout_filename":layout_filename}), |
256 |
DeprecationWarning) |
257 |
+ |
258 |
data['manifest-hashes'] = manifest_hashes |
259 |
+ data['manifest-required-hashes'] = manifest_required_hashes |
260 |
|
261 |
data['update-changelog'] = layout_data.get('update-changelog', 'false').lower() \ |
262 |
== 'true' |
263 |
diff --git a/pym/portage/tests/ebuild/test_config.py b/pym/portage/tests/ebuild/test_config.py |
264 |
index 1dd828538..dcb5ffe0d 100644 |
265 |
--- a/pym/portage/tests/ebuild/test_config.py |
266 |
+++ b/pym/portage/tests/ebuild/test_config.py |
267 |
@@ -228,6 +228,7 @@ class ConfigTestCase(TestCase): |
268 |
"profile-formats = pms", |
269 |
"thin-manifests = true", |
270 |
"manifest-hashes = SHA256 SHA512 WHIRLPOOL", |
271 |
+ "manifest-required-hashes = SHA512", |
272 |
"# use implicit masters" |
273 |
), |
274 |
} |
275 |
diff --git a/repoman/pym/repoman/repos.py b/repoman/pym/repoman/repos.py |
276 |
index 11a6231de..e942a599e 100644 |
277 |
--- a/repoman/pym/repoman/repos.py |
278 |
+++ b/repoman/pym/repoman/repos.py |
279 |
@@ -100,18 +100,22 @@ class RepoSettings(object): |
280 |
sys.exit(1) |
281 |
|
282 |
manifest_hashes = self.repo_config.manifest_hashes |
283 |
+ manifest_required_hashes = self.repo_config.manifest_required_hashes |
284 |
if manifest_hashes is None: |
285 |
manifest_hashes = portage.const.MANIFEST2_HASH_DEFAULTS |
286 |
+ manifest_required_hashes = manifest_hashes |
287 |
|
288 |
if options.mode in ("commit", "fix", "manifest"): |
289 |
- if portage.const.MANIFEST2_REQUIRED_HASH not in manifest_hashes: |
290 |
+ missing_required_hashes = manifest_required_hashes.difference( |
291 |
+ manifest_hashes) |
292 |
+ if missing_required_hashes: |
293 |
msg = ( |
294 |
"The 'manifest-hashes' setting in the '%s' repository's " |
295 |
- "metadata/layout.conf does not contain the '%s' hash which " |
296 |
- "is required by this portage version. You will have to " |
297 |
- "upgrade portage if you want to generate valid manifests for " |
298 |
+ "metadata/layout.conf does not contain the '%s' hashes which " |
299 |
+ "are listed in 'manifest-required-hashes'. Please fix that " |
300 |
+ "file if you want to generate valid manifests for " |
301 |
"this repository.") % ( |
302 |
- self.repo_config.name, portage.const.MANIFEST2_REQUIRED_HASH) |
303 |
+ self.repo_config.name, ' '.join(missing_required_hashes)) |
304 |
for line in textwrap.wrap(msg, 70): |
305 |
logging.error(line) |
306 |
sys.exit(1) |
307 |
-- |
308 |
2.15.0 |