1 |
commit: d1d01faf13dc051a4b8f6b6cdbe6b5b4e064b2c4 |
2 |
Author: Brian Dolbec <dolsen <AT> gentoo <DOT> org> |
3 |
AuthorDate: Mon Jan 4 07:57:36 2016 +0000 |
4 |
Commit: Brian Dolbec <dolsen <AT> gentoo <DOT> org> |
5 |
CommitDate: Sat Jan 30 20:25:23 2016 +0000 |
6 |
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=d1d01faf |
7 |
|
8 |
repoman: Move the large depency checks loop to a new plugin ProfileDependsChecks class |
9 |
|
10 |
pym/repoman/modules/scan/depend/__init__.py | 9 ++ |
11 |
pym/repoman/modules/scan/depend/profile.py | 211 ++++++++++++++++++++++++++++ |
12 |
pym/repoman/repos.py | 1 + |
13 |
pym/repoman/scanner.py | 181 +----------------------- |
14 |
4 files changed, 228 insertions(+), 174 deletions(-) |
15 |
|
16 |
diff --git a/pym/repoman/modules/scan/depend/__init__.py b/pym/repoman/modules/scan/depend/__init__.py |
17 |
index ebc716c..cddb7f1 100644 |
18 |
--- a/pym/repoman/modules/scan/depend/__init__.py |
19 |
+++ b/pym/repoman/modules/scan/depend/__init__.py |
20 |
@@ -19,6 +19,15 @@ module_spec = { |
21 |
'func_desc': { |
22 |
}, |
23 |
}, |
24 |
+ 'profile-module': { |
25 |
+ 'name': "profile", |
26 |
+ 'sourcefile': "profile", |
27 |
+ 'class': "ProfileDependsChecks", |
28 |
+ 'description': doc, |
29 |
+ 'functions': ['check'], |
30 |
+ 'func_desc': { |
31 |
+ }, |
32 |
+ }, |
33 |
} |
34 |
} |
35 |
|
36 |
|
37 |
diff --git a/pym/repoman/modules/scan/depend/profile.py b/pym/repoman/modules/scan/depend/profile.py |
38 |
new file mode 100644 |
39 |
index 0000000..91c52cc |
40 |
--- /dev/null |
41 |
+++ b/pym/repoman/modules/scan/depend/profile.py |
42 |
@@ -0,0 +1,211 @@ |
43 |
+# -*- coding:utf-8 -*- |
44 |
+ |
45 |
+ |
46 |
+import copy |
47 |
+from pprint import pformat |
48 |
+ |
49 |
+from _emerge.Package import Package |
50 |
+ |
51 |
+# import our initialized portage instance |
52 |
+from repoman._portage import portage |
53 |
+from portage.dep import Atom |
54 |
+ |
55 |
+ |
56 |
+def sort_key(item): |
57 |
+ return item[2].sub_path |
58 |
+ |
59 |
+ |
60 |
+class ProfileDependsChecks(object): |
61 |
+ |
62 |
+ def __init__(self, **kwargs): |
63 |
+ self.qatracker = kwargs.get('qatracker') |
64 |
+ self.portdb = kwargs.get('portdb') |
65 |
+ self.profiles = kwargs.get('profiles') |
66 |
+ self.options = kwargs.get('options') |
67 |
+ self.repo_settings = kwargs.get('repo_settings') |
68 |
+ self.include_arches = kwargs.get('include_arches') |
69 |
+ self.caches = kwargs.get('caches') |
70 |
+ self.repoman_incrementals = kwargs.get('repoman_incrementals') |
71 |
+ self.env = kwargs.get('env') |
72 |
+ self.have = kwargs.get('have') |
73 |
+ self.dev_keywords = kwargs.get('dev_keywords') |
74 |
+ |
75 |
+ def check(self, **kwargs): |
76 |
+ arches = kwargs.get('arches') |
77 |
+ ebuild = kwargs.get('ebuild') |
78 |
+ pkg = kwargs.get('pkg') |
79 |
+ baddepsyntax = kwargs.get('baddepsyntax') |
80 |
+ unknown_pkgs = kwargs.get('unknown_pkgs') |
81 |
+ |
82 |
+ relevant_profiles = [] |
83 |
+ for keyword, arch, groups in arches: |
84 |
+ if arch not in self.profiles: |
85 |
+ # A missing profile will create an error further down |
86 |
+ # during the KEYWORDS verification. |
87 |
+ continue |
88 |
+ |
89 |
+ if self.include_arches is not None: |
90 |
+ if arch not in self.include_arches: |
91 |
+ continue |
92 |
+ |
93 |
+ relevant_profiles.extend( |
94 |
+ (keyword, groups, prof) for prof in self.profiles[arch]) |
95 |
+ |
96 |
+ relevant_profiles.sort(key=sort_key) |
97 |
+ |
98 |
+ for keyword, groups, prof in relevant_profiles: |
99 |
+ |
100 |
+ is_stable_profile = prof.status == "stable" |
101 |
+ is_dev_profile = prof.status == "dev" and \ |
102 |
+ self.options.include_dev |
103 |
+ is_exp_profile = prof.status == "exp" and \ |
104 |
+ self.options.include_exp_profiles == 'y' |
105 |
+ if not (is_stable_profile or is_dev_profile or is_exp_profile): |
106 |
+ continue |
107 |
+ |
108 |
+ dep_settings = self.caches['arch'].get(prof.sub_path) |
109 |
+ if dep_settings is None: |
110 |
+ dep_settings = portage.config( |
111 |
+ config_profile_path=prof.abs_path, |
112 |
+ config_incrementals=self.repoman_incrementals, |
113 |
+ config_root=self.repo_settings.config_root, |
114 |
+ local_config=False, |
115 |
+ _unmatched_removal=self.options.unmatched_removal, |
116 |
+ env=self.env, repositories=self.repo_settings.repoman_settings.repositories) |
117 |
+ dep_settings.categories = self.repo_settings.repoman_settings.categories |
118 |
+ if self.options.without_mask: |
119 |
+ dep_settings._mask_manager_obj = \ |
120 |
+ copy.deepcopy(dep_settings._mask_manager) |
121 |
+ dep_settings._mask_manager._pmaskdict.clear() |
122 |
+ self.caches['arch'][prof.sub_path] = dep_settings |
123 |
+ |
124 |
+ xmatch_cache_key = (prof.sub_path, tuple(groups)) |
125 |
+ xcache = self.caches['arch_xmatch'].get(xmatch_cache_key) |
126 |
+ if xcache is None: |
127 |
+ self.portdb.melt() |
128 |
+ self.portdb.freeze() |
129 |
+ xcache = self.portdb.xcache |
130 |
+ xcache.update(self.caches['shared_xmatch']) |
131 |
+ self.caches['arch_xmatch'][xmatch_cache_key] = xcache |
132 |
+ |
133 |
+ self.repo_settings.trees[self.repo_settings.root]["porttree"].settings = dep_settings |
134 |
+ self.portdb.settings = dep_settings |
135 |
+ self.portdb.xcache = xcache |
136 |
+ |
137 |
+ dep_settings["ACCEPT_KEYWORDS"] = " ".join(groups) |
138 |
+ # just in case, prevent config.reset() from nuking these. |
139 |
+ dep_settings.backup_changes("ACCEPT_KEYWORDS") |
140 |
+ |
141 |
+ # This attribute is used in dbapi._match_use() to apply |
142 |
+ # use.stable.{mask,force} settings based on the stable |
143 |
+ # status of the parent package. This is required in order |
144 |
+ # for USE deps of unstable packages to be resolved correctly, |
145 |
+ # since otherwise use.stable.{mask,force} settings of |
146 |
+ # dependencies may conflict (see bug #456342). |
147 |
+ dep_settings._parent_stable = dep_settings._isStable(pkg) |
148 |
+ |
149 |
+ # Handle package.use*.{force,mask) calculation, for use |
150 |
+ # in dep_check. |
151 |
+ dep_settings.useforce = dep_settings._use_manager.getUseForce( |
152 |
+ pkg, stable=dep_settings._parent_stable) |
153 |
+ dep_settings.usemask = dep_settings._use_manager.getUseMask( |
154 |
+ pkg, stable=dep_settings._parent_stable) |
155 |
+ |
156 |
+ if not baddepsyntax: |
157 |
+ ismasked = not ebuild.archs or \ |
158 |
+ pkg.cpv not in self.portdb.xmatch("match-visible", |
159 |
+ Atom("%s::%s" % (pkg.cp, self.repo_settings.repo_config.name))) |
160 |
+ if ismasked: |
161 |
+ if not self.have['pmasked']: |
162 |
+ self.have['pmasked'] = bool(dep_settings._getMaskAtom( |
163 |
+ pkg.cpv, ebuild.metadata)) |
164 |
+ if self.options.ignore_masked: |
165 |
+ continue |
166 |
+ # we are testing deps for a masked package; give it some lee-way |
167 |
+ suffix = "masked" |
168 |
+ matchmode = "minimum-all-ignore-profile" |
169 |
+ else: |
170 |
+ suffix = "" |
171 |
+ matchmode = "minimum-visible" |
172 |
+ |
173 |
+ if not self.have['dev_keywords']: |
174 |
+ self.have['dev_keywords'] = \ |
175 |
+ bool(self.dev_keywords.intersection(ebuild.keywords)) |
176 |
+ |
177 |
+ if prof.status == "dev": |
178 |
+ suffix = suffix + "indev" |
179 |
+ |
180 |
+ for mytype in Package._dep_keys: |
181 |
+ |
182 |
+ mykey = "dependency.bad" + suffix |
183 |
+ myvalue = ebuild.metadata[mytype] |
184 |
+ if not myvalue: |
185 |
+ continue |
186 |
+ |
187 |
+ success, atoms = portage.dep_check( |
188 |
+ myvalue, self.portdb, dep_settings, |
189 |
+ use="all", mode=matchmode, trees=self.repo_settings.trees) |
190 |
+ |
191 |
+ if success: |
192 |
+ if atoms: |
193 |
+ |
194 |
+ # Don't bother with dependency.unknown for |
195 |
+ # cases in which *DEPEND.bad is triggered. |
196 |
+ for atom in atoms: |
197 |
+ # dep_check returns all blockers and they |
198 |
+ # aren't counted for *DEPEND.bad, so we |
199 |
+ # ignore them here. |
200 |
+ if not atom.blocker: |
201 |
+ unknown_pkgs.discard( |
202 |
+ (mytype, atom.unevaluated_atom)) |
203 |
+ |
204 |
+ if not prof.sub_path: |
205 |
+ # old-style virtuals currently aren't |
206 |
+ # resolvable with empty profile, since |
207 |
+ # 'virtuals' mappings are unavailable |
208 |
+ # (it would be expensive to search |
209 |
+ # for PROVIDE in all ebuilds) |
210 |
+ atoms = [ |
211 |
+ atom for atom in atoms if not ( |
212 |
+ atom.cp.startswith('virtual/') |
213 |
+ and not self.portdb.cp_list(atom.cp))] |
214 |
+ |
215 |
+ # we have some unsolvable deps |
216 |
+ # remove ! deps, which always show up as unsatisfiable |
217 |
+ atoms = [ |
218 |
+ str(atom.unevaluated_atom) |
219 |
+ for atom in atoms if not atom.blocker] |
220 |
+ |
221 |
+ # if we emptied out our list, continue: |
222 |
+ if not atoms: |
223 |
+ continue |
224 |
+ if self.options.output_style in ['column']: |
225 |
+ self.qatracker.add_error(mykey, |
226 |
+ "%s: %s: %s(%s) %s" |
227 |
+ % (ebuild.relative_path, mytype, keyword, |
228 |
+ prof, repr(atoms))) |
229 |
+ else: |
230 |
+ self.qatracker.add_error(mykey, |
231 |
+ "%s: %s: %s(%s)\n%s" |
232 |
+ % (ebuild.relative_path, mytype, keyword, |
233 |
+ prof, pformat(atoms, indent=6))) |
234 |
+ else: |
235 |
+ if self.options.output_style in ['column']: |
236 |
+ self.qatracker.add_error(mykey, |
237 |
+ "%s: %s: %s(%s) %s" |
238 |
+ % (ebuild.relative_path, mytype, keyword, |
239 |
+ prof, repr(atoms))) |
240 |
+ else: |
241 |
+ self.qatracker.add_error(mykey, |
242 |
+ "%s: %s: %s(%s)\n%s" |
243 |
+ % (ebuild.relative_path, mytype, keyword, |
244 |
+ prof, pformat(atoms, indent=6))) |
245 |
+ return {'continue': False} |
246 |
+ |
247 |
+ @property |
248 |
+ def runInPkgs(self): |
249 |
+ return (False, []) |
250 |
+ |
251 |
+ @property |
252 |
+ def runInEbuilds(self): |
253 |
+ return (True, [self.check]) |
254 |
|
255 |
diff --git a/pym/repoman/repos.py b/pym/repoman/repos.py |
256 |
index a34f785..39f53c1 100644 |
257 |
--- a/pym/repoman/repos.py |
258 |
+++ b/pym/repoman/repos.py |
259 |
@@ -28,6 +28,7 @@ class RepoSettings(object): |
260 |
self, config_root, portdir, portdir_overlay, |
261 |
repoman_settings=None, vcs_settings=None, options=None, |
262 |
qawarnings=None): |
263 |
+ self.config_root = config_root |
264 |
self.repoman_settings = repoman_settings |
265 |
self.vcs_settings = vcs_settings |
266 |
|
267 |
|
268 |
diff --git a/pym/repoman/scanner.py b/pym/repoman/scanner.py |
269 |
index 2625de4..b82cefb 100644 |
270 |
--- a/pym/repoman/scanner.py |
271 |
+++ b/pym/repoman/scanner.py |
272 |
@@ -2,17 +2,12 @@ |
273 |
|
274 |
from __future__ import print_function, unicode_literals |
275 |
|
276 |
-import copy |
277 |
import logging |
278 |
from itertools import chain |
279 |
-from pprint import pformat |
280 |
- |
281 |
-from _emerge.Package import Package |
282 |
|
283 |
import portage |
284 |
from portage import normalize_path |
285 |
from portage import os |
286 |
-from portage.dep import Atom |
287 |
from portage.output import green |
288 |
from repoman.modules.commit import repochecks |
289 |
from repoman.profile import check_profiles, dev_profile_keywords, setup_profile |
290 |
@@ -33,10 +28,6 @@ MODULE_CONTROLLER = Modules(path=MODULES_PATH, namepath="repoman.modules.scan") |
291 |
MODULE_NAMES = MODULE_CONTROLLER.module_names[:] |
292 |
|
293 |
|
294 |
-def sort_key(item): |
295 |
- return item[2].sub_path |
296 |
- |
297 |
- |
298 |
class Scanner(object): |
299 |
'''Primary scan class. Operates all the small Q/A tests and checks''' |
300 |
|
301 |
@@ -197,6 +188,12 @@ class Scanner(object): |
302 |
"checks": self.checks, |
303 |
"repo_metadata": self.repo_metadata, |
304 |
"profiles": self.profiles, |
305 |
+ "include_arches": self.include_arches, |
306 |
+ "caches": self.caches, |
307 |
+ "repoman_incrementals": self.repoman_incrementals, |
308 |
+ "env": self.env, |
309 |
+ "have": self.have, |
310 |
+ "dev_keywords": self.dev_keywords, |
311 |
} |
312 |
# initialize the plugin checks here |
313 |
self.modules = {} |
314 |
@@ -292,7 +289,7 @@ class Scanner(object): |
315 |
('license', 'LicenseChecks'), ('restrict', 'RestrictChecks'), |
316 |
('mtime', 'MtimeChecks'), ('multicheck', 'MultiCheck'), |
317 |
# Options.is_forced() is used to bypass further checks |
318 |
- ('options', 'Options'), |
319 |
+ ('options', 'Options'), ('profile', 'ProfileDependsChecks'), |
320 |
]: |
321 |
if mod[0]: |
322 |
mod_class = MODULE_CONTROLLER.get_class(mod[0]) |
323 |
@@ -320,170 +317,6 @@ class Scanner(object): |
324 |
if y_ebuild_continue: |
325 |
continue |
326 |
|
327 |
- relevant_profiles = [] |
328 |
- for keyword, arch, groups in dynamic_data['arches']: |
329 |
- if arch not in self.profiles: |
330 |
- # A missing profile will create an error further down |
331 |
- # during the KEYWORDS verification. |
332 |
- continue |
333 |
- |
334 |
- if self.include_arches is not None: |
335 |
- if arch not in self.include_arches: |
336 |
- continue |
337 |
- |
338 |
- relevant_profiles.extend( |
339 |
- (keyword, groups, prof) for prof in self.profiles[arch]) |
340 |
- |
341 |
- relevant_profiles.sort(key=sort_key) |
342 |
- |
343 |
- for keyword, groups, prof in relevant_profiles: |
344 |
- |
345 |
- is_stable_profile = prof.status == "stable" |
346 |
- is_dev_profile = prof.status == "dev" and \ |
347 |
- self.options.include_dev |
348 |
- is_exp_profile = prof.status == "exp" and \ |
349 |
- self.options.include_exp_profiles == 'y' |
350 |
- if not (is_stable_profile or is_dev_profile or is_exp_profile): |
351 |
- continue |
352 |
- |
353 |
- dep_settings = self.caches['arch'].get(prof.sub_path) |
354 |
- if dep_settings is None: |
355 |
- dep_settings = portage.config( |
356 |
- config_profile_path=prof.abs_path, |
357 |
- config_incrementals=self.repoman_incrementals, |
358 |
- config_root=self.config_root, |
359 |
- local_config=False, |
360 |
- _unmatched_removal=self.options.unmatched_removal, |
361 |
- env=self.env, repositories=self.repo_settings.repoman_settings.repositories) |
362 |
- dep_settings.categories = self.repo_settings.repoman_settings.categories |
363 |
- if self.options.without_mask: |
364 |
- dep_settings._mask_manager_obj = \ |
365 |
- copy.deepcopy(dep_settings._mask_manager) |
366 |
- dep_settings._mask_manager._pmaskdict.clear() |
367 |
- self.caches['arch'][prof.sub_path] = dep_settings |
368 |
- |
369 |
- xmatch_cache_key = (prof.sub_path, tuple(groups)) |
370 |
- xcache = self.caches['arch_xmatch'].get(xmatch_cache_key) |
371 |
- if xcache is None: |
372 |
- self.portdb.melt() |
373 |
- self.portdb.freeze() |
374 |
- xcache = self.portdb.xcache |
375 |
- xcache.update(self.caches['shared_xmatch']) |
376 |
- self.caches['arch_xmatch'][xmatch_cache_key] = xcache |
377 |
- |
378 |
- self.repo_settings.trees[self.repo_settings.root]["porttree"].settings = dep_settings |
379 |
- self.portdb.settings = dep_settings |
380 |
- self.portdb.xcache = xcache |
381 |
- |
382 |
- dep_settings["ACCEPT_KEYWORDS"] = " ".join(groups) |
383 |
- # just in case, prevent config.reset() from nuking these. |
384 |
- dep_settings.backup_changes("ACCEPT_KEYWORDS") |
385 |
- |
386 |
- # This attribute is used in dbapi._match_use() to apply |
387 |
- # use.stable.{mask,force} settings based on the stable |
388 |
- # status of the parent package. This is required in order |
389 |
- # for USE deps of unstable packages to be resolved correctly, |
390 |
- # since otherwise use.stable.{mask,force} settings of |
391 |
- # dependencies may conflict (see bug #456342). |
392 |
- dep_settings._parent_stable = dep_settings._isStable(dynamic_data['pkg']) |
393 |
- |
394 |
- # Handle package.use*.{force,mask) calculation, for use |
395 |
- # in dep_check. |
396 |
- dep_settings.useforce = dep_settings._use_manager.getUseForce( |
397 |
- dynamic_data['pkg'], stable=dep_settings._parent_stable) |
398 |
- dep_settings.usemask = dep_settings._use_manager.getUseMask( |
399 |
- dynamic_data['pkg'], stable=dep_settings._parent_stable) |
400 |
- |
401 |
- if not dynamic_data['baddepsyntax']: |
402 |
- ismasked = not dynamic_data['ebuild'].archs or \ |
403 |
- dynamic_data['pkg'].cpv not in self.portdb.xmatch("match-visible", |
404 |
- Atom("%s::%s" % (dynamic_data['pkg'].cp, self.repo_settings.repo_config.name))) |
405 |
- if ismasked: |
406 |
- if not self.have['pmasked']: |
407 |
- self.have['pmasked'] = bool(dep_settings._getMaskAtom( |
408 |
- dynamic_data['pkg'].cpv, dynamic_data['pkg']._metadata)) |
409 |
- if self.options.ignore_masked: |
410 |
- continue |
411 |
- # we are testing deps for a masked package; give it some lee-way |
412 |
- suffix = "masked" |
413 |
- matchmode = "minimum-all-ignore-profile" |
414 |
- else: |
415 |
- suffix = "" |
416 |
- matchmode = "minimum-visible" |
417 |
- |
418 |
- if not self.have['dev_keywords']: |
419 |
- self.have['dev_keywords'] = \ |
420 |
- bool(self.dev_keywords.intersection(dynamic_data['ebuild'].keywords)) |
421 |
- |
422 |
- if prof.status == "dev": |
423 |
- suffix = suffix + "indev" |
424 |
- |
425 |
- for mytype in Package._dep_keys: |
426 |
- |
427 |
- mykey = "dependency.bad" + suffix |
428 |
- myvalue = dynamic_data['ebuild'].metadata[mytype] |
429 |
- if not myvalue: |
430 |
- continue |
431 |
- |
432 |
- success, atoms = portage.dep_check( |
433 |
- myvalue, self.portdb, dep_settings, |
434 |
- use="all", mode=matchmode, trees=self.repo_settings.trees) |
435 |
- |
436 |
- if success: |
437 |
- if atoms: |
438 |
- |
439 |
- # Don't bother with dependency.unknown for |
440 |
- # cases in which *DEPEND.bad is triggered. |
441 |
- for atom in atoms: |
442 |
- # dep_check returns all blockers and they |
443 |
- # aren't counted for *DEPEND.bad, so we |
444 |
- # ignore them here. |
445 |
- if not atom.blocker: |
446 |
- dynamic_data['unknown_pkgs'].discard( |
447 |
- (mytype, atom.unevaluated_atom)) |
448 |
- |
449 |
- if not prof.sub_path: |
450 |
- # old-style virtuals currently aren't |
451 |
- # resolvable with empty profile, since |
452 |
- # 'virtuals' mappings are unavailable |
453 |
- # (it would be expensive to search |
454 |
- # for PROVIDE in all ebuilds) |
455 |
- atoms = [ |
456 |
- atom for atom in atoms if not ( |
457 |
- atom.cp.startswith('virtual/') |
458 |
- and not self.portdb.cp_list(atom.cp))] |
459 |
- |
460 |
- # we have some unsolvable deps |
461 |
- # remove ! deps, which always show up as unsatisfiable |
462 |
- atoms = [ |
463 |
- str(atom.unevaluated_atom) |
464 |
- for atom in atoms if not atom.blocker] |
465 |
- |
466 |
- # if we emptied out our list, continue: |
467 |
- if not atoms: |
468 |
- continue |
469 |
- if self.options.output_style in ['column']: |
470 |
- self.qatracker.add_error(mykey, |
471 |
- "%s: %s: %s(%s) %s" |
472 |
- % (dynamic_data['ebuild'].relative_path, mytype, keyword, |
473 |
- prof, repr(atoms))) |
474 |
- else: |
475 |
- self.qatracker.add_error(mykey, |
476 |
- "%s: %s: %s(%s)\n%s" |
477 |
- % (dynamic_data['ebuild'].relative_path, mytype, keyword, |
478 |
- prof, pformat(atoms, indent=6))) |
479 |
- else: |
480 |
- if self.options.output_style in ['column']: |
481 |
- self.qatracker.add_error(mykey, |
482 |
- "%s: %s: %s(%s) %s" |
483 |
- % (dynamic_data['ebuild'].relative_path, mytype, keyword, |
484 |
- prof, repr(atoms))) |
485 |
- else: |
486 |
- self.qatracker.add_error(mykey, |
487 |
- "%s: %s: %s(%s)\n%s" |
488 |
- % (dynamic_data['ebuild'].relative_path, mytype, keyword, |
489 |
- prof, pformat(atoms, indent=6))) |
490 |
- |
491 |
if not dynamic_data['baddepsyntax'] and dynamic_data['unknown_pkgs']: |
492 |
type_map = {} |
493 |
for mytype, atom in dynamic_data['unknown_pkgs']: |