1 |
commit: 9f63c395ee23b00d77d00e667a28624de5baff49 |
2 |
Author: Brian Dolbec <dolsen <AT> gentoo <DOT> org> |
3 |
AuthorDate: Thu Sep 17 15:29:11 2015 +0000 |
4 |
Commit: Brian Dolbec <dolsen <AT> gentoo <DOT> org> |
5 |
CommitDate: Mon Sep 21 23:42:46 2015 +0000 |
6 |
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=9f63c395 |
7 |
|
8 |
repoman: Move the primary checks loop to it's own class and file |
9 |
|
10 |
Only minimal changes were done for this initial move. |
11 |
The _scan_ebuilds() needs major hacking up into manageable chunks. |
12 |
Clean out code separation demarcation lines |
13 |
These lines were originally used to mark places where code was removed. |
14 |
And replaced with a class instance and/or function call. |
15 |
|
16 |
Signed-off-by: Brian Dolbec <dolsen <AT> gentoo.org> |
17 |
|
18 |
pym/repoman/main.py | 756 ++----------------------------------------------- |
19 |
pym/repoman/scanner.py | 715 ++++++++++++++++++++++++++++++++++++++++++++++ |
20 |
2 files changed, 743 insertions(+), 728 deletions(-) |
21 |
|
22 |
diff --git a/pym/repoman/main.py b/pym/repoman/main.py |
23 |
index e3d0472..2b2f91d 100755 |
24 |
--- a/pym/repoman/main.py |
25 |
+++ b/pym/repoman/main.py |
26 |
@@ -4,7 +4,6 @@ |
27 |
|
28 |
from __future__ import print_function, unicode_literals |
29 |
|
30 |
-import copy |
31 |
import errno |
32 |
import io |
33 |
import logging |
34 |
@@ -15,7 +14,6 @@ import sys |
35 |
import tempfile |
36 |
import platform |
37 |
from itertools import chain |
38 |
-from pprint import pformat |
39 |
|
40 |
from os import path as osp |
41 |
if osp.isfile(osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), ".portage_not_installed")): |
42 |
@@ -30,14 +28,12 @@ portage._disable_legacy_globals() |
43 |
from portage import os |
44 |
from portage import _encodings |
45 |
from portage import _unicode_encode |
46 |
-from _emerge.Package import Package |
47 |
from _emerge.UserQuery import UserQuery |
48 |
import portage.checksum |
49 |
import portage.const |
50 |
import portage.repository.config |
51 |
-from portage import cvstree, normalize_path |
52 |
+from portage import cvstree |
53 |
from portage import util |
54 |
-from portage.dep import Atom |
55 |
from portage.process import find_binary, spawn |
56 |
from portage.output import ( |
57 |
bold, create_color_func, green, nocolor, red) |
58 |
@@ -47,40 +43,18 @@ from portage.util import writemsg_level |
59 |
from portage.package.ebuild.digestgen import digestgen |
60 |
|
61 |
from repoman.argparser import parse_args |
62 |
-from repoman.checks.directories.files import FileChecks |
63 |
-from repoman.checks.ebuilds.checks import run_checks, checks_init |
64 |
-from repoman.checks.ebuilds.eclasses.live import LiveEclassChecks |
65 |
-from repoman.checks.ebuilds.eclasses.ruby import RubyEclassChecks |
66 |
-from repoman.checks.ebuilds.fetches import FetchChecks |
67 |
-from repoman.checks.ebuilds.keywords import KeywordChecks |
68 |
-from repoman.checks.ebuilds.isebuild import IsEbuild |
69 |
-from repoman.checks.ebuilds.thirdpartymirrors import ThirdPartyMirrors |
70 |
-from repoman.checks.ebuilds.manifests import Manifests |
71 |
-from repoman.check_missingslot import check_missingslot |
72 |
-from repoman.checks.ebuilds.misc import bad_split_check, pkg_invalid |
73 |
-from repoman.checks.ebuilds.pkgmetadata import PkgMetadata |
74 |
-from repoman.checks.ebuilds.use_flags import USEFlagChecks |
75 |
-from repoman.checks.ebuilds.variables.description import DescriptionChecks |
76 |
-from repoman.checks.ebuilds.variables.eapi import EAPIChecks |
77 |
-from repoman.checks.ebuilds.variables.license import LicenseChecks |
78 |
-from repoman.checks.ebuilds.variables.restrict import RestrictChecks |
79 |
-from repoman.ebuild import Ebuild |
80 |
+from repoman.checks.ebuilds.checks import checks_init |
81 |
from repoman.errors import err |
82 |
from repoman.gpg import gpgsign, need_signature |
83 |
-from repoman.modules.commit import repochecks |
84 |
-from repoman.profile import check_profiles, dev_profile_keywords, setup_profile |
85 |
from repoman.qa_data import ( |
86 |
format_qa_output, format_qa_output_column, qahelp, |
87 |
- qawarnings, qacats, missingvars, |
88 |
- suspect_virtual, suspect_rdepend) |
89 |
-from repoman.qa_tracker import QATracker |
90 |
-from repoman.repos import RepoSettings, repo_metadata |
91 |
-from repoman.scan import Changes, scan |
92 |
+ qawarnings, qacats) |
93 |
+from repoman.repos import RepoSettings |
94 |
+from repoman.scanner import Scanner |
95 |
from repoman._subprocess import repoman_popen, repoman_getstatusoutput |
96 |
from repoman import utilities |
97 |
from repoman.vcs.vcs import ( |
98 |
git_supports_gpg_sign, vcs_files_to_cps, VCSSettings) |
99 |
-from repoman.vcs.vcsstatus import VCSStatus |
100 |
|
101 |
|
102 |
if sys.hexversion >= 0x3000000: |
103 |
@@ -90,21 +64,11 @@ util.initialize_logger() |
104 |
|
105 |
bad = create_color_func("BAD") |
106 |
|
107 |
-live_eclasses = portage.const.LIVE_ECLASSES |
108 |
-non_ascii_re = re.compile(r'[^\x00-\x7f]') |
109 |
- |
110 |
# A sane umask is needed for files that portage creates. |
111 |
os.umask(0o22) |
112 |
|
113 |
-def sort_key(item): |
114 |
- return item[2].sub_path |
115 |
- |
116 |
|
117 |
def repoman_main(argv): |
118 |
- # Repoman sets it's own ACCEPT_KEYWORDS and we don't want it to |
119 |
- # behave incrementally. |
120 |
- repoman_incrementals = tuple( |
121 |
- x for x in portage.const.INCREMENTALS if x != 'ACCEPT_KEYWORDS') |
122 |
config_root = os.environ.get("PORTAGE_CONFIGROOT") |
123 |
repoman_settings = portage.config(config_root=config_root, local_config=False) |
124 |
|
125 |
@@ -142,30 +106,9 @@ def repoman_main(argv): |
126 |
repo_settings = RepoSettings( |
127 |
config_root, portdir, portdir_overlay, |
128 |
repoman_settings, vcs_settings, options, qawarnings) |
129 |
- |
130 |
repoman_settings = repo_settings.repoman_settings |
131 |
- |
132 |
portdb = repo_settings.portdb |
133 |
|
134 |
- if options.echangelog is None and repo_settings.repo_config.update_changelog: |
135 |
- options.echangelog = 'y' |
136 |
- |
137 |
- if vcs_settings.vcs is None: |
138 |
- options.echangelog = 'n' |
139 |
- |
140 |
- # The --echangelog option causes automatic ChangeLog generation, |
141 |
- # which invalidates changelog.ebuildadded and changelog.missing |
142 |
- # checks. |
143 |
- # Note: Some don't use ChangeLogs in distributed SCMs. |
144 |
- # It will be generated on server side from scm log, |
145 |
- # before package moves to the rsync server. |
146 |
- # This is needed because they try to avoid merge collisions. |
147 |
- # Gentoo's Council decided to always use the ChangeLog file. |
148 |
- # TODO: shouldn't this just be switched on the repo, iso the VCS? |
149 |
- is_echangelog_enabled = options.echangelog in ('y', 'force') |
150 |
- vcs_settings.vcs_is_cvs_or_svn = vcs_settings.vcs in ('cvs', 'svn') |
151 |
- check_changelog = not is_echangelog_enabled and vcs_settings.vcs_is_cvs_or_svn |
152 |
- |
153 |
if 'digest' in repoman_settings.features and options.digest != 'n': |
154 |
options.digest = 'y' |
155 |
|
156 |
@@ -178,663 +121,16 @@ def repoman_main(argv): |
157 |
env = os.environ.copy() |
158 |
env['FEATURES'] = env.get('FEATURES', '') + ' -unknown-features-warn' |
159 |
|
160 |
- categories = [] |
161 |
- for path in repo_settings.repo_config.eclass_db.porttrees: |
162 |
- categories.extend(portage.util.grabfile( |
163 |
- os.path.join(path, 'profiles', 'categories'))) |
164 |
- repoman_settings.categories = frozenset( |
165 |
- portage.util.stack_lists([categories], incremental=1)) |
166 |
- categories = repoman_settings.categories |
167 |
- |
168 |
- portdb.settings = repoman_settings |
169 |
- # We really only need to cache the metadata that's necessary for visibility |
170 |
- # filtering. Anything else can be discarded to reduce memory consumption. |
171 |
- portdb._aux_cache_keys.clear() |
172 |
- portdb._aux_cache_keys.update( |
173 |
- ["EAPI", "IUSE", "KEYWORDS", "repository", "SLOT"]) |
174 |
- |
175 |
- reposplit = myreporoot.split(os.path.sep) |
176 |
- repolevel = len(reposplit) |
177 |
- |
178 |
- ################### |
179 |
- commitmessage = None |
180 |
- if options.mode == 'commit': |
181 |
- repochecks.commit_check(repolevel, reposplit) |
182 |
- repochecks.conflict_check(vcs_settings, options) |
183 |
- |
184 |
- ################### |
185 |
- |
186 |
- # Make startdir relative to the canonical repodir, so that we can pass |
187 |
- # it to digestgen and it won't have to be canonicalized again. |
188 |
- if repolevel == 1: |
189 |
- startdir = repo_settings.repodir |
190 |
- else: |
191 |
- startdir = normalize_path(mydir) |
192 |
- startdir = os.path.join( |
193 |
- repo_settings.repodir, *startdir.split(os.sep)[-2 - repolevel + 3:]) |
194 |
- ################### |
195 |
- |
196 |
- |
197 |
- # get lists of valid keywords, licenses, and use |
198 |
- new_data = repo_metadata(repo_settings.portdb, repoman_settings) |
199 |
- kwlist, liclist, uselist, profile_list, \ |
200 |
- global_pmaskdict, liclist_deprecated = new_data |
201 |
- |
202 |
- repoman_settings['PORTAGE_ARCHLIST'] = ' '.join(sorted(kwlist)) |
203 |
- repoman_settings.backup_changes('PORTAGE_ARCHLIST') |
204 |
- |
205 |
- #################### |
206 |
- |
207 |
- profiles = setup_profile(profile_list) |
208 |
- |
209 |
- #################### |
210 |
- |
211 |
- check_profiles(profiles, repoman_settings.archlist()) |
212 |
- |
213 |
- #################### |
214 |
- |
215 |
- scanlist = scan(repolevel, reposplit, startdir, categories, repo_settings) |
216 |
- |
217 |
- #################### |
218 |
- |
219 |
- dev_keywords = dev_profile_keywords(profiles) |
220 |
- |
221 |
- qatracker = QATracker() |
222 |
- |
223 |
- |
224 |
- if options.mode == "manifest": |
225 |
- pass |
226 |
- elif options.pretend: |
227 |
- print(green("\nRepoMan does a once-over of the neighborhood...")) |
228 |
- else: |
229 |
- print(green("\nRepoMan scours the neighborhood...")) |
230 |
- |
231 |
- ##################### |
232 |
- |
233 |
- changed = Changes(options) |
234 |
- changed.scan(vcs_settings) |
235 |
- |
236 |
- ###################### |
237 |
- |
238 |
- have_pmasked = False |
239 |
- have_dev_keywords = False |
240 |
- dofail = 0 |
241 |
- |
242 |
- # NOTE: match-all caches are not shared due to potential |
243 |
- # differences between profiles in _get_implicit_iuse. |
244 |
- arch_caches = {} |
245 |
- arch_xmatch_caches = {} |
246 |
- shared_xmatch_caches = {"cp-list": {}} |
247 |
- |
248 |
- include_arches = None |
249 |
- if options.include_arches: |
250 |
- include_arches = set() |
251 |
- include_arches.update(*[x.split() for x in options.include_arches]) |
252 |
- |
253 |
- # Disable the "ebuild.notadded" check when not in commit mode and |
254 |
- # running `svn status` in every package dir will be too expensive. |
255 |
- |
256 |
- check_ebuild_notadded = not \ |
257 |
- (vcs_settings.vcs == "svn" and repolevel < 3 and options.mode != "commit") |
258 |
- |
259 |
- effective_scanlist = scanlist |
260 |
- if options.if_modified == "y": |
261 |
- effective_scanlist = sorted(vcs_files_to_cps( |
262 |
- chain(changed.changed, changed.new, changed.removed), |
263 |
- repolevel, reposplit, categories)) |
264 |
- |
265 |
- ###################### |
266 |
- # initialize our checks classes here before the big xpkg loop |
267 |
- manifester = Manifests(options, qatracker, repoman_settings) |
268 |
- is_ebuild = IsEbuild(repoman_settings, repo_settings, portdb, qatracker) |
269 |
- filescheck = FileChecks( |
270 |
- qatracker, repoman_settings, repo_settings, portdb, vcs_settings) |
271 |
- status_check = VCSStatus(vcs_settings, qatracker) |
272 |
- fetchcheck = FetchChecks( |
273 |
- qatracker, repoman_settings, repo_settings, portdb, vcs_settings) |
274 |
- pkgmeta = PkgMetadata(options, qatracker, repoman_settings) |
275 |
- thirdparty = ThirdPartyMirrors(repoman_settings, qatracker) |
276 |
- use_flag_checks = USEFlagChecks(qatracker, uselist) |
277 |
- keywordcheck = KeywordChecks(qatracker, options) |
278 |
- liveeclasscheck = LiveEclassChecks(qatracker) |
279 |
- rubyeclasscheck = RubyEclassChecks(qatracker) |
280 |
- eapicheck = EAPIChecks(qatracker, repo_settings) |
281 |
- descriptioncheck = DescriptionChecks(qatracker) |
282 |
- licensecheck = LicenseChecks(qatracker, liclist, liclist_deprecated) |
283 |
- restrictcheck = RestrictChecks(qatracker) |
284 |
- ###################### |
285 |
- |
286 |
- for xpkg in effective_scanlist: |
287 |
- # ebuilds and digests added to cvs respectively. |
288 |
- logging.info("checking package %s" % xpkg) |
289 |
- # save memory by discarding xmatch caches from previous package(s) |
290 |
- arch_xmatch_caches.clear() |
291 |
- eadded = [] |
292 |
- catdir, pkgdir = xpkg.split("/") |
293 |
- checkdir = repo_settings.repodir + "/" + xpkg |
294 |
- checkdir_relative = "" |
295 |
- if repolevel < 3: |
296 |
- checkdir_relative = os.path.join(pkgdir, checkdir_relative) |
297 |
- if repolevel < 2: |
298 |
- checkdir_relative = os.path.join(catdir, checkdir_relative) |
299 |
- checkdir_relative = os.path.join(".", checkdir_relative) |
300 |
- |
301 |
- ##################### |
302 |
- if manifester.run(checkdir, portdb): |
303 |
- continue |
304 |
- if not manifester.generated_manifest: |
305 |
- manifester.digest_check(xpkg, checkdir) |
306 |
- ###################### |
307 |
- |
308 |
- if options.mode == 'manifest-check': |
309 |
- continue |
310 |
- |
311 |
- checkdirlist = os.listdir(checkdir) |
312 |
- |
313 |
- ###################### |
314 |
- pkgs, allvalid = is_ebuild.check(checkdirlist, checkdir, xpkg) |
315 |
- if is_ebuild.continue_: |
316 |
- # If we can't access all the metadata then it's totally unsafe to |
317 |
- # commit since there's no way to generate a correct Manifest. |
318 |
- # Do not try to do any more QA checks on this package since missing |
319 |
- # metadata leads to false positives for several checks, and false |
320 |
- # positives confuse users. |
321 |
- can_force = False |
322 |
- continue |
323 |
- ###################### |
324 |
- |
325 |
- keywordcheck.prepare() |
326 |
- |
327 |
- # Sort ebuilds in ascending order for the KEYWORDS.dropped check. |
328 |
- ebuildlist = sorted(pkgs.values()) |
329 |
- ebuildlist = [pkg.pf for pkg in ebuildlist] |
330 |
- ####################### |
331 |
- filescheck.check( |
332 |
- checkdir, checkdirlist, checkdir_relative, changed.changed, changed.new) |
333 |
- ####################### |
334 |
- status_check.check(check_ebuild_notadded, checkdir, checkdir_relative, xpkg) |
335 |
- eadded.extend(status_check.eadded) |
336 |
- |
337 |
- ################# |
338 |
- fetchcheck.check( |
339 |
- xpkg, checkdir, checkdir_relative, changed.changed, changed.new) |
340 |
- ################# |
341 |
- |
342 |
- if check_changelog and "ChangeLog" not in checkdirlist: |
343 |
- qatracker.add_error("changelog.missing", xpkg + "/ChangeLog") |
344 |
- ################# |
345 |
- pkgmeta.check(xpkg, checkdir, checkdirlist, repolevel) |
346 |
- muselist = frozenset(pkgmeta.musedict) |
347 |
- ################# |
348 |
- |
349 |
- changelog_path = os.path.join(checkdir_relative, "ChangeLog") |
350 |
- changelog_modified = changelog_path in changed.changelogs |
351 |
- |
352 |
- # detect unused local USE-descriptions |
353 |
- used_useflags = set() |
354 |
- |
355 |
- for y_ebuild in ebuildlist: |
356 |
- ################## |
357 |
- ebuild = Ebuild( |
358 |
- repo_settings, repolevel, pkgdir, catdir, vcs_settings, |
359 |
- xpkg, y_ebuild) |
360 |
- ################## |
361 |
- |
362 |
- if check_changelog and not changelog_modified \ |
363 |
- and ebuild.ebuild_path in changed.new_ebuilds: |
364 |
- qatracker.add_error('changelog.ebuildadded', ebuild.relative_path) |
365 |
- |
366 |
- if ebuild.untracked(check_ebuild_notadded, y_ebuild, eadded): |
367 |
- # ebuild not added to vcs |
368 |
- qatracker.add_error( |
369 |
- "ebuild.notadded", xpkg + "/" + y_ebuild + ".ebuild") |
370 |
- |
371 |
- ################## |
372 |
- if bad_split_check(xpkg, y_ebuild, pkgdir, qatracker): |
373 |
- continue |
374 |
- ################### |
375 |
- pkg = pkgs[y_ebuild] |
376 |
- if pkg_invalid(pkg, qatracker, ebuild): |
377 |
- allvalid = False |
378 |
- continue |
379 |
- |
380 |
- myaux = pkg._metadata |
381 |
- eapi = myaux["EAPI"] |
382 |
- inherited = pkg.inherited |
383 |
- live_ebuild = live_eclasses.intersection(inherited) |
384 |
- |
385 |
- ####################### |
386 |
- eapicheck.check(pkg, ebuild) |
387 |
- ####################### |
388 |
- |
389 |
- for k, v in myaux.items(): |
390 |
- if not isinstance(v, basestring): |
391 |
- continue |
392 |
- m = non_ascii_re.search(v) |
393 |
- if m is not None: |
394 |
- qatracker.add_error( |
395 |
- "variable.invalidchar", |
396 |
- "%s: %s variable contains non-ASCII " |
397 |
- "character at position %s" % |
398 |
- (ebuild.relative_path, k, m.start() + 1)) |
399 |
- |
400 |
- if not fetchcheck.src_uri_error: |
401 |
- ####################### |
402 |
- thirdparty.check(myaux, ebuild.relative_path) |
403 |
- ####################### |
404 |
- if myaux.get("PROVIDE"): |
405 |
- qatracker.add_error("virtual.oldstyle", ebuild.relative_path) |
406 |
- |
407 |
- for pos, missing_var in enumerate(missingvars): |
408 |
- if not myaux.get(missing_var): |
409 |
- if catdir == "virtual" and \ |
410 |
- missing_var in ("HOMEPAGE", "LICENSE"): |
411 |
- continue |
412 |
- if live_ebuild and missing_var == "KEYWORDS": |
413 |
- continue |
414 |
- myqakey = missingvars[pos] + ".missing" |
415 |
- qatracker.add_error(myqakey, xpkg + "/" + y_ebuild + ".ebuild") |
416 |
- |
417 |
- if catdir == "virtual": |
418 |
- for var in ("HOMEPAGE", "LICENSE"): |
419 |
- if myaux.get(var): |
420 |
- myqakey = var + ".virtual" |
421 |
- qatracker.add_error(myqakey, ebuild.relative_path) |
422 |
- |
423 |
- ####################### |
424 |
- descriptioncheck.check(pkg, ebuild) |
425 |
- ####################### |
426 |
- |
427 |
- keywords = myaux["KEYWORDS"].split() |
428 |
- |
429 |
- ebuild_archs = set( |
430 |
- kw.lstrip("~") for kw in keywords if not kw.startswith("-")) |
431 |
- |
432 |
- ####################### |
433 |
- keywordcheck.check( |
434 |
- pkg, xpkg, ebuild, y_ebuild, keywords, ebuild_archs, changed, |
435 |
- live_ebuild, kwlist, profiles) |
436 |
- ####################### |
437 |
- |
438 |
- if live_ebuild and repo_settings.repo_config.name == "gentoo": |
439 |
- ####################### |
440 |
- liveeclasscheck.check( |
441 |
- pkg, xpkg, ebuild, y_ebuild, keywords, global_pmaskdict) |
442 |
- ####################### |
443 |
- |
444 |
- if options.ignore_arches: |
445 |
- arches = [[ |
446 |
- repoman_settings["ARCH"], repoman_settings["ARCH"], |
447 |
- repoman_settings["ACCEPT_KEYWORDS"].split()]] |
448 |
- else: |
449 |
- arches = set() |
450 |
- for keyword in keywords: |
451 |
- if keyword[0] == "-": |
452 |
- continue |
453 |
- elif keyword[0] == "~": |
454 |
- arch = keyword[1:] |
455 |
- if arch == "*": |
456 |
- for expanded_arch in profiles: |
457 |
- if expanded_arch == "**": |
458 |
- continue |
459 |
- arches.add( |
460 |
- (keyword, expanded_arch, ( |
461 |
- expanded_arch, "~" + expanded_arch))) |
462 |
- else: |
463 |
- arches.add((keyword, arch, (arch, keyword))) |
464 |
- else: |
465 |
- if keyword == "*": |
466 |
- for expanded_arch in profiles: |
467 |
- if expanded_arch == "**": |
468 |
- continue |
469 |
- arches.add( |
470 |
- (keyword, expanded_arch, (expanded_arch,))) |
471 |
- else: |
472 |
- arches.add((keyword, keyword, (keyword,))) |
473 |
- if not arches: |
474 |
- # Use an empty profile for checking dependencies of |
475 |
- # packages that have empty KEYWORDS. |
476 |
- arches.add(('**', '**', ('**',))) |
477 |
- |
478 |
- unknown_pkgs = set() |
479 |
- baddepsyntax = False |
480 |
- badlicsyntax = False |
481 |
- badprovsyntax = False |
482 |
- # catpkg = catdir + "/" + y_ebuild |
483 |
- |
484 |
- inherited_java_eclass = "java-pkg-2" in inherited or \ |
485 |
- "java-pkg-opt-2" in inherited |
486 |
- inherited_wxwidgets_eclass = "wxwidgets" in inherited |
487 |
- # operator_tokens = set(["||", "(", ")"]) |
488 |
- type_list, badsyntax = [], [] |
489 |
- for mytype in Package._dep_keys + ("LICENSE", "PROPERTIES", "PROVIDE"): |
490 |
- mydepstr = myaux[mytype] |
491 |
- |
492 |
- buildtime = mytype in Package._buildtime_keys |
493 |
- runtime = mytype in Package._runtime_keys |
494 |
- token_class = None |
495 |
- if mytype.endswith("DEPEND"): |
496 |
- token_class = portage.dep.Atom |
497 |
+ # Perform the main checks |
498 |
+ scanner = Scanner(repo_settings, myreporoot, config_root, options, |
499 |
+ vcs_settings, mydir, env) |
500 |
+ qatracker, can_force = scanner.scan_pkgs(can_force) |
501 |
|
502 |
- try: |
503 |
- atoms = portage.dep.use_reduce( |
504 |
- mydepstr, matchall=1, flat=True, |
505 |
- is_valid_flag=pkg.iuse.is_valid_flag, token_class=token_class) |
506 |
- except portage.exception.InvalidDependString as e: |
507 |
- atoms = None |
508 |
- badsyntax.append(str(e)) |
509 |
- |
510 |
- if atoms and mytype.endswith("DEPEND"): |
511 |
- if runtime and \ |
512 |
- "test?" in mydepstr.split(): |
513 |
- qatracker.add_error( |
514 |
- mytype + '.suspect', |
515 |
- "%s: 'test?' USE conditional in %s" % |
516 |
- (ebuild.relative_path, mytype)) |
517 |
- |
518 |
- for atom in atoms: |
519 |
- if atom == "||": |
520 |
- continue |
521 |
- |
522 |
- is_blocker = atom.blocker |
523 |
- |
524 |
- # Skip dependency.unknown for blockers, so that we |
525 |
- # don't encourage people to remove necessary blockers, |
526 |
- # as discussed in bug 382407. We use atom.without_use |
527 |
- # due to bug 525376. |
528 |
- if not is_blocker and \ |
529 |
- not portdb.xmatch("match-all", atom.without_use) and \ |
530 |
- not atom.cp.startswith("virtual/"): |
531 |
- unknown_pkgs.add((mytype, atom.unevaluated_atom)) |
532 |
- |
533 |
- if catdir != "virtual": |
534 |
- if not is_blocker and \ |
535 |
- atom.cp in suspect_virtual: |
536 |
- qatracker.add_error( |
537 |
- 'virtual.suspect', ebuild.relative_path + |
538 |
- ": %s: consider using '%s' instead of '%s'" % |
539 |
- (mytype, suspect_virtual[atom.cp], atom)) |
540 |
- if not is_blocker and \ |
541 |
- atom.cp.startswith("perl-core/"): |
542 |
- qatracker.add_error('dependency.perlcore', |
543 |
- ebuild.relative_path + |
544 |
- ": %s: please use '%s' instead of '%s'" % |
545 |
- (mytype, |
546 |
- atom.replace("perl-core/","virtual/perl-"), |
547 |
- atom)) |
548 |
- |
549 |
- if buildtime and \ |
550 |
- not is_blocker and \ |
551 |
- not inherited_java_eclass and \ |
552 |
- atom.cp == "virtual/jdk": |
553 |
- qatracker.add_error( |
554 |
- 'java.eclassesnotused', ebuild.relative_path) |
555 |
- elif buildtime and \ |
556 |
- not is_blocker and \ |
557 |
- not inherited_wxwidgets_eclass and \ |
558 |
- atom.cp == "x11-libs/wxGTK": |
559 |
- qatracker.add_error( |
560 |
- 'wxwidgets.eclassnotused', |
561 |
- "%s: %ss on x11-libs/wxGTK without inheriting" |
562 |
- " wxwidgets.eclass" % (ebuild.relative_path, mytype)) |
563 |
- elif runtime: |
564 |
- if not is_blocker and \ |
565 |
- atom.cp in suspect_rdepend: |
566 |
- qatracker.add_error( |
567 |
- mytype + '.suspect', |
568 |
- ebuild.relative_path + ": '%s'" % atom) |
569 |
- |
570 |
- if atom.operator == "~" and \ |
571 |
- portage.versions.catpkgsplit(atom.cpv)[3] != "r0": |
572 |
- qacat = 'dependency.badtilde' |
573 |
- qatracker.add_error( |
574 |
- qacat, "%s: %s uses the ~ operator" |
575 |
- " with a non-zero revision: '%s'" % |
576 |
- (ebuild.relative_path, mytype, atom)) |
577 |
- |
578 |
- check_missingslot(atom, mytype, eapi, portdb, qatracker, |
579 |
- ebuild.relative_path, myaux) |
580 |
- |
581 |
- type_list.extend([mytype] * (len(badsyntax) - len(type_list))) |
582 |
- |
583 |
- for m, b in zip(type_list, badsyntax): |
584 |
- if m.endswith("DEPEND"): |
585 |
- qacat = "dependency.syntax" |
586 |
- else: |
587 |
- qacat = m + ".syntax" |
588 |
- qatracker.add_error( |
589 |
- qacat, "%s: %s: %s" % (ebuild.relative_path, m, b)) |
590 |
- |
591 |
- badlicsyntax = len([z for z in type_list if z == "LICENSE"]) |
592 |
- badprovsyntax = len([z for z in type_list if z == "PROVIDE"]) |
593 |
- baddepsyntax = len(type_list) != badlicsyntax + badprovsyntax |
594 |
- badlicsyntax = badlicsyntax > 0 |
595 |
- badprovsyntax = badprovsyntax > 0 |
596 |
- |
597 |
- ################# |
598 |
- use_flag_checks.check(pkg, xpkg, ebuild, y_ebuild, muselist) |
599 |
- |
600 |
- ebuild_used_useflags = use_flag_checks.getUsedUseFlags() |
601 |
- used_useflags = used_useflags.union(ebuild_used_useflags) |
602 |
- ################# |
603 |
- rubyeclasscheck.check(pkg, ebuild) |
604 |
- ################# |
605 |
- |
606 |
- # license checks |
607 |
- if not badlicsyntax: |
608 |
- ################# |
609 |
- licensecheck.check(pkg, xpkg, ebuild, y_ebuild) |
610 |
- ################# |
611 |
- |
612 |
- ################# |
613 |
- restrictcheck.check(pkg, xpkg, ebuild, y_ebuild) |
614 |
- ################# |
615 |
- |
616 |
- # Syntax Checks |
617 |
- |
618 |
- if not vcs_settings.vcs_preserves_mtime: |
619 |
- if ebuild.ebuild_path not in changed.new_ebuilds and \ |
620 |
- ebuild.ebuild_path not in changed.ebuilds: |
621 |
- pkg.mtime = None |
622 |
- try: |
623 |
- # All ebuilds should have utf_8 encoding. |
624 |
- f = io.open( |
625 |
- _unicode_encode( |
626 |
- ebuild.full_path, encoding=_encodings['fs'], errors='strict'), |
627 |
- mode='r', encoding=_encodings['repo.content']) |
628 |
- try: |
629 |
- for check_name, e in run_checks(f, pkg): |
630 |
- qatracker.add_error( |
631 |
- check_name, ebuild.relative_path + ': %s' % e) |
632 |
- finally: |
633 |
- f.close() |
634 |
- except UnicodeDecodeError: |
635 |
- # A file.UTF8 failure will have already been recorded above. |
636 |
- pass |
637 |
- |
638 |
- if options.force: |
639 |
- # The dep_check() calls are the most expensive QA test. If --force |
640 |
- # is enabled, there's no point in wasting time on these since the |
641 |
- # user is intent on forcing the commit anyway. |
642 |
- continue |
643 |
- |
644 |
- relevant_profiles = [] |
645 |
- for keyword, arch, groups in arches: |
646 |
- if arch not in profiles: |
647 |
- # A missing profile will create an error further down |
648 |
- # during the KEYWORDS verification. |
649 |
- continue |
650 |
- |
651 |
- if include_arches is not None: |
652 |
- if arch not in include_arches: |
653 |
- continue |
654 |
- |
655 |
- relevant_profiles.extend( |
656 |
- (keyword, groups, prof) for prof in profiles[arch]) |
657 |
- |
658 |
- relevant_profiles.sort(key=sort_key) |
659 |
- |
660 |
- for keyword, groups, prof in relevant_profiles: |
661 |
- |
662 |
- is_stable_profile = prof.status == "stable" |
663 |
- is_dev_profile = prof.status == "dev" and \ |
664 |
- options.include_dev |
665 |
- is_exp_profile = prof.status == "exp" and \ |
666 |
- options.include_exp_profiles == 'y' |
667 |
- if not (is_stable_profile or is_dev_profile or is_exp_profile): |
668 |
- continue |
669 |
+ commitmessage = None |
670 |
|
671 |
- dep_settings = arch_caches.get(prof.sub_path) |
672 |
- if dep_settings is None: |
673 |
- dep_settings = portage.config( |
674 |
- config_profile_path=prof.abs_path, |
675 |
- config_incrementals=repoman_incrementals, |
676 |
- config_root=config_root, |
677 |
- local_config=False, |
678 |
- _unmatched_removal=options.unmatched_removal, |
679 |
- env=env, repositories=repoman_settings.repositories) |
680 |
- dep_settings.categories = repoman_settings.categories |
681 |
- if options.without_mask: |
682 |
- dep_settings._mask_manager_obj = \ |
683 |
- copy.deepcopy(dep_settings._mask_manager) |
684 |
- dep_settings._mask_manager._pmaskdict.clear() |
685 |
- arch_caches[prof.sub_path] = dep_settings |
686 |
- |
687 |
- xmatch_cache_key = (prof.sub_path, tuple(groups)) |
688 |
- xcache = arch_xmatch_caches.get(xmatch_cache_key) |
689 |
- if xcache is None: |
690 |
- portdb.melt() |
691 |
- portdb.freeze() |
692 |
- xcache = portdb.xcache |
693 |
- xcache.update(shared_xmatch_caches) |
694 |
- arch_xmatch_caches[xmatch_cache_key] = xcache |
695 |
- |
696 |
- repo_settings.trees[repo_settings.root]["porttree"].settings = dep_settings |
697 |
- portdb.settings = dep_settings |
698 |
- portdb.xcache = xcache |
699 |
- |
700 |
- dep_settings["ACCEPT_KEYWORDS"] = " ".join(groups) |
701 |
- # just in case, prevent config.reset() from nuking these. |
702 |
- dep_settings.backup_changes("ACCEPT_KEYWORDS") |
703 |
- |
704 |
- # This attribute is used in dbapi._match_use() to apply |
705 |
- # use.stable.{mask,force} settings based on the stable |
706 |
- # status of the parent package. This is required in order |
707 |
- # for USE deps of unstable packages to be resolved correctly, |
708 |
- # since otherwise use.stable.{mask,force} settings of |
709 |
- # dependencies may conflict (see bug #456342). |
710 |
- dep_settings._parent_stable = dep_settings._isStable(pkg) |
711 |
- |
712 |
- # Handle package.use*.{force,mask) calculation, for use |
713 |
- # in dep_check. |
714 |
- dep_settings.useforce = dep_settings._use_manager.getUseForce( |
715 |
- pkg, stable=dep_settings._parent_stable) |
716 |
- dep_settings.usemask = dep_settings._use_manager.getUseMask( |
717 |
- pkg, stable=dep_settings._parent_stable) |
718 |
- |
719 |
- if not baddepsyntax: |
720 |
- ismasked = not ebuild_archs or \ |
721 |
- pkg.cpv not in portdb.xmatch("match-visible", |
722 |
- Atom("%s::%s" % (pkg.cp, repo_settings.repo_config.name))) |
723 |
- if ismasked: |
724 |
- if not have_pmasked: |
725 |
- have_pmasked = bool(dep_settings._getMaskAtom( |
726 |
- pkg.cpv, pkg._metadata)) |
727 |
- if options.ignore_masked: |
728 |
- continue |
729 |
- # we are testing deps for a masked package; give it some lee-way |
730 |
- suffix = "masked" |
731 |
- matchmode = "minimum-all" |
732 |
- else: |
733 |
- suffix = "" |
734 |
- matchmode = "minimum-visible" |
735 |
- |
736 |
- if not have_dev_keywords: |
737 |
- have_dev_keywords = \ |
738 |
- bool(dev_keywords.intersection(keywords)) |
739 |
- |
740 |
- if prof.status == "dev": |
741 |
- suffix = suffix + "indev" |
742 |
- |
743 |
- for mytype in Package._dep_keys: |
744 |
- |
745 |
- mykey = "dependency.bad" + suffix |
746 |
- myvalue = myaux[mytype] |
747 |
- if not myvalue: |
748 |
- continue |
749 |
- |
750 |
- success, atoms = portage.dep_check( |
751 |
- myvalue, portdb, dep_settings, |
752 |
- use="all", mode=matchmode, trees=repo_settings.trees) |
753 |
- |
754 |
- if success: |
755 |
- if atoms: |
756 |
- |
757 |
- # Don't bother with dependency.unknown for |
758 |
- # cases in which *DEPEND.bad is triggered. |
759 |
- for atom in atoms: |
760 |
- # dep_check returns all blockers and they |
761 |
- # aren't counted for *DEPEND.bad, so we |
762 |
- # ignore them here. |
763 |
- if not atom.blocker: |
764 |
- unknown_pkgs.discard( |
765 |
- (mytype, atom.unevaluated_atom)) |
766 |
- |
767 |
- if not prof.sub_path: |
768 |
- # old-style virtuals currently aren't |
769 |
- # resolvable with empty profile, since |
770 |
- # 'virtuals' mappings are unavailable |
771 |
- # (it would be expensive to search |
772 |
- # for PROVIDE in all ebuilds) |
773 |
- atoms = [ |
774 |
- atom for atom in atoms if not ( |
775 |
- atom.cp.startswith('virtual/') |
776 |
- and not portdb.cp_list(atom.cp))] |
777 |
- |
778 |
- # we have some unsolvable deps |
779 |
- # remove ! deps, which always show up as unsatisfiable |
780 |
- atoms = [ |
781 |
- str(atom.unevaluated_atom) |
782 |
- for atom in atoms if not atom.blocker] |
783 |
- |
784 |
- # if we emptied out our list, continue: |
785 |
- if not atoms: |
786 |
- continue |
787 |
- qatracker.add_error(mykey, |
788 |
- "%s: %s: %s(%s)\n%s" |
789 |
- % (ebuild.relative_path, mytype, keyword, prof, |
790 |
- pformat(atoms, indent=6))) |
791 |
- else: |
792 |
- qatracker.add_error(mykey, |
793 |
- "%s: %s: %s(%s)\n%s" |
794 |
- % (ebuild.relative_path, mytype, keyword, prof, |
795 |
- pformat(atoms, indent=6))) |
796 |
- |
797 |
- if not baddepsyntax and unknown_pkgs: |
798 |
- type_map = {} |
799 |
- for mytype, atom in unknown_pkgs: |
800 |
- type_map.setdefault(mytype, set()).add(atom) |
801 |
- for mytype, atoms in type_map.items(): |
802 |
- qatracker.add_error( |
803 |
- "dependency.unknown", "%s: %s: %s" |
804 |
- % (ebuild.relative_path, mytype, ", ".join(sorted(atoms)))) |
805 |
- |
806 |
- # check if there are unused local USE-descriptions in metadata.xml |
807 |
- # (unless there are any invalids, to avoid noise) |
808 |
- if allvalid: |
809 |
- for myflag in muselist.difference(used_useflags): |
810 |
- qatracker.add_error( |
811 |
- "metadata.warning", |
812 |
- "%s/metadata.xml: unused local USE-description: '%s'" |
813 |
- % (xpkg, myflag)) |
814 |
- |
815 |
- |
816 |
- if options.if_modified == "y" and len(effective_scanlist) < 1: |
817 |
+ if options.if_modified == "y" and len(scanner.effective_scanlist) < 1: |
818 |
logging.warning("--if-modified is enabled, but no modified packages were found!") |
819 |
|
820 |
- if options.mode == "manifest": |
821 |
- sys.exit(dofail) |
822 |
- |
823 |
# dofail will be true if we have failed in at least one non-warning category |
824 |
dofail = 0 |
825 |
# dowarn will be true if we tripped any warnings |
826 |
@@ -842,6 +138,10 @@ def repoman_main(argv): |
827 |
# dofull will be true if we should print a "repoman full" informational message |
828 |
dofull = options.mode != 'full' |
829 |
|
830 |
+ # early out for manifest generation |
831 |
+ if options.mode == "manifest": |
832 |
+ sys.exit(dofail) |
833 |
+ |
834 |
for x in qacats: |
835 |
if x not in qatracker.fails: |
836 |
continue |
837 |
@@ -884,9 +184,9 @@ def repoman_main(argv): |
838 |
suggest_ignore_masked = False |
839 |
suggest_include_dev = False |
840 |
|
841 |
- if have_pmasked and not (options.without_mask or options.ignore_masked): |
842 |
+ if scanner.have['pmasked'] and not (options.without_mask or options.ignore_masked): |
843 |
suggest_ignore_masked = True |
844 |
- if have_dev_keywords and not options.include_dev: |
845 |
+ if scanner.have['dev_keywords'] and not options.include_dev: |
846 |
suggest_include_dev = True |
847 |
|
848 |
if suggest_ignore_masked or suggest_include_dev: |
849 |
@@ -1164,8 +464,8 @@ def repoman_main(argv): |
850 |
commitmessagefile = None |
851 |
if not commitmessage or not commitmessage.strip(): |
852 |
msg_prefix = "" |
853 |
- if repolevel > 1: |
854 |
- msg_prefix = "/".join(reposplit[1:]) + ": " |
855 |
+ if scanner.repolevel > 1: |
856 |
+ msg_prefix = "/".join(scanner.reposplit[1:]) + ": " |
857 |
|
858 |
try: |
859 |
editor = os.environ.get("EDITOR") |
860 |
@@ -1196,10 +496,10 @@ def repoman_main(argv): |
861 |
report_options.append("--force") |
862 |
if options.ignore_arches: |
863 |
report_options.append("--ignore-arches") |
864 |
- if include_arches is not None: |
865 |
+ if scanner.include_arches is not None: |
866 |
report_options.append( |
867 |
"--include-arches=\"%s\"" % |
868 |
- " ".join(sorted(include_arches))) |
869 |
+ " ".join(sorted(scanner.include_arches))) |
870 |
|
871 |
if vcs_settings.vcs == "git": |
872 |
# Use new footer only for git (see bug #438364). |
873 |
@@ -1238,18 +538,18 @@ def repoman_main(argv): |
874 |
committer_name = utilities.get_committer_name(env=repoman_settings) |
875 |
for x in sorted(vcs_files_to_cps( |
876 |
chain(myupdates, mymanifests, myremoved), |
877 |
- repolevel, reposplit, categories)): |
878 |
+ scanner.repolevel, scanner.reposplit, scanner.categories)): |
879 |
catdir, pkgdir = x.split("/") |
880 |
checkdir = repo_settings.repodir + "/" + x |
881 |
checkdir_relative = "" |
882 |
- if repolevel < 3: |
883 |
+ if scanner.repolevel < 3: |
884 |
checkdir_relative = os.path.join(pkgdir, checkdir_relative) |
885 |
- if repolevel < 2: |
886 |
+ if scanner.repolevel < 2: |
887 |
checkdir_relative = os.path.join(catdir, checkdir_relative) |
888 |
checkdir_relative = os.path.join(".", checkdir_relative) |
889 |
|
890 |
changelog_path = os.path.join(checkdir_relative, "ChangeLog") |
891 |
- changelog_modified = changelog_path in changed.changelogs |
892 |
+ changelog_modified = changelog_path in scanner.changed.changelogs |
893 |
if changelog_modified and options.echangelog != 'force': |
894 |
continue |
895 |
|
896 |
@@ -1459,7 +759,7 @@ def repoman_main(argv): |
897 |
if modified: |
898 |
portage.util.write_atomic(x, b''.join(mylines), mode='wb') |
899 |
|
900 |
- if repolevel == 1: |
901 |
+ if scanner.repolevel == 1: |
902 |
utilities.repoman_sez( |
903 |
"\"You're rather crazy... " |
904 |
"doing the entire repository.\"\n") |
905 |
@@ -1467,7 +767,7 @@ def repoman_main(argv): |
906 |
if vcs_settings.vcs in ('cvs', 'svn') and (myupdates or myremoved): |
907 |
for x in sorted(vcs_files_to_cps( |
908 |
chain(myupdates, myremoved, mymanifests), |
909 |
- repolevel, reposplit, categories)): |
910 |
+ scanner.repolevel, scanner.reposplit, scanner.categories)): |
911 |
repoman_settings["O"] = os.path.join(repo_settings.repodir, x) |
912 |
digestgen(mysettings=repoman_settings, myportdb=portdb) |
913 |
|
914 |
@@ -1480,7 +780,7 @@ def repoman_main(argv): |
915 |
try: |
916 |
for x in sorted(vcs_files_to_cps( |
917 |
chain(myupdates, myremoved, mymanifests), |
918 |
- repolevel, reposplit, categories)): |
919 |
+ scanner.repolevel, scanner.reposplit, scanner.categories)): |
920 |
repoman_settings["O"] = os.path.join(repo_settings.repodir, x) |
921 |
manifest_path = os.path.join(repoman_settings["O"], "Manifest") |
922 |
if not need_signature(manifest_path): |
923 |
|
924 |
diff --git a/pym/repoman/scanner.py b/pym/repoman/scanner.py |
925 |
new file mode 100644 |
926 |
index 0000000..44ff33b |
927 |
--- /dev/null |
928 |
+++ b/pym/repoman/scanner.py |
929 |
@@ -0,0 +1,715 @@ |
930 |
+ |
931 |
+import copy |
932 |
+import io |
933 |
+import logging |
934 |
+import re |
935 |
+import sys |
936 |
+from itertools import chain |
937 |
+from pprint import pformat |
938 |
+ |
939 |
+from _emerge.Package import Package |
940 |
+ |
941 |
+import portage |
942 |
+from portage import normalize_path |
943 |
+from portage import os |
944 |
+from portage import _encodings |
945 |
+from portage import _unicode_encode |
946 |
+from portage.dep import Atom |
947 |
+from portage.output import green |
948 |
+from repoman.checks.directories.files import FileChecks |
949 |
+from repoman.checks.ebuilds.checks import run_checks |
950 |
+from repoman.checks.ebuilds.eclasses.live import LiveEclassChecks |
951 |
+from repoman.checks.ebuilds.eclasses.ruby import RubyEclassChecks |
952 |
+from repoman.checks.ebuilds.fetches import FetchChecks |
953 |
+from repoman.checks.ebuilds.keywords import KeywordChecks |
954 |
+from repoman.checks.ebuilds.isebuild import IsEbuild |
955 |
+from repoman.checks.ebuilds.thirdpartymirrors import ThirdPartyMirrors |
956 |
+from repoman.checks.ebuilds.manifests import Manifests |
957 |
+from repoman.check_missingslot import check_missingslot |
958 |
+from repoman.checks.ebuilds.misc import bad_split_check, pkg_invalid |
959 |
+from repoman.checks.ebuilds.pkgmetadata import PkgMetadata |
960 |
+from repoman.checks.ebuilds.use_flags import USEFlagChecks |
961 |
+from repoman.checks.ebuilds.variables.description import DescriptionChecks |
962 |
+from repoman.checks.ebuilds.variables.eapi import EAPIChecks |
963 |
+from repoman.checks.ebuilds.variables.license import LicenseChecks |
964 |
+from repoman.checks.ebuilds.variables.restrict import RestrictChecks |
965 |
+from repoman.ebuild import Ebuild |
966 |
+from repoman.modules.commit import repochecks |
967 |
+from repoman.profile import check_profiles, dev_profile_keywords, setup_profile |
968 |
+from repoman.qa_data import missingvars, suspect_virtual, suspect_rdepend |
969 |
+from repoman.qa_tracker import QATracker |
970 |
+from repoman.repos import repo_metadata |
971 |
+from repoman.scan import Changes, scan |
972 |
+from repoman.vcs.vcsstatus import VCSStatus |
973 |
+from repoman.vcs.vcs import vcs_files_to_cps |
974 |
+ |
975 |
+if sys.hexversion >= 0x3000000: |
976 |
+ basestring = str |
977 |
+ |
978 |
+NON_ASCII_RE = re.compile(r'[^\x00-\x7f]') |
979 |
+ |
980 |
+ |
981 |
+def sort_key(item): |
982 |
+ return item[2].sub_path |
983 |
+ |
984 |
+ |
985 |
+ |
986 |
+class Scanner(object): |
987 |
+ '''Primary scan class. Operates all the small Q/A tests and checks''' |
988 |
+ |
989 |
+ def __init__(self, repo_settings, myreporoot, config_root, options, |
990 |
+ vcs_settings, mydir, env): |
991 |
+ '''Class __init__''' |
992 |
+ self.repo_settings = repo_settings |
993 |
+ self.config_root = config_root |
994 |
+ self.options = options |
995 |
+ self.vcs_settings = vcs_settings |
996 |
+ self.env = env |
997 |
+ |
998 |
+ # Repoman sets it's own ACCEPT_KEYWORDS and we don't want it to |
999 |
+ # behave incrementally. |
1000 |
+ self.repoman_incrementals = tuple( |
1001 |
+ x for x in portage.const.INCREMENTALS if x != 'ACCEPT_KEYWORDS') |
1002 |
+ |
1003 |
+ self.categories = [] |
1004 |
+ for path in self.repo_settings.repo_config.eclass_db.porttrees: |
1005 |
+ self.categories.extend(portage.util.grabfile( |
1006 |
+ os.path.join(path, 'profiles', 'categories'))) |
1007 |
+ self.repo_settings.repoman_settings.categories = frozenset( |
1008 |
+ portage.util.stack_lists([self.categories], incremental=1)) |
1009 |
+ self.categories = self.repo_settings.repoman_settings.categories |
1010 |
+ |
1011 |
+ self.portdb = repo_settings.portdb |
1012 |
+ self.portdb.settings = self.repo_settings.repoman_settings |
1013 |
+ # We really only need to cache the metadata that's necessary for visibility |
1014 |
+ # filtering. Anything else can be discarded to reduce memory consumption. |
1015 |
+ self.portdb._aux_cache_keys.clear() |
1016 |
+ self.portdb._aux_cache_keys.update( |
1017 |
+ ["EAPI", "IUSE", "KEYWORDS", "repository", "SLOT"]) |
1018 |
+ |
1019 |
+ self.reposplit = myreporoot.split(os.path.sep) |
1020 |
+ self.repolevel = len(self.reposplit) |
1021 |
+ |
1022 |
+ if self.options.mode == 'commit': |
1023 |
+ repochecks.commit_check(self.repolevel, self.reposplit) |
1024 |
+ repochecks.conflict_check(self.vcs_settings, self.options) |
1025 |
+ |
1026 |
+ # Make startdir relative to the canonical repodir, so that we can pass |
1027 |
+ # it to digestgen and it won't have to be canonicalized again. |
1028 |
+ if self.repolevel == 1: |
1029 |
+ startdir = self.repo_settings.repodir |
1030 |
+ else: |
1031 |
+ startdir = normalize_path(mydir) |
1032 |
+ startdir = os.path.join( |
1033 |
+ self.repo_settings.repodir, *startdir.split(os.sep)[-2 - self.repolevel + 3:]) |
1034 |
+ |
1035 |
+ # get lists of valid keywords, licenses, and use |
1036 |
+ new_data = repo_metadata(self.portdb, self.repo_settings.repoman_settings) |
1037 |
+ kwlist, liclist, uselist, profile_list, \ |
1038 |
+ global_pmaskdict, liclist_deprecated = new_data |
1039 |
+ self.repo_metadata = { |
1040 |
+ 'kwlist': kwlist, |
1041 |
+ 'liclist': liclist, |
1042 |
+ 'uselist': uselist, |
1043 |
+ 'profile_list': profile_list, |
1044 |
+ 'pmaskdict': global_pmaskdict, |
1045 |
+ 'lic_deprecated': liclist_deprecated, |
1046 |
+ } |
1047 |
+ |
1048 |
+ self.repo_settings.repoman_settings['PORTAGE_ARCHLIST'] = ' '.join(sorted(kwlist)) |
1049 |
+ self.repo_settings.repoman_settings.backup_changes('PORTAGE_ARCHLIST') |
1050 |
+ |
1051 |
+ self.profiles = setup_profile(profile_list) |
1052 |
+ |
1053 |
+ check_profiles(self.profiles, self.repo_settings.repoman_settings.archlist()) |
1054 |
+ |
1055 |
+ scanlist = scan(self.repolevel, self.reposplit, startdir, self.categories, self.repo_settings) |
1056 |
+ |
1057 |
+ self.dev_keywords = dev_profile_keywords(self.profiles) |
1058 |
+ |
1059 |
+ self.qatracker = QATracker() |
1060 |
+ |
1061 |
+ if self.options.echangelog is None and self.repo_settings.repo_config.update_changelog: |
1062 |
+ self.options.echangelog = 'y' |
1063 |
+ |
1064 |
+ if self.vcs_settings.vcs is None: |
1065 |
+ self.options.echangelog = 'n' |
1066 |
+ |
1067 |
+ self.check = {} |
1068 |
+ # The --echangelog option causes automatic ChangeLog generation, |
1069 |
+ # which invalidates changelog.ebuildadded and changelog.missing |
1070 |
+ # checks. |
1071 |
+ # Note: Some don't use ChangeLogs in distributed SCMs. |
1072 |
+ # It will be generated on server side from scm log, |
1073 |
+ # before package moves to the rsync server. |
1074 |
+ # This is needed because they try to avoid merge collisions. |
1075 |
+ # Gentoo's Council decided to always use the ChangeLog file. |
1076 |
+ # TODO: shouldn't this just be switched on the repo, iso the VCS? |
1077 |
+ is_echangelog_enabled = self.options.echangelog in ('y', 'force') |
1078 |
+ self.vcs_settings.vcs_is_cvs_or_svn = self.vcs_settings.vcs in ('cvs', 'svn') |
1079 |
+ self.check['changelog'] = not is_echangelog_enabled and self.vcs_settings.vcs_is_cvs_or_svn |
1080 |
+ |
1081 |
+ if self.options.mode == "manifest": |
1082 |
+ pass |
1083 |
+ elif self.options.pretend: |
1084 |
+ print(green("\nRepoMan does a once-over of the neighborhood...")) |
1085 |
+ else: |
1086 |
+ print(green("\nRepoMan scours the neighborhood...")) |
1087 |
+ |
1088 |
+ self.changed = Changes(self.options) |
1089 |
+ self.changed.scan(self.vcs_settings) |
1090 |
+ |
1091 |
+ self.have = { |
1092 |
+ 'pmasked': False, |
1093 |
+ 'dev_keywords': False, |
1094 |
+ } |
1095 |
+ |
1096 |
+ # NOTE: match-all caches are not shared due to potential |
1097 |
+ # differences between profiles in _get_implicit_iuse. |
1098 |
+ self.caches = { |
1099 |
+ 'arch': {}, |
1100 |
+ 'arch_xmatch': {}, |
1101 |
+ 'shared_xmatch': {"cp-list": {}}, |
1102 |
+ } |
1103 |
+ |
1104 |
+ self.include_arches = None |
1105 |
+ if self.options.include_arches: |
1106 |
+ self.include_arches = set() |
1107 |
+ self.include_arches.update(*[x.split() for x in self.options.include_arches]) |
1108 |
+ |
1109 |
+ # Disable the "ebuild.notadded" check when not in commit mode and |
1110 |
+ # running `svn status` in every package dir will be too expensive. |
1111 |
+ self.check['ebuild_notadded'] = not \ |
1112 |
+ (self.vcs_settings.vcs == "svn" and self.repolevel < 3 and self.options.mode != "commit") |
1113 |
+ |
1114 |
+ self.effective_scanlist = scanlist |
1115 |
+ if self.options.if_modified == "y": |
1116 |
+ self.effective_scanlist = sorted(vcs_files_to_cps( |
1117 |
+ chain(self.changed.changed, self.changed.new, self.changed.removed), |
1118 |
+ self.repolevel, self.reposplit, self.categories)) |
1119 |
+ |
1120 |
+ self.live_eclasses = portage.const.LIVE_ECLASSES |
1121 |
+ |
1122 |
+ # initialize our checks classes here before the big xpkg loop |
1123 |
+ self.manifester = Manifests(self.options, self.qatracker, self.repo_settings.repoman_settings) |
1124 |
+ self.is_ebuild = IsEbuild(self.repo_settings.repoman_settings, self.repo_settings, self.portdb, self.qatracker) |
1125 |
+ self.filescheck = FileChecks( |
1126 |
+ self.qatracker, self.repo_settings.repoman_settings, self.repo_settings, self.portdb, self.vcs_settings) |
1127 |
+ self.status_check = VCSStatus(self.vcs_settings, self.qatracker) |
1128 |
+ self.fetchcheck = FetchChecks( |
1129 |
+ self.qatracker, self.repo_settings.repoman_settings, self.repo_settings, self.portdb, self.vcs_settings) |
1130 |
+ self.pkgmeta = PkgMetadata(self.options, self.qatracker, self.repo_settings.repoman_settings) |
1131 |
+ self.thirdparty = ThirdPartyMirrors(self.repo_settings.repoman_settings, self.qatracker) |
1132 |
+ self.use_flag_checks = USEFlagChecks(self.qatracker, uselist) |
1133 |
+ self.keywordcheck = KeywordChecks(self.qatracker, self.options) |
1134 |
+ self.liveeclasscheck = LiveEclassChecks(self.qatracker) |
1135 |
+ self.rubyeclasscheck = RubyEclassChecks(self.qatracker) |
1136 |
+ self.eapicheck = EAPIChecks(self.qatracker, self.repo_settings) |
1137 |
+ self.descriptioncheck = DescriptionChecks(self.qatracker) |
1138 |
+ self.licensecheck = LicenseChecks(self.qatracker, liclist, liclist_deprecated) |
1139 |
+ self.restrictcheck = RestrictChecks(self.qatracker) |
1140 |
+ |
1141 |
+ |
1142 |
+ def scan_pkgs(self, can_force): |
1143 |
+ for xpkg in self.effective_scanlist: |
1144 |
+ # ebuilds and digests added to cvs respectively. |
1145 |
+ logging.info("checking package %s" % xpkg) |
1146 |
+ # save memory by discarding xmatch caches from previous package(s) |
1147 |
+ self.caches['arch_xmatch'].clear() |
1148 |
+ self.eadded = [] |
1149 |
+ catdir, pkgdir = xpkg.split("/") |
1150 |
+ checkdir = self.repo_settings.repodir + "/" + xpkg |
1151 |
+ checkdir_relative = "" |
1152 |
+ if self.repolevel < 3: |
1153 |
+ checkdir_relative = os.path.join(pkgdir, checkdir_relative) |
1154 |
+ if self.repolevel < 2: |
1155 |
+ checkdir_relative = os.path.join(catdir, checkdir_relative) |
1156 |
+ checkdir_relative = os.path.join(".", checkdir_relative) |
1157 |
+ |
1158 |
+ if self.manifester.run(checkdir, self.portdb): |
1159 |
+ continue |
1160 |
+ if not self.manifester.generated_manifest: |
1161 |
+ self.manifester.digest_check(xpkg, checkdir) |
1162 |
+ if self.options.mode == 'manifest-check': |
1163 |
+ continue |
1164 |
+ |
1165 |
+ checkdirlist = os.listdir(checkdir) |
1166 |
+ |
1167 |
+ self.pkgs, self.allvalid = self.is_ebuild.check(checkdirlist, checkdir, xpkg) |
1168 |
+ if self.is_ebuild.continue_: |
1169 |
+ # If we can't access all the metadata then it's totally unsafe to |
1170 |
+ # commit since there's no way to generate a correct Manifest. |
1171 |
+ # Do not try to do any more QA checks on this package since missing |
1172 |
+ # metadata leads to false positives for several checks, and false |
1173 |
+ # positives confuse users. |
1174 |
+ can_force = False |
1175 |
+ continue |
1176 |
+ |
1177 |
+ self.keywordcheck.prepare() |
1178 |
+ |
1179 |
+ # Sort ebuilds in ascending order for the KEYWORDS.dropped check. |
1180 |
+ ebuildlist = sorted(self.pkgs.values()) |
1181 |
+ ebuildlist = [pkg.pf for pkg in ebuildlist] |
1182 |
+ |
1183 |
+ self.filescheck.check( |
1184 |
+ checkdir, checkdirlist, checkdir_relative, self.changed.changed, self.changed.new) |
1185 |
+ |
1186 |
+ self.status_check.check(self.check['ebuild_notadded'], checkdir, checkdir_relative, xpkg) |
1187 |
+ self.eadded.extend(self.status_check.eadded) |
1188 |
+ |
1189 |
+ self.fetchcheck.check( |
1190 |
+ xpkg, checkdir, checkdir_relative, self.changed.changed, self.changed.new) |
1191 |
+ |
1192 |
+ if self.check['changelog'] and "ChangeLog" not in checkdirlist: |
1193 |
+ self.qatracker.add_error("changelog.missing", xpkg + "/ChangeLog") |
1194 |
+ |
1195 |
+ self.pkgmeta.check(xpkg, checkdir, checkdirlist, self.repolevel) |
1196 |
+ self.muselist = frozenset(self.pkgmeta.musedict) |
1197 |
+ |
1198 |
+ changelog_path = os.path.join(checkdir_relative, "ChangeLog") |
1199 |
+ self.changelog_modified = changelog_path in self.changed.changelogs |
1200 |
+ |
1201 |
+ self._scan_ebuilds(ebuildlist, xpkg, catdir, pkgdir) |
1202 |
+ return self.qatracker, can_force |
1203 |
+ |
1204 |
+ |
1205 |
+ def _scan_ebuilds(self, ebuildlist, xpkg, catdir, pkgdir): |
1206 |
+ # detect unused local USE-descriptions |
1207 |
+ used_useflags = set() |
1208 |
+ |
1209 |
+ for y_ebuild in ebuildlist: |
1210 |
+ |
1211 |
+ ebuild = Ebuild( |
1212 |
+ self.repo_settings, self.repolevel, pkgdir, catdir, self.vcs_settings, |
1213 |
+ xpkg, y_ebuild) |
1214 |
+ |
1215 |
+ if self.check['changelog'] and not self.changelog_modified \ |
1216 |
+ and ebuild.ebuild_path in self.changed.new_ebuilds: |
1217 |
+ self.qatracker.add_error('changelog.ebuildadded', ebuild.relative_path) |
1218 |
+ |
1219 |
+ if ebuild.untracked(self.check['ebuild_notadded'], y_ebuild, self.eadded): |
1220 |
+ # ebuild not added to vcs |
1221 |
+ self.qatracker.add_error( |
1222 |
+ "ebuild.notadded", xpkg + "/" + y_ebuild + ".ebuild") |
1223 |
+ |
1224 |
+ if bad_split_check(xpkg, y_ebuild, pkgdir, self.qatracker): |
1225 |
+ continue |
1226 |
+ |
1227 |
+ pkg = self.pkgs[y_ebuild] |
1228 |
+ if pkg_invalid(pkg, self.qatracker, ebuild): |
1229 |
+ self.allvalid = False |
1230 |
+ continue |
1231 |
+ |
1232 |
+ myaux = pkg._metadata |
1233 |
+ eapi = myaux["EAPI"] |
1234 |
+ inherited = pkg.inherited |
1235 |
+ live_ebuild = self.live_eclasses.intersection(inherited) |
1236 |
+ |
1237 |
+ self.eapicheck.check(pkg, ebuild) |
1238 |
+ |
1239 |
+ for k, v in myaux.items(): |
1240 |
+ if not isinstance(v, basestring): |
1241 |
+ continue |
1242 |
+ m = NON_ASCII_RE.search(v) |
1243 |
+ if m is not None: |
1244 |
+ self.qatracker.add_error( |
1245 |
+ "variable.invalidchar", |
1246 |
+ "%s: %s variable contains non-ASCII " |
1247 |
+ "character at position %s" % |
1248 |
+ (ebuild.relative_path, k, m.start() + 1)) |
1249 |
+ |
1250 |
+ if not self.fetchcheck.src_uri_error: |
1251 |
+ self.thirdparty.check(myaux, ebuild.relative_path) |
1252 |
+ |
1253 |
+ if myaux.get("PROVIDE"): |
1254 |
+ self.qatracker.add_error("virtual.oldstyle", ebuild.relative_path) |
1255 |
+ |
1256 |
+ for pos, missing_var in enumerate(missingvars): |
1257 |
+ if not myaux.get(missing_var): |
1258 |
+ if catdir == "virtual" and \ |
1259 |
+ missing_var in ("HOMEPAGE", "LICENSE"): |
1260 |
+ continue |
1261 |
+ if live_ebuild and missing_var == "KEYWORDS": |
1262 |
+ continue |
1263 |
+ myqakey = missingvars[pos] + ".missing" |
1264 |
+ self.qatracker.add_error(myqakey, xpkg + "/" + y_ebuild + ".ebuild") |
1265 |
+ |
1266 |
+ if catdir == "virtual": |
1267 |
+ for var in ("HOMEPAGE", "LICENSE"): |
1268 |
+ if myaux.get(var): |
1269 |
+ myqakey = var + ".virtual" |
1270 |
+ self.qatracker.add_error(myqakey, ebuild.relative_path) |
1271 |
+ |
1272 |
+ self.descriptioncheck.check(pkg, ebuild) |
1273 |
+ |
1274 |
+ keywords = myaux["KEYWORDS"].split() |
1275 |
+ |
1276 |
+ ebuild_archs = set( |
1277 |
+ kw.lstrip("~") for kw in keywords if not kw.startswith("-")) |
1278 |
+ |
1279 |
+ self.keywordcheck.check( |
1280 |
+ pkg, xpkg, ebuild, y_ebuild, keywords, ebuild_archs, self.changed, |
1281 |
+ live_ebuild, self.repo_metadata['kwlist'], self.profiles) |
1282 |
+ |
1283 |
+ if live_ebuild and self.repo_settings.repo_config.name == "gentoo": |
1284 |
+ self.liveeclasscheck.check( |
1285 |
+ pkg, xpkg, ebuild, y_ebuild, keywords, self.repo_metadata['pmaskdict']) |
1286 |
+ |
1287 |
+ if self.options.ignore_arches: |
1288 |
+ arches = [[ |
1289 |
+ self.repo_settings.repoman_settings["ARCH"], self.repo_settings.repoman_settings["ARCH"], |
1290 |
+ self.repo_settings.repoman_settings["ACCEPT_KEYWORDS"].split()]] |
1291 |
+ else: |
1292 |
+ arches = set() |
1293 |
+ for keyword in keywords: |
1294 |
+ if keyword[0] == "-": |
1295 |
+ continue |
1296 |
+ elif keyword[0] == "~": |
1297 |
+ arch = keyword[1:] |
1298 |
+ if arch == "*": |
1299 |
+ for expanded_arch in self.profiles: |
1300 |
+ if expanded_arch == "**": |
1301 |
+ continue |
1302 |
+ arches.add( |
1303 |
+ (keyword, expanded_arch, ( |
1304 |
+ expanded_arch, "~" + expanded_arch))) |
1305 |
+ else: |
1306 |
+ arches.add((keyword, arch, (arch, keyword))) |
1307 |
+ else: |
1308 |
+ if keyword == "*": |
1309 |
+ for expanded_arch in self.profiles: |
1310 |
+ if expanded_arch == "**": |
1311 |
+ continue |
1312 |
+ arches.add( |
1313 |
+ (keyword, expanded_arch, (expanded_arch,))) |
1314 |
+ else: |
1315 |
+ arches.add((keyword, keyword, (keyword,))) |
1316 |
+ if not arches: |
1317 |
+ # Use an empty profile for checking dependencies of |
1318 |
+ # packages that have empty KEYWORDS. |
1319 |
+ arches.add(('**', '**', ('**',))) |
1320 |
+ |
1321 |
+ unknown_pkgs = set() |
1322 |
+ baddepsyntax = False |
1323 |
+ badlicsyntax = False |
1324 |
+ badprovsyntax = False |
1325 |
+ # catpkg = catdir + "/" + y_ebuild |
1326 |
+ |
1327 |
+ inherited_java_eclass = "java-pkg-2" in inherited or \ |
1328 |
+ "java-pkg-opt-2" in inherited |
1329 |
+ inherited_wxwidgets_eclass = "wxwidgets" in inherited |
1330 |
+ # operator_tokens = set(["||", "(", ")"]) |
1331 |
+ type_list, badsyntax = [], [] |
1332 |
+ for mytype in Package._dep_keys + ("LICENSE", "PROPERTIES", "PROVIDE"): |
1333 |
+ mydepstr = myaux[mytype] |
1334 |
+ |
1335 |
+ buildtime = mytype in Package._buildtime_keys |
1336 |
+ runtime = mytype in Package._runtime_keys |
1337 |
+ token_class = None |
1338 |
+ if mytype.endswith("DEPEND"): |
1339 |
+ token_class = portage.dep.Atom |
1340 |
+ |
1341 |
+ try: |
1342 |
+ atoms = portage.dep.use_reduce( |
1343 |
+ mydepstr, matchall=1, flat=True, |
1344 |
+ is_valid_flag=pkg.iuse.is_valid_flag, token_class=token_class) |
1345 |
+ except portage.exception.InvalidDependString as e: |
1346 |
+ atoms = None |
1347 |
+ badsyntax.append(str(e)) |
1348 |
+ |
1349 |
+ if atoms and mytype.endswith("DEPEND"): |
1350 |
+ if runtime and \ |
1351 |
+ "test?" in mydepstr.split(): |
1352 |
+ self.qatracker.add_error( |
1353 |
+ mytype + '.suspect', |
1354 |
+ "%s: 'test?' USE conditional in %s" % |
1355 |
+ (ebuild.relative_path, mytype)) |
1356 |
+ |
1357 |
+ for atom in atoms: |
1358 |
+ if atom == "||": |
1359 |
+ continue |
1360 |
+ |
1361 |
+ is_blocker = atom.blocker |
1362 |
+ |
1363 |
+ # Skip dependency.unknown for blockers, so that we |
1364 |
+ # don't encourage people to remove necessary blockers, |
1365 |
+ # as discussed in bug 382407. We use atom.without_use |
1366 |
+ # due to bug 525376. |
1367 |
+ if not is_blocker and \ |
1368 |
+ not self.portdb.xmatch("match-all", atom.without_use) and \ |
1369 |
+ not atom.cp.startswith("virtual/"): |
1370 |
+ unknown_pkgs.add((mytype, atom.unevaluated_atom)) |
1371 |
+ |
1372 |
+ if catdir != "virtual": |
1373 |
+ if not is_blocker and \ |
1374 |
+ atom.cp in suspect_virtual: |
1375 |
+ self.qatracker.add_error( |
1376 |
+ 'virtual.suspect', ebuild.relative_path + |
1377 |
+ ": %s: consider using '%s' instead of '%s'" % |
1378 |
+ (mytype, suspect_virtual[atom.cp], atom)) |
1379 |
+ if not is_blocker and \ |
1380 |
+ atom.cp.startswith("perl-core/"): |
1381 |
+ self.qatracker.add_error('dependency.perlcore', |
1382 |
+ ebuild.relative_path + |
1383 |
+ ": %s: please use '%s' instead of '%s'" % |
1384 |
+ (mytype, |
1385 |
+ atom.replace("perl-core/","virtual/perl-"), |
1386 |
+ atom)) |
1387 |
+ |
1388 |
+ if buildtime and \ |
1389 |
+ not is_blocker and \ |
1390 |
+ not inherited_java_eclass and \ |
1391 |
+ atom.cp == "virtual/jdk": |
1392 |
+ self.qatracker.add_error( |
1393 |
+ 'java.eclassesnotused', ebuild.relative_path) |
1394 |
+ elif buildtime and \ |
1395 |
+ not is_blocker and \ |
1396 |
+ not inherited_wxwidgets_eclass and \ |
1397 |
+ atom.cp == "x11-libs/wxGTK": |
1398 |
+ self.qatracker.add_error( |
1399 |
+ 'wxwidgets.eclassnotused', |
1400 |
+ "%s: %ss on x11-libs/wxGTK without inheriting" |
1401 |
+ " wxwidgets.eclass" % (ebuild.relative_path, mytype)) |
1402 |
+ elif runtime: |
1403 |
+ if not is_blocker and \ |
1404 |
+ atom.cp in suspect_rdepend: |
1405 |
+ self.qatracker.add_error( |
1406 |
+ mytype + '.suspect', |
1407 |
+ ebuild.relative_path + ": '%s'" % atom) |
1408 |
+ |
1409 |
+ if atom.operator == "~" and \ |
1410 |
+ portage.versions.catpkgsplit(atom.cpv)[3] != "r0": |
1411 |
+ qacat = 'dependency.badtilde' |
1412 |
+ self.qatracker.add_error( |
1413 |
+ qacat, "%s: %s uses the ~ operator" |
1414 |
+ " with a non-zero revision: '%s'" % |
1415 |
+ (ebuild.relative_path, mytype, atom)) |
1416 |
+ |
1417 |
+ check_missingslot(atom, mytype, eapi, self.portdb, self.qatracker, |
1418 |
+ ebuild.relative_path, myaux) |
1419 |
+ |
1420 |
+ type_list.extend([mytype] * (len(badsyntax) - len(type_list))) |
1421 |
+ |
1422 |
+ for m, b in zip(type_list, badsyntax): |
1423 |
+ if m.endswith("DEPEND"): |
1424 |
+ qacat = "dependency.syntax" |
1425 |
+ else: |
1426 |
+ qacat = m + ".syntax" |
1427 |
+ self.qatracker.add_error( |
1428 |
+ qacat, "%s: %s: %s" % (ebuild.relative_path, m, b)) |
1429 |
+ |
1430 |
+ badlicsyntax = len([z for z in type_list if z == "LICENSE"]) |
1431 |
+ badprovsyntax = len([z for z in type_list if z == "PROVIDE"]) |
1432 |
+ baddepsyntax = len(type_list) != badlicsyntax + badprovsyntax |
1433 |
+ badlicsyntax = badlicsyntax > 0 |
1434 |
+ badprovsyntax = badprovsyntax > 0 |
1435 |
+ |
1436 |
+ self.use_flag_checks.check(pkg, xpkg, ebuild, y_ebuild, self.muselist) |
1437 |
+ |
1438 |
+ ebuild_used_useflags = self.use_flag_checks.getUsedUseFlags() |
1439 |
+ used_useflags = used_useflags.union(ebuild_used_useflags) |
1440 |
+ |
1441 |
+ self.rubyeclasscheck.check(pkg, ebuild) |
1442 |
+ |
1443 |
+ # license checks |
1444 |
+ if not badlicsyntax: |
1445 |
+ self.licensecheck.check(pkg, xpkg, ebuild, y_ebuild) |
1446 |
+ |
1447 |
+ self.restrictcheck.check(pkg, xpkg, ebuild, y_ebuild) |
1448 |
+ |
1449 |
+ # Syntax Checks |
1450 |
+ if not self.vcs_settings.vcs_preserves_mtime: |
1451 |
+ if ebuild.ebuild_path not in self.changed.new_ebuilds and \ |
1452 |
+ ebuild.ebuild_path not in self.changed.ebuilds: |
1453 |
+ pkg.mtime = None |
1454 |
+ try: |
1455 |
+ # All ebuilds should have utf_8 encoding. |
1456 |
+ f = io.open( |
1457 |
+ _unicode_encode( |
1458 |
+ ebuild.full_path, encoding=_encodings['fs'], errors='strict'), |
1459 |
+ mode='r', encoding=_encodings['repo.content']) |
1460 |
+ try: |
1461 |
+ for check_name, e in run_checks(f, pkg): |
1462 |
+ self.qatracker.add_error( |
1463 |
+ check_name, ebuild.relative_path + ': %s' % e) |
1464 |
+ finally: |
1465 |
+ f.close() |
1466 |
+ except UnicodeDecodeError: |
1467 |
+ # A file.UTF8 failure will have already been recorded above. |
1468 |
+ pass |
1469 |
+ |
1470 |
+ if self.options.force: |
1471 |
+ # The dep_check() calls are the most expensive QA test. If --force |
1472 |
+ # is enabled, there's no point in wasting time on these since the |
1473 |
+ # user is intent on forcing the commit anyway. |
1474 |
+ continue |
1475 |
+ |
1476 |
+ relevant_profiles = [] |
1477 |
+ for keyword, arch, groups in arches: |
1478 |
+ if arch not in self.profiles: |
1479 |
+ # A missing profile will create an error further down |
1480 |
+ # during the KEYWORDS verification. |
1481 |
+ continue |
1482 |
+ |
1483 |
+ if self.include_arches is not None: |
1484 |
+ if arch not in self.include_arches: |
1485 |
+ continue |
1486 |
+ |
1487 |
+ relevant_profiles.extend( |
1488 |
+ (keyword, groups, prof) for prof in self.profiles[arch]) |
1489 |
+ |
1490 |
+ relevant_profiles.sort(key=sort_key) |
1491 |
+ |
1492 |
+ for keyword, groups, prof in relevant_profiles: |
1493 |
+ |
1494 |
+ is_stable_profile = prof.status == "stable" |
1495 |
+ is_dev_profile = prof.status == "dev" and \ |
1496 |
+ self.options.include_dev |
1497 |
+ is_exp_profile = prof.status == "exp" and \ |
1498 |
+ self.options.include_exp_profiles == 'y' |
1499 |
+ if not (is_stable_profile or is_dev_profile or is_exp_profile): |
1500 |
+ continue |
1501 |
+ |
1502 |
+ dep_settings = self.caches['arch'].get(prof.sub_path) |
1503 |
+ if dep_settings is None: |
1504 |
+ dep_settings = portage.config( |
1505 |
+ config_profile_path=prof.abs_path, |
1506 |
+ config_incrementals=self.repoman_incrementals, |
1507 |
+ config_root=self.config_root, |
1508 |
+ local_config=False, |
1509 |
+ _unmatched_removal=self.options.unmatched_removal, |
1510 |
+ env=self.env, repositories=self.repo_settings.repoman_settings.repositories) |
1511 |
+ dep_settings.categories = self.repo_settings.repoman_settings.categories |
1512 |
+ if self.options.without_mask: |
1513 |
+ dep_settings._mask_manager_obj = \ |
1514 |
+ copy.deepcopy(dep_settings._mask_manager) |
1515 |
+ dep_settings._mask_manager._pmaskdict.clear() |
1516 |
+ self.caches['arch'][prof.sub_path] = dep_settings |
1517 |
+ |
1518 |
+ xmatch_cache_key = (prof.sub_path, tuple(groups)) |
1519 |
+ xcache = self.caches['arch_xmatch'].get(xmatch_cache_key) |
1520 |
+ if xcache is None: |
1521 |
+ self.portdb.melt() |
1522 |
+ self.portdb.freeze() |
1523 |
+ xcache = self.portdb.xcache |
1524 |
+ xcache.update(self.caches['shared_xmatch']) |
1525 |
+ self.caches['arch_xmatch'][xmatch_cache_key] = xcache |
1526 |
+ |
1527 |
+ self.repo_settings.trees[self.repo_settings.root]["porttree"].settings = dep_settings |
1528 |
+ self.portdb.settings = dep_settings |
1529 |
+ self.portdb.xcache = xcache |
1530 |
+ |
1531 |
+ dep_settings["ACCEPT_KEYWORDS"] = " ".join(groups) |
1532 |
+ # just in case, prevent config.reset() from nuking these. |
1533 |
+ dep_settings.backup_changes("ACCEPT_KEYWORDS") |
1534 |
+ |
1535 |
+ # This attribute is used in dbapi._match_use() to apply |
1536 |
+ # use.stable.{mask,force} settings based on the stable |
1537 |
+ # status of the parent package. This is required in order |
1538 |
+ # for USE deps of unstable packages to be resolved correctly, |
1539 |
+ # since otherwise use.stable.{mask,force} settings of |
1540 |
+ # dependencies may conflict (see bug #456342). |
1541 |
+ dep_settings._parent_stable = dep_settings._isStable(pkg) |
1542 |
+ |
1543 |
+ # Handle package.use*.{force,mask) calculation, for use |
1544 |
+ # in dep_check. |
1545 |
+ dep_settings.useforce = dep_settings._use_manager.getUseForce( |
1546 |
+ pkg, stable=dep_settings._parent_stable) |
1547 |
+ dep_settings.usemask = dep_settings._use_manager.getUseMask( |
1548 |
+ pkg, stable=dep_settings._parent_stable) |
1549 |
+ |
1550 |
+ if not baddepsyntax: |
1551 |
+ ismasked = not ebuild_archs or \ |
1552 |
+ pkg.cpv not in self.portdb.xmatch("match-visible", |
1553 |
+ Atom("%s::%s" % (pkg.cp, self.repo_settings.repo_config.name))) |
1554 |
+ if ismasked: |
1555 |
+ if not self.have['pmasked']: |
1556 |
+ self.have['pmasked'] = bool(dep_settings._getMaskAtom( |
1557 |
+ pkg.cpv, pkg._metadata)) |
1558 |
+ if self.options.ignore_masked: |
1559 |
+ continue |
1560 |
+ # we are testing deps for a masked package; give it some lee-way |
1561 |
+ suffix = "masked" |
1562 |
+ matchmode = "minimum-all" |
1563 |
+ else: |
1564 |
+ suffix = "" |
1565 |
+ matchmode = "minimum-visible" |
1566 |
+ |
1567 |
+ if not self.have['dev_keywords']: |
1568 |
+ self.have['dev_keywords'] = \ |
1569 |
+ bool(self.dev_keywords.intersection(keywords)) |
1570 |
+ |
1571 |
+ if prof.status == "dev": |
1572 |
+ suffix = suffix + "indev" |
1573 |
+ |
1574 |
+ for mytype in Package._dep_keys: |
1575 |
+ |
1576 |
+ mykey = "dependency.bad" + suffix |
1577 |
+ myvalue = myaux[mytype] |
1578 |
+ if not myvalue: |
1579 |
+ continue |
1580 |
+ |
1581 |
+ success, atoms = portage.dep_check( |
1582 |
+ myvalue, self.portdb, dep_settings, |
1583 |
+ use="all", mode=matchmode, trees=self.repo_settings.trees) |
1584 |
+ |
1585 |
+ if success: |
1586 |
+ if atoms: |
1587 |
+ |
1588 |
+ # Don't bother with dependency.unknown for |
1589 |
+ # cases in which *DEPEND.bad is triggered. |
1590 |
+ for atom in atoms: |
1591 |
+ # dep_check returns all blockers and they |
1592 |
+ # aren't counted for *DEPEND.bad, so we |
1593 |
+ # ignore them here. |
1594 |
+ if not atom.blocker: |
1595 |
+ unknown_pkgs.discard( |
1596 |
+ (mytype, atom.unevaluated_atom)) |
1597 |
+ |
1598 |
+ if not prof.sub_path: |
1599 |
+ # old-style virtuals currently aren't |
1600 |
+ # resolvable with empty profile, since |
1601 |
+ # 'virtuals' mappings are unavailable |
1602 |
+ # (it would be expensive to search |
1603 |
+ # for PROVIDE in all ebuilds) |
1604 |
+ atoms = [ |
1605 |
+ atom for atom in atoms if not ( |
1606 |
+ atom.cp.startswith('virtual/') |
1607 |
+ and not self.portdb.cp_list(atom.cp))] |
1608 |
+ |
1609 |
+ # we have some unsolvable deps |
1610 |
+ # remove ! deps, which always show up as unsatisfiable |
1611 |
+ atoms = [ |
1612 |
+ str(atom.unevaluated_atom) |
1613 |
+ for atom in atoms if not atom.blocker] |
1614 |
+ |
1615 |
+ # if we emptied out our list, continue: |
1616 |
+ if not atoms: |
1617 |
+ continue |
1618 |
+ self.qatracker.add_error(mykey, |
1619 |
+ "%s: %s: %s(%s)\n%s" |
1620 |
+ % (ebuild.relative_path, mytype, keyword, prof, |
1621 |
+ pformat(atoms, indent=6))) |
1622 |
+ else: |
1623 |
+ self.qatracker.add_error(mykey, |
1624 |
+ "%s: %s: %s(%s)\n%s" |
1625 |
+ % (ebuild.relative_path, mytype, keyword, prof, |
1626 |
+ pformat(atoms, indent=6))) |
1627 |
+ |
1628 |
+ if not baddepsyntax and unknown_pkgs: |
1629 |
+ type_map = {} |
1630 |
+ for mytype, atom in unknown_pkgs: |
1631 |
+ type_map.setdefault(mytype, set()).add(atom) |
1632 |
+ for mytype, atoms in type_map.items(): |
1633 |
+ self.qatracker.add_error( |
1634 |
+ "dependency.unknown", "%s: %s: %s" |
1635 |
+ % (ebuild.relative_path, mytype, ", ".join(sorted(atoms)))) |
1636 |
+ |
1637 |
+ # check if there are unused local USE-descriptions in metadata.xml |
1638 |
+ # (unless there are any invalids, to avoid noise) |
1639 |
+ if self.allvalid: |
1640 |
+ for myflag in self.muselist.difference(used_useflags): |
1641 |
+ self.qatracker.add_error( |
1642 |
+ "metadata.warning", |
1643 |
+ "%s/metadata.xml: unused local USE-description: '%s'" |
1644 |
+ % (xpkg, myflag)) |