1 |
commit: 26524e5fbc8019e22d15765a35570df7c7a315ca |
2 |
Author: Zac Medico <zmedico <AT> gentoo <DOT> org> |
3 |
AuthorDate: Sat Aug 15 02:51:38 2020 +0000 |
4 |
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org> |
5 |
CommitDate: Mon Aug 17 05:39:06 2020 +0000 |
6 |
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=26524e5f |
7 |
|
8 |
repoman: Add --jobs and --load-average options (bug 448462) |
9 |
|
10 |
Add --jobs and --load-average options which allow dependency checks |
11 |
for multiple profiles to run in parallel. The increase in performance |
12 |
is non-linear for the number of jobs, but it can be worthwhile |
13 |
(I measured a 35% decrease in time when running 'repoman -j8 full' |
14 |
on sys-apps/portage). For the -j1 case (default), all dependency |
15 |
checks run in the main process as usual, so there is no significant |
16 |
performance penalty for the default case. |
17 |
|
18 |
Bug: https://bugs.gentoo.org/448462 |
19 |
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org> |
20 |
|
21 |
repoman/lib/repoman/argparser.py | 9 ++ |
22 |
repoman/lib/repoman/modules/scan/depend/profile.py | 117 +++++++++++++++++---- |
23 |
repoman/man/repoman.1 | 9 +- |
24 |
3 files changed, 116 insertions(+), 19 deletions(-) |
25 |
|
26 |
diff --git a/repoman/lib/repoman/argparser.py b/repoman/lib/repoman/argparser.py |
27 |
index 670a0e91d..6d545ccca 100644 |
28 |
--- a/repoman/lib/repoman/argparser.py |
29 |
+++ b/repoman/lib/repoman/argparser.py |
30 |
@@ -199,6 +199,15 @@ def parse_args(argv, repoman_default_opts): |
31 |
'--output-style', dest='output_style', choices=output_keys, |
32 |
help='select output type', default='default') |
33 |
|
34 |
+ parser.add_argument( |
35 |
+ '-j', '--jobs', dest='jobs', action='store', type=int, default=1, |
36 |
+ help='Specifies the number of jobs (processes) to run simultaneously.') |
37 |
+ |
38 |
+ parser.add_argument( |
39 |
+ '-l', '--load-average', dest='load_average', action='store', type=float, default=None, |
40 |
+ help='Specifies that no new jobs (processes) should be started if there are others ' |
41 |
+ 'jobs running and the load average is at least load (a floating-point number).') |
42 |
+ |
43 |
parser.add_argument( |
44 |
'--mode', dest='mode', choices=mode_keys, |
45 |
help='specify which mode repoman will run in (default=full)') |
46 |
|
47 |
diff --git a/repoman/lib/repoman/modules/scan/depend/profile.py b/repoman/lib/repoman/modules/scan/depend/profile.py |
48 |
index 39d8b550c..1eb69422a 100644 |
49 |
--- a/repoman/lib/repoman/modules/scan/depend/profile.py |
50 |
+++ b/repoman/lib/repoman/modules/scan/depend/profile.py |
51 |
@@ -2,7 +2,9 @@ |
52 |
|
53 |
|
54 |
import copy |
55 |
+import functools |
56 |
import os |
57 |
+import types |
58 |
from pprint import pformat |
59 |
|
60 |
from _emerge.Package import Package |
61 |
@@ -15,6 +17,10 @@ from repoman.modules.scan.depend._gen_arches import _gen_arches |
62 |
from portage.dep import Atom |
63 |
from portage.package.ebuild.profile_iuse import iter_iuse_vars |
64 |
from portage.util import getconfig |
65 |
+from portage.util.futures import asyncio |
66 |
+from portage.util.futures.compat_coroutine import coroutine, coroutine_return |
67 |
+from portage.util.futures.executor.fork import ForkExecutor |
68 |
+from portage.util.futures.iter_completed import async_iter_completed |
69 |
|
70 |
|
71 |
def sort_key(item): |
72 |
@@ -58,16 +64,14 @@ class ProfileDependsChecks(ScanBase): |
73 |
def check(self, **kwargs): |
74 |
'''Perform profile dependant dependency checks |
75 |
|
76 |
- @param arches: |
77 |
@param pkg: Package in which we check (object). |
78 |
@param ebuild: Ebuild which we check (object). |
79 |
- @param baddepsyntax: boolean |
80 |
- @param unknown_pkgs: set of tuples (type, atom.unevaluated_atom) |
81 |
@returns: dictionary |
82 |
''' |
83 |
ebuild = kwargs.get('ebuild').get() |
84 |
pkg = kwargs.get('pkg').get() |
85 |
- unknown_pkgs, baddepsyntax = _depend_checks( |
86 |
+ |
87 |
+ ebuild.unknown_pkgs, ebuild.baddepsyntax = _depend_checks( |
88 |
ebuild, pkg, self.portdb, self.qatracker, self.repo_metadata, |
89 |
self.repo_settings.qadata) |
90 |
|
91 |
@@ -90,8 +94,64 @@ class ProfileDependsChecks(ScanBase): |
92 |
relevant_profiles.append((keyword, groups, prof)) |
93 |
|
94 |
relevant_profiles.sort(key=sort_key) |
95 |
+ ebuild.relevant_profiles = relevant_profiles |
96 |
+ |
97 |
+ if self.options.jobs <= 1: |
98 |
+ for task in self._iter_tasks(None, None, ebuild, pkg): |
99 |
+ task, results = task |
100 |
+ for result in results: |
101 |
+ self._check_result(task, result) |
102 |
+ |
103 |
+ loop = asyncio._wrap_loop() |
104 |
+ loop.run_until_complete(self._async_check(loop=loop, **kwargs)) |
105 |
+ |
106 |
+ return False |
107 |
+ |
108 |
+ @coroutine |
109 |
+ def _async_check(self, loop=None, **kwargs): |
110 |
+ '''Perform async profile dependant dependency checks |
111 |
+ |
112 |
+ @param arches: |
113 |
+ @param pkg: Package in which we check (object). |
114 |
+ @param ebuild: Ebuild which we check (object). |
115 |
+ @param baddepsyntax: boolean |
116 |
+ @param unknown_pkgs: set of tuples (type, atom.unevaluated_atom) |
117 |
+ @returns: dictionary |
118 |
+ ''' |
119 |
+ loop = asyncio._wrap_loop(loop) |
120 |
+ ebuild = kwargs.get('ebuild').get() |
121 |
+ pkg = kwargs.get('pkg').get() |
122 |
+ unknown_pkgs = ebuild.unknown_pkgs |
123 |
+ baddepsyntax = ebuild.baddepsyntax |
124 |
+ |
125 |
+ # Use max_workers=True to ensure immediate fork, since _iter_tasks |
126 |
+ # needs the fork to create a snapshot of current state. |
127 |
+ executor = ForkExecutor(max_workers=self.options.jobs) |
128 |
+ |
129 |
+ if self.options.jobs > 1: |
130 |
+ for future_done_set in async_iter_completed(self._iter_tasks(loop, executor, ebuild, pkg), |
131 |
+ max_jobs=self.options.jobs, max_load=self.options.load_average, loop=loop): |
132 |
+ for task in (yield future_done_set): |
133 |
+ task, results = task.result() |
134 |
+ for result in results: |
135 |
+ self._check_result(task, result) |
136 |
+ |
137 |
+ if not baddepsyntax and unknown_pkgs: |
138 |
+ type_map = {} |
139 |
+ for mytype, atom in unknown_pkgs: |
140 |
+ type_map.setdefault(mytype, set()).add(atom) |
141 |
+ for mytype, atoms in type_map.items(): |
142 |
+ self.qatracker.add_error( |
143 |
+ "dependency.unknown", "%s: %s: %s" |
144 |
+ % (ebuild.relative_path, mytype, ", ".join(sorted(atoms)))) |
145 |
|
146 |
- for keyword, groups, prof in relevant_profiles: |
147 |
+ @coroutine |
148 |
+ def _task(self, task): |
149 |
+ yield task.future |
150 |
+ coroutine_return((task, task.future.result())) |
151 |
+ |
152 |
+ def _iter_tasks(self, loop, executor, ebuild, pkg): |
153 |
+ for keyword, groups, prof in ebuild.relevant_profiles: |
154 |
|
155 |
is_stable_profile = prof.status == "stable" |
156 |
is_dev_profile = prof.status == "dev" and \ |
157 |
@@ -154,6 +214,22 @@ class ProfileDependsChecks(ScanBase): |
158 |
dep_settings.usemask = dep_settings._use_manager.getUseMask( |
159 |
pkg, stable=dep_settings._parent_stable) |
160 |
|
161 |
+ task = types.SimpleNamespace(ebuild=ebuild, prof=prof, keyword=keyword) |
162 |
+ |
163 |
+ target = functools.partial(self._task_subprocess, task, pkg, dep_settings) |
164 |
+ |
165 |
+ if self.options.jobs <= 1: |
166 |
+ yield (task, target()) |
167 |
+ else: |
168 |
+ task.future = asyncio.ensure_future(loop.run_in_executor(executor, target), loop=loop) |
169 |
+ yield self._task(task) |
170 |
+ |
171 |
+ |
172 |
+ def _task_subprocess(self, task, pkg, dep_settings): |
173 |
+ ebuild = task.ebuild |
174 |
+ baddepsyntax = ebuild.baddepsyntax |
175 |
+ results = [] |
176 |
+ prof = task.prof |
177 |
if not baddepsyntax: |
178 |
ismasked = not ebuild.archs or \ |
179 |
pkg.cpv not in self.portdb.xmatch("match-visible", |
180 |
@@ -163,7 +239,7 @@ class ProfileDependsChecks(ScanBase): |
181 |
self.have['pmasked'] = bool(dep_settings._getMaskAtom( |
182 |
pkg.cpv, ebuild.metadata)) |
183 |
if self.options.ignore_masked: |
184 |
- continue |
185 |
+ return results |
186 |
# we are testing deps for a masked package; give it some lee-way |
187 |
suffix = "masked" |
188 |
matchmode = "minimum-all-ignore-profile" |
189 |
@@ -191,6 +267,22 @@ class ProfileDependsChecks(ScanBase): |
190 |
myvalue, self.portdb, dep_settings, |
191 |
use="all", mode=matchmode, trees=self.repo_settings.trees) |
192 |
|
193 |
+ results.append(types.SimpleNamespace(atoms=atoms, success=success, mykey=mykey, mytype=mytype)) |
194 |
+ |
195 |
+ return results |
196 |
+ |
197 |
+ |
198 |
+ def _check_result(self, task, result): |
199 |
+ prof = task.prof |
200 |
+ keyword = task.keyword |
201 |
+ ebuild = task.ebuild |
202 |
+ unknown_pkgs = ebuild.unknown_pkgs |
203 |
+ |
204 |
+ success = result.success |
205 |
+ atoms = result.atoms |
206 |
+ mykey = result.mykey |
207 |
+ mytype = result.mytype |
208 |
+ |
209 |
if success: |
210 |
if atoms: |
211 |
|
212 |
@@ -223,7 +315,7 @@ class ProfileDependsChecks(ScanBase): |
213 |
|
214 |
# if we emptied out our list, continue: |
215 |
if not all_atoms: |
216 |
- continue |
217 |
+ return |
218 |
|
219 |
# Filter out duplicates. We do this by hand (rather |
220 |
# than use a set) so the order is stable and better |
221 |
@@ -255,17 +347,6 @@ class ProfileDependsChecks(ScanBase): |
222 |
% (ebuild.relative_path, mytype, keyword, |
223 |
prof, pformat(atoms, indent=6))) |
224 |
|
225 |
- if not baddepsyntax and unknown_pkgs: |
226 |
- type_map = {} |
227 |
- for mytype, atom in unknown_pkgs: |
228 |
- type_map.setdefault(mytype, set()).add(atom) |
229 |
- for mytype, atoms in type_map.items(): |
230 |
- self.qatracker.add_error( |
231 |
- "dependency.unknown", "%s: %s: %s" |
232 |
- % (ebuild.relative_path, mytype, ", ".join(sorted(atoms)))) |
233 |
- |
234 |
- return False |
235 |
- |
236 |
@property |
237 |
def runInEbuilds(self): |
238 |
'''Ebuild level scans''' |
239 |
|
240 |
diff --git a/repoman/man/repoman.1 b/repoman/man/repoman.1 |
241 |
index a6a9937e5..6f9a24544 100644 |
242 |
--- a/repoman/man/repoman.1 |
243 |
+++ b/repoman/man/repoman.1 |
244 |
@@ -1,4 +1,4 @@ |
245 |
-.TH "REPOMAN" "1" "Mar 2018" "Repoman VERSION" "Repoman" |
246 |
+.TH "REPOMAN" "1" "Aug 2020" "Repoman VERSION" "Repoman" |
247 |
.SH NAME |
248 |
repoman \- Gentoo's program to enforce a minimal level of quality assurance in |
249 |
packages added to the ebuild repository |
250 |
@@ -83,6 +83,13 @@ Be less verbose about extraneous info |
251 |
\fB-p\fR, \fB--pretend\fR |
252 |
Don't commit or fix anything; just show what would be done |
253 |
.TP |
254 |
+\fB\-j\fR, \fB\-\-jobs\fR |
255 |
+Specifies the number of jobs (processes) to run simultaneously. |
256 |
+.TP |
257 |
+\fB\-l\fR, \fB\-\-load-average\fR |
258 |
+Specifies that no new jobs (processes) should be started if there are others |
259 |
+jobs running and the load average is at least load (a floating\-point number). |
260 |
+.TP |
261 |
\fB-x\fR, \fB--xmlparse\fR |
262 |
Forces the metadata.xml parse check to be carried out |
263 |
.TP |