1 |
repoman slows down ~linearly with amount of profiles being scanned. |
2 |
In case of amd64 we have 28 stable profiles. |
3 |
|
4 |
To speed up processing and fit into time budged of various CIs we can |
5 |
split the work across different processes that handle different profiles. |
6 |
|
7 |
Example benchmark on ::haskell overlay: |
8 |
$ ./repoman full --include-arches=amd64 |
9 |
~65 minutes |
10 |
$ ./repoman full --include-profiles=default/linux/amd64/17.0 |
11 |
~4 minutes |
12 |
This allows for a crude sharding of work across processes and allows for |
13 |
cheap tree-wide scans for early failures. |
14 |
|
15 |
Bug: https://bugs.gentoo.org/700456 |
16 |
Signed-off-by: Sergei Trofimovich <slyfox@g.o> |
17 |
--- |
18 |
repoman/lib/repoman/actions.py | 4 ++++ |
19 |
repoman/lib/repoman/argparser.py | 7 +++++++ |
20 |
repoman/lib/repoman/modules/scan/depend/__init__.py | 3 ++- |
21 |
repoman/lib/repoman/modules/scan/depend/profile.py | 9 +++++++-- |
22 |
repoman/lib/repoman/scanner.py | 5 +++++ |
23 |
repoman/man/repoman.1 | 4 ++++ |
24 |
6 files changed, 29 insertions(+), 3 deletions(-) |
25 |
|
26 |
diff --git a/repoman/lib/repoman/actions.py b/repoman/lib/repoman/actions.py |
27 |
index 1c9989a72..92d4d4e94 100644 |
28 |
--- a/repoman/lib/repoman/actions.py |
29 |
+++ b/repoman/lib/repoman/actions.py |
30 |
@@ -412,6 +412,10 @@ the whole commit message to abort. |
31 |
report_options.append( |
32 |
"--include-arches=\"%s\"" % |
33 |
" ".join(sorted(self.scanner.include_arches))) |
34 |
+ if self.scanner.include_profiles is not None: |
35 |
+ report_options.append( |
36 |
+ "--include-profiles=\"%s\"" % |
37 |
+ " ".join(sorted(self.scanner.include_profiles))) |
38 |
|
39 |
if portage_version is None: |
40 |
sys.stderr.write("Failed to insert portage version in message!\n") |
41 |
diff --git a/repoman/lib/repoman/argparser.py b/repoman/lib/repoman/argparser.py |
42 |
index fa0e6ff90..670a0e91d 100644 |
43 |
--- a/repoman/lib/repoman/argparser.py |
44 |
+++ b/repoman/lib/repoman/argparser.py |
45 |
@@ -164,6 +164,13 @@ def parse_args(argv, repoman_default_opts): |
46 |
'A space separated list of arches used to ' |
47 |
'filter the selection of profiles for dependency checks')) |
48 |
|
49 |
+ parser.add_argument( |
50 |
+ '--include-profiles', |
51 |
+ dest='include_profiles', metavar='PROFILES', action='append', |
52 |
+ help=( |
53 |
+ 'A space separated list of profiles used to ' |
54 |
+ 'define the selection of profiles for dependency checks')) |
55 |
+ |
56 |
parser.add_argument( |
57 |
'-d', '--include-dev', dest='include_dev', action='store_true', |
58 |
default=False, |
59 |
diff --git a/repoman/lib/repoman/modules/scan/depend/__init__.py b/repoman/lib/repoman/modules/scan/depend/__init__.py |
60 |
index c3cc0ddeb..9068760bb 100644 |
61 |
--- a/repoman/lib/repoman/modules/scan/depend/__init__.py |
62 |
+++ b/repoman/lib/repoman/modules/scan/depend/__init__.py |
63 |
@@ -19,7 +19,8 @@ module_spec = { |
64 |
'func_desc': { |
65 |
}, |
66 |
'mod_kwargs': ['qatracker', 'portdb', 'profiles', 'options', |
67 |
- 'repo_metadata', 'repo_settings', 'include_arches', 'caches', |
68 |
+ 'repo_metadata', 'repo_settings', 'include_arches', |
69 |
+ 'include_profiles', 'caches', |
70 |
'repoman_incrementals', 'env', 'have', 'dev_keywords' |
71 |
], |
72 |
'func_kwargs': { |
73 |
diff --git a/repoman/lib/repoman/modules/scan/depend/profile.py b/repoman/lib/repoman/modules/scan/depend/profile.py |
74 |
index d980f4eca..0b1d74483 100644 |
75 |
--- a/repoman/lib/repoman/modules/scan/depend/profile.py |
76 |
+++ b/repoman/lib/repoman/modules/scan/depend/profile.py |
77 |
@@ -33,6 +33,7 @@ class ProfileDependsChecks(ScanBase): |
78 |
@param options: cli options |
79 |
@param repo_settings: repository settings instance |
80 |
@param include_arches: set |
81 |
+ @param include_profiles: set |
82 |
@param caches: dictionary of our caches |
83 |
@param repoman_incrementals: tuple |
84 |
@param env: the environment |
85 |
@@ -46,6 +47,7 @@ class ProfileDependsChecks(ScanBase): |
86 |
self.options = kwargs.get('options') |
87 |
self.repo_settings = kwargs.get('repo_settings') |
88 |
self.include_arches = kwargs.get('include_arches') |
89 |
+ self.include_profiles = kwargs.get('include_profiles') |
90 |
self.caches = kwargs.get('caches') |
91 |
self.repoman_incrementals = kwargs.get('repoman_incrementals') |
92 |
self.env = kwargs.get('env') |
93 |
@@ -81,8 +83,11 @@ class ProfileDependsChecks(ScanBase): |
94 |
if arch not in self.include_arches: |
95 |
continue |
96 |
|
97 |
- relevant_profiles.extend( |
98 |
- (keyword, groups, prof) for prof in self.profiles[arch]) |
99 |
+ for prof in self.profiles[arch]: |
100 |
+ if self.include_profiles is not None: |
101 |
+ if prof not in self.include_profiles: |
102 |
+ continue |
103 |
+ relevant_profiles.append((keyword, groups, prof)) |
104 |
|
105 |
relevant_profiles.sort(key=sort_key) |
106 |
|
107 |
diff --git a/repoman/lib/repoman/scanner.py b/repoman/lib/repoman/scanner.py |
108 |
index 1b3242a51..06234b0ad 100644 |
109 |
--- a/repoman/lib/repoman/scanner.py |
110 |
+++ b/repoman/lib/repoman/scanner.py |
111 |
@@ -164,6 +164,10 @@ class Scanner(object): |
112 |
if self.options.include_arches: |
113 |
self.include_arches = set() |
114 |
self.include_arches.update(*[x.split() for x in self.options.include_arches]) |
115 |
+ self.include_profiles = None |
116 |
+ if self.options.include_profiles: |
117 |
+ self.include_profiles = set() |
118 |
+ self.include_profiles.update(*[x.split() for x in self.options.include_profiles]) |
119 |
|
120 |
# Disable the "self.modules['Ebuild'].notadded" check when not in commit mode and |
121 |
# running `svn status` in every package dir will be too expensive. |
122 |
@@ -190,6 +194,7 @@ class Scanner(object): |
123 |
"repo_metadata": self.repo_metadata, |
124 |
"profiles": profiles, |
125 |
"include_arches": self.include_arches, |
126 |
+ "include_profiles": self.include_profiles, |
127 |
"caches": self.caches, |
128 |
"repoman_incrementals": self.repoman_incrementals, |
129 |
"env": self.env, |
130 |
diff --git a/repoman/man/repoman.1 b/repoman/man/repoman.1 |
131 |
index 766146f57..f33fb6098 100644 |
132 |
--- a/repoman/man/repoman.1 |
133 |
+++ b/repoman/man/repoman.1 |
134 |
@@ -120,6 +120,10 @@ Ignore masked packages (not allowed with commit mode) |
135 |
A space separated list of arches used to filter the selection of |
136 |
profiles for dependency checks. |
137 |
.TP |
138 |
+.BR "\-\-include\-profiles " PROFILES |
139 |
+A space separated list of profiles used to |
140 |
+define the selection of profiles for dependency checks. |
141 |
+.TP |
142 |
\fB\-d\fR, \fB\-\-include\-dev\fR |
143 |
Include dev profiles in dependency checks. |
144 |
.TP |
145 |
-- |
146 |
2.24.0 |