| 1 |
On 11/18/19 11:41 PM, Sergei Trofimovich wrote: |
| 2 |
> On Mon, 18 Nov 2019 16:45:58 -0800 |
| 3 |
> Zac Medico <zmedico@g.o> wrote: |
| 4 |
> |
| 5 |
>> On 11/18/19 4:21 PM, Sergei Trofimovich wrote: |
| 6 |
>>> repoman slows down ~linearly with amount of profiles being scanned. |
| 7 |
>>> In case of amd64 we have 28 stable profiles. |
| 8 |
>>> |
| 9 |
>>> To speed up processing and fit into time budged of various CIs we can |
| 10 |
>>> split the work across different processes that handle different profiles. |
| 11 |
>>> |
| 12 |
>>> Example benchmark on ::haskell overlay: |
| 13 |
>>> $ ./repoman full --include-arches=amd64 |
| 14 |
>>> ~65 minutes |
| 15 |
>>> $ ./repoman full --include-profiles=default/linux/amd64/17.0 |
| 16 |
>>> ~4 minutes |
| 17 |
>>> This allows for a crude sharding of work across processes and allows for |
| 18 |
>>> cheap tree-wide scans for early failures. |
| 19 |
>>> |
| 20 |
>>> Bug: https://bugs.gentoo.org/700456 |
| 21 |
>>> Signed-off-by: Sergei Trofimovich <slyfox@g.o> |
| 22 |
>>> --- |
| 23 |
>>> repoman/lib/repoman/actions.py | 4 ++++ |
| 24 |
>>> repoman/lib/repoman/argparser.py | 7 +++++++ |
| 25 |
>>> repoman/lib/repoman/modules/scan/depend/__init__.py | 3 ++- |
| 26 |
>>> repoman/lib/repoman/modules/scan/depend/profile.py | 9 +++++++-- |
| 27 |
>>> repoman/lib/repoman/scanner.py | 5 +++++ |
| 28 |
>>> repoman/man/repoman.1 | 4 ++++ |
| 29 |
>>> 6 files changed, 29 insertions(+), 3 deletions(-) |
| 30 |
>>> |
| 31 |
>>> diff --git a/repoman/lib/repoman/actions.py b/repoman/lib/repoman/actions.py |
| 32 |
>>> index 1c9989a72..92d4d4e94 100644 |
| 33 |
>>> --- a/repoman/lib/repoman/actions.py |
| 34 |
>>> +++ b/repoman/lib/repoman/actions.py |
| 35 |
>>> @@ -412,6 +412,10 @@ the whole commit message to abort. |
| 36 |
>>> report_options.append( |
| 37 |
>>> "--include-arches=\"%s\"" % |
| 38 |
>>> " ".join(sorted(self.scanner.include_arches))) |
| 39 |
>>> + if self.scanner.include_profiles is not None: |
| 40 |
>>> + report_options.append( |
| 41 |
>>> + "--include-profiles=\"%s\"" % |
| 42 |
>>> + " ".join(sorted(self.scanner.include_profiles))) |
| 43 |
>>> |
| 44 |
>>> if portage_version is None: |
| 45 |
>>> sys.stderr.write("Failed to insert portage version in message!\n") |
| 46 |
>>> diff --git a/repoman/lib/repoman/argparser.py b/repoman/lib/repoman/argparser.py |
| 47 |
>>> index fa0e6ff90..670a0e91d 100644 |
| 48 |
>>> --- a/repoman/lib/repoman/argparser.py |
| 49 |
>>> +++ b/repoman/lib/repoman/argparser.py |
| 50 |
>>> @@ -164,6 +164,13 @@ def parse_args(argv, repoman_default_opts): |
| 51 |
>>> 'A space separated list of arches used to ' |
| 52 |
>>> 'filter the selection of profiles for dependency checks')) |
| 53 |
>>> |
| 54 |
>>> + parser.add_argument( |
| 55 |
>>> + '--include-profiles', |
| 56 |
>>> + dest='include_profiles', metavar='PROFILES', action='append', |
| 57 |
>>> + help=( |
| 58 |
>>> + 'A space separated list of profiles used to ' |
| 59 |
>>> + 'define the selection of profiles for dependency checks')) |
| 60 |
>>> + |
| 61 |
>>> parser.add_argument( |
| 62 |
>>> '-d', '--include-dev', dest='include_dev', action='store_true', |
| 63 |
>>> default=False, |
| 64 |
>>> diff --git a/repoman/lib/repoman/modules/scan/depend/__init__.py b/repoman/lib/repoman/modules/scan/depend/__init__.py |
| 65 |
>>> index c3cc0ddeb..9068760bb 100644 |
| 66 |
>>> --- a/repoman/lib/repoman/modules/scan/depend/__init__.py |
| 67 |
>>> +++ b/repoman/lib/repoman/modules/scan/depend/__init__.py |
| 68 |
>>> @@ -19,7 +19,8 @@ module_spec = { |
| 69 |
>>> 'func_desc': { |
| 70 |
>>> }, |
| 71 |
>>> 'mod_kwargs': ['qatracker', 'portdb', 'profiles', 'options', |
| 72 |
>>> - 'repo_metadata', 'repo_settings', 'include_arches', 'caches', |
| 73 |
>>> + 'repo_metadata', 'repo_settings', 'include_arches', |
| 74 |
>>> + 'include_profiles', 'caches', |
| 75 |
>>> 'repoman_incrementals', 'env', 'have', 'dev_keywords' |
| 76 |
>>> ], |
| 77 |
>>> 'func_kwargs': { |
| 78 |
>>> diff --git a/repoman/lib/repoman/modules/scan/depend/profile.py b/repoman/lib/repoman/modules/scan/depend/profile.py |
| 79 |
>>> index d980f4eca..0b1d74483 100644 |
| 80 |
>>> --- a/repoman/lib/repoman/modules/scan/depend/profile.py |
| 81 |
>>> +++ b/repoman/lib/repoman/modules/scan/depend/profile.py |
| 82 |
>>> @@ -33,6 +33,7 @@ class ProfileDependsChecks(ScanBase): |
| 83 |
>>> @param options: cli options |
| 84 |
>>> @param repo_settings: repository settings instance |
| 85 |
>>> @param include_arches: set |
| 86 |
>>> + @param include_profiles: set |
| 87 |
>>> @param caches: dictionary of our caches |
| 88 |
>>> @param repoman_incrementals: tuple |
| 89 |
>>> @param env: the environment |
| 90 |
>>> @@ -46,6 +47,7 @@ class ProfileDependsChecks(ScanBase): |
| 91 |
>>> self.options = kwargs.get('options') |
| 92 |
>>> self.repo_settings = kwargs.get('repo_settings') |
| 93 |
>>> self.include_arches = kwargs.get('include_arches') |
| 94 |
>>> + self.include_profiles = kwargs.get('include_profiles') |
| 95 |
>>> self.caches = kwargs.get('caches') |
| 96 |
>>> self.repoman_incrementals = kwargs.get('repoman_incrementals') |
| 97 |
>>> self.env = kwargs.get('env') |
| 98 |
>>> @@ -81,8 +83,11 @@ class ProfileDependsChecks(ScanBase): |
| 99 |
>>> if arch not in self.include_arches: |
| 100 |
>>> continue |
| 101 |
>>> |
| 102 |
>>> - relevant_profiles.extend( |
| 103 |
>>> - (keyword, groups, prof) for prof in self.profiles[arch]) |
| 104 |
>>> + for prof in self.profiles[arch]: |
| 105 |
>>> + if self.include_profiles is not None: |
| 106 |
>>> + if prof not in self.include_profiles: |
| 107 |
>>> + continue |
| 108 |
>>> + relevant_profiles.append((keyword, groups, prof)) |
| 109 |
>>> |
| 110 |
>>> relevant_profiles.sort(key=sort_key) |
| 111 |
>>> |
| 112 |
>>> diff --git a/repoman/lib/repoman/scanner.py b/repoman/lib/repoman/scanner.py |
| 113 |
>>> index 1b3242a51..06234b0ad 100644 |
| 114 |
>>> --- a/repoman/lib/repoman/scanner.py |
| 115 |
>>> +++ b/repoman/lib/repoman/scanner.py |
| 116 |
>>> @@ -164,6 +164,10 @@ class Scanner(object): |
| 117 |
>>> if self.options.include_arches: |
| 118 |
>>> self.include_arches = set() |
| 119 |
>>> self.include_arches.update(*[x.split() for x in self.options.include_arches]) |
| 120 |
>>> + self.include_profiles = None |
| 121 |
>>> + if self.options.include_profiles: |
| 122 |
>>> + self.include_profiles = set() |
| 123 |
>>> + self.include_profiles.update(*[x.split() for x in self.options.include_profiles]) |
| 124 |
>>> |
| 125 |
>>> # Disable the "self.modules['Ebuild'].notadded" check when not in commit mode and |
| 126 |
>>> # running `svn status` in every package dir will be too expensive. |
| 127 |
>>> @@ -190,6 +194,7 @@ class Scanner(object): |
| 128 |
>>> "repo_metadata": self.repo_metadata, |
| 129 |
>>> "profiles": profiles, |
| 130 |
>>> "include_arches": self.include_arches, |
| 131 |
>>> + "include_profiles": self.include_profiles, |
| 132 |
>>> "caches": self.caches, |
| 133 |
>>> "repoman_incrementals": self.repoman_incrementals, |
| 134 |
>>> "env": self.env, |
| 135 |
>>> diff --git a/repoman/man/repoman.1 b/repoman/man/repoman.1 |
| 136 |
>>> index 766146f57..f33fb6098 100644 |
| 137 |
>>> --- a/repoman/man/repoman.1 |
| 138 |
>>> +++ b/repoman/man/repoman.1 |
| 139 |
>>> @@ -120,6 +120,10 @@ Ignore masked packages (not allowed with commit mode) |
| 140 |
>>> A space separated list of arches used to filter the selection of |
| 141 |
>>> profiles for dependency checks. |
| 142 |
>>> .TP |
| 143 |
>>> +.BR "\-\-include\-profiles " PROFILES |
| 144 |
>>> +A space separated list of profiles used to |
| 145 |
>>> +define the selection of profiles for dependency checks. |
| 146 |
>>> +.TP |
| 147 |
>>> \fB\-d\fR, \fB\-\-include\-dev\fR |
| 148 |
>>> Include dev profiles in dependency checks. |
| 149 |
>>> .TP |
| 150 |
>>> |
| 151 |
>> |
| 152 |
>> Looks good. Please merge. |
| 153 |
>> -- |
| 154 |
>> Thanks, |
| 155 |
>> Zac |
| 156 |
>> |
| 157 |
> |
| 158 |
> Thank you! |
| 159 |
> |
| 160 |
> Does not look like I have permissions to do it: |
| 161 |
|
| 162 |
Ok, merged with prof.sub_path fix: |
| 163 |
|
| 164 |
https://gitweb.gentoo.org/proj/portage.git/commit/?id=e9bb1e9681685f4e4d7174f51751356fd3f67d0c |
| 165 |
|
| 166 |
-- |
| 167 |
Thanks, |
| 168 |
Zac |
Archives