Re: [gentoo-portage-dev] [PATCH] repoman: add --include-profiles=PROFILES

[Sergei Trofimovich]

On Mon, 18 Nov 2019 16:45:58 -0800
Zac Medico <zmedico@g.o> wrote:

> On 11/18/19 4:21 PM, Sergei Trofimovich wrote:
> > repoman slows down ~linearly with amount of profiles being scanned.
> > In case of amd64 we have 28 stable profiles.
> >
> > To speed up processing and fit into time budged of various CIs we can
> > split the work across different processes that handle different profiles.
> >
> > Example benchmark on ::haskell overlay:
> >     $ ./repoman full --include-arches=amd64
> >     ~65 minutes
> >     $ ./repoman full --include-profiles=default/linux/amd64/17.0
> >     ~4 minutes
> > This allows for a crude sharding of work across processes and allows for
> > cheap tree-wide scans for early failures.
> >
> > Bug: https://bugs.gentoo.org/700456
> > Signed-off-by: Sergei Trofimovich <slyfox@g.o>
> > ---
> >  repoman/lib/repoman/actions.py                      | 4 ++++
> >  repoman/lib/repoman/argparser.py                    | 7 +++++++
> >  repoman/lib/repoman/modules/scan/depend/__init__.py | 3 ++-
> >  repoman/lib/repoman/modules/scan/depend/profile.py  | 9 +++++++--
> >  repoman/lib/repoman/scanner.py                      | 5 +++++
> >  repoman/man/repoman.1                               | 4 ++++
> >  6 files changed, 29 insertions(+), 3 deletions(-)
> >
> > diff --git a/repoman/lib/repoman/actions.py b/repoman/lib/repoman/actions.py
> > index 1c9989a72..92d4d4e94 100644
> > --- a/repoman/lib/repoman/actions.py
> > +++ b/repoman/lib/repoman/actions.py
> > @@ -412,6 +412,10 @@ the whole commit message to abort.
> >  			report_options.append(
> >  				"--include-arches=\"%s\"" %
> >  				" ".join(sorted(self.scanner.include_arches)))
> > +		if self.scanner.include_profiles is not None:
> > +			report_options.append(
> > +				"--include-profiles=\"%s\"" %
> > +				" ".join(sorted(self.scanner.include_profiles)))
> >
> >  		if portage_version is None:
> >  			sys.stderr.write("Failed to insert portage version in message!\n")
> > diff --git a/repoman/lib/repoman/argparser.py b/repoman/lib/repoman/argparser.py
> > index fa0e6ff90..670a0e91d 100644
> > --- a/repoman/lib/repoman/argparser.py
> > +++ b/repoman/lib/repoman/argparser.py
> > @@ -164,6 +164,13 @@ def parse_args(argv, repoman_default_opts):
> >  			'A space separated list of arches used to '
> >  			'filter the selection of profiles for dependency checks'))
> >
> > +	parser.add_argument(
> > +		'--include-profiles',
> > +		dest='include_profiles', metavar='PROFILES', action='append',
> > +		help=(
> > +			'A space separated list of profiles used to '
> > +			'define the selection of profiles for dependency checks'))
> > +
> >  	parser.add_argument(
> >  		'-d', '--include-dev', dest='include_dev', action='store_true',
> >  		default=False,
> > diff --git a/repoman/lib/repoman/modules/scan/depend/__init__.py b/repoman/lib/repoman/modules/scan/depend/__init__.py
> > index c3cc0ddeb..9068760bb 100644
> > --- a/repoman/lib/repoman/modules/scan/depend/__init__.py
> > +++ b/repoman/lib/repoman/modules/scan/depend/__init__.py
> > @@ -19,7 +19,8 @@ module_spec = {
> >  			'func_desc': {
> >  			},
> >  			'mod_kwargs': ['qatracker', 'portdb', 'profiles', 'options',
> > -				'repo_metadata', 'repo_settings', 'include_arches', 'caches',
> > +				'repo_metadata', 'repo_settings', 'include_arches',
> > +				'include_profiles', 'caches',
> >  				'repoman_incrementals', 'env', 'have', 'dev_keywords'
> >  			],
> >  			'func_kwargs': {
> > diff --git a/repoman/lib/repoman/modules/scan/depend/profile.py b/repoman/lib/repoman/modules/scan/depend/profile.py
> > index d980f4eca..0b1d74483 100644
> > --- a/repoman/lib/repoman/modules/scan/depend/profile.py
> > +++ b/repoman/lib/repoman/modules/scan/depend/profile.py
> > @@ -33,6 +33,7 @@ class ProfileDependsChecks(ScanBase):
> >  		@param options: cli options
> >  		@param repo_settings: repository settings instance
> >  		@param include_arches: set
> > +		@param include_profiles: set
> >  		@param caches: dictionary of our caches
> >  		@param repoman_incrementals: tuple
> >  		@param env: the environment
> > @@ -46,6 +47,7 @@ class ProfileDependsChecks(ScanBase):
> >  		self.options = kwargs.get('options')
> >  		self.repo_settings = kwargs.get('repo_settings')
> >  		self.include_arches = kwargs.get('include_arches')
> > +		self.include_profiles = kwargs.get('include_profiles')
> >  		self.caches = kwargs.get('caches')
> >  		self.repoman_incrementals = kwargs.get('repoman_incrementals')
> >  		self.env = kwargs.get('env')
> > @@ -81,8 +83,11 @@ class ProfileDependsChecks(ScanBase):
> >  				if arch not in self.include_arches:
> >  					continue
> >
> > -			relevant_profiles.extend(
> > -				(keyword, groups, prof) for prof in self.profiles[arch])
> > +			for prof in self.profiles[arch]:
> > +				if self.include_profiles is not None:
> > +					if prof not in self.include_profiles:
> > +						continue
> > +				relevant_profiles.append((keyword, groups, prof))
> >
> >  		relevant_profiles.sort(key=sort_key)
> >
> > diff --git a/repoman/lib/repoman/scanner.py b/repoman/lib/repoman/scanner.py
> > index 1b3242a51..06234b0ad 100644
> > --- a/repoman/lib/repoman/scanner.py
> > +++ b/repoman/lib/repoman/scanner.py
> > @@ -164,6 +164,10 @@ class Scanner(object):
> >  		if self.options.include_arches:
> >  			self.include_arches = set()
> >  			self.include_arches.update(*[x.split() for x in self.options.include_arches])
> > +		self.include_profiles = None
> > +		if self.options.include_profiles:
> > +			self.include_profiles = set()
> > +			self.include_profiles.update(*[x.split() for x in self.options.include_profiles])
> >
> >  		# Disable the "self.modules['Ebuild'].notadded" check when not in commit mode and
> >  		# running `svn status` in every package dir will be too expensive.
> > @@ -190,6 +194,7 @@ class Scanner(object):
> >  			"repo_metadata": self.repo_metadata,
> >  			"profiles": profiles,
> >  			"include_arches": self.include_arches,
> > +			"include_profiles": self.include_profiles,
> >  			"caches": self.caches,
> >  			"repoman_incrementals": self.repoman_incrementals,
> >  			"env": self.env,
> > diff --git a/repoman/man/repoman.1 b/repoman/man/repoman.1
> > index 766146f57..f33fb6098 100644
> > --- a/repoman/man/repoman.1
> > +++ b/repoman/man/repoman.1
> > @@ -120,6 +120,10 @@ Ignore masked packages (not allowed with commit mode)
> >  A space separated list of arches used to filter the selection of
> >  profiles for dependency checks.
> >  .TP
> > +.BR "\-\-include\-profiles " PROFILES
> > +A space separated list of profiles used to
> > +define the selection of profiles for dependency checks.
> > +.TP
> >  \fB\-d\fR, \fB\-\-include\-dev\fR
> >  Include dev profiles in dependency checks.
> >  .TP
> >
>
> Looks good. Please merge.
> --
> Thanks,
> Zac
>

Thank you!

Does not look like I have permissions to do it:

"""
$ cat .git/config
[core]
	repositoryformatversion = 0
	filemode = true
	bare = false
	logallrefupdates = true
[remote "origin"]
	url = git+ssh://git@××××××××××.org/proj/portage.git
	fetch = +refs/heads/*:refs/remotes/origin/*
[branch "master"]
	remote = origin
	merge = refs/heads/master
$ LANG=C git push origin master

FATAL -- ACCESS DENIED
Repo            proj/portage
User            slyfox@g.o
Stage           Before git was called
Operation       Repo write

FATAL: W any proj/portage slyfox@g.o DENIED by fallthru
(or you mis-spelled the reponame)
fatal: Could not read from remote repository.

Please make sure you have the correct access rights
and the repository exists.
"""

-- 

  Sergei