[gentoo-commits] proj/portage:master commit in: pym/_emerge/, pym/portage/dbapi/

[Zac Medico]

commit:     27eeeb6e4fc8e68efa003b2db5bbd8cc4afe336a
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Fri Jul 13 22:19:29 2018 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sun Jul 15 05:10:51 2018 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=27eeeb6e

portdbapi.cp_list: cache repo associations (bug 650814)

Make portdbapi.cp_list return _pkg_str instances that have a 'repo'
attribute (bindbapi.cp_list already does this), with results
in ascending order by (pkg.version, repo.priority). Optimize
portdbapi.findname2 to use the 'repo' attribute to enable cached
results for files previously found by the portdbapi.cp_list
method, avoiding filesystem access when possible. Optimize the
depgraph._iter_match_pkgs_atom method by elimination of the repo
loop, since portdbapi.cp_list now returns separate items when the
same package version is found in multiple repos.

Bug: https://bugs.gentoo.org/650814
Reviewed-by: Brian Dolbec <dolsen <AT> gentoo.org>

 pym/_emerge/depgraph.py       | 12 +++---------
 pym/portage/dbapi/porttree.py | 41 +++++++++++++++++++++++++++++------------
 2 files changed, 32 insertions(+), 21 deletions(-)

diff --git a/pym/_emerge/depgraph.py b/pym/_emerge/depgraph.py
index 42857c1a5..b63d4f242 100644
--- a/pym/_emerge/depgraph.py
+++ b/pym/_emerge/depgraph.py
@@ -5613,10 +5613,6 @@ class depgraph(object):
 		if cp_list:
 			atom_set = InternalPackageSet(initial_atoms=(atom,),
 				allow_repo=True)
-			if atom.repo is None and hasattr(db, "getRepositories"):
-				repo_list = db.getRepositories(catpkg=atom_exp.cp)
-			else:
-				repo_list = [atom.repo]
 
 			# descending order
 			cp_list.reverse()
@@ -5624,13 +5620,11 @@ class depgraph(object):
 				# Call match_from_list on one cpv at a time, in order
 				# to avoid unnecessary match_from_list comparisons on
 				# versions that are never yielded from this method.
-				if not match_from_list(atom_exp, [cpv]):
-					continue
-				for repo in repo_list:
-
+				if match_from_list(atom_exp, [cpv]):
 					try:
 						pkg = self._pkg(cpv, pkg_type, root_config,
-							installed=installed, onlydeps=onlydeps, myrepo=repo)
+							installed=installed, onlydeps=onlydeps,
+							myrepo=getattr(cpv, 'repo', None))
 					except portage.exception.PackageNotFound:
 						pass
 					else:

diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py
index 3e36024ff..f6076ee2b 100644
--- a/pym/portage/dbapi/porttree.py
+++ b/pym/portage/dbapi/porttree.py
@@ -459,6 +459,9 @@ class portdbapi(dbapi):
 			mytree = self.treemap.get(myrepo)
 			if mytree is None:
 				return (None, 0)
+		elif mytree is not None:
+			# myrepo enables cached results when available
+			myrepo = self.repositories.location_map.get(mytree)
 
 		mysplit = mycpv.split("/")
 		psplit = pkgsplit(mysplit[1])
@@ -495,6 +498,14 @@ class portdbapi(dbapi):
 		relative_path = mysplit[0] + _os.sep + psplit[0] + _os.sep + \
 			mysplit[1] + ".ebuild"
 
+		# There is no need to access the filesystem when the package
+		# comes from this db and the package repo attribute corresponds
+		# to the desired repo, since the file was previously found by
+		# the cp_list method.
+		if (myrepo is not None and myrepo == getattr(mycpv, 'repo', None)
+			and self is getattr(mycpv, '_db', None)):
+			return (mytree + _os.sep + relative_path, mytree)
+
 		for x in mytrees:
 			filename = x + _os.sep + relative_path
 			if _os.access(_unicode_encode(filename,
@@ -950,18 +961,23 @@ class portdbapi(dbapi):
 				return cachelist[:]
 		mysplit = mycp.split("/")
 		invalid_category = mysplit[0] not in self._categories
-		d={}
+		# Process repos in ascending order by repo.priority, so that
+		# stable sort by version produces results ordered by
+		# (pkg.version, repo.priority).
 		if mytree is not None:
 			if isinstance(mytree, basestring):
-				mytrees = [mytree]
+				repos = [self.repositories.get_repo_for_location(mytree)]
 			else:
 				# assume it's iterable
-				mytrees = mytree
+				repos = [self.repositories.get_repo_for_location(location)
+					for location in mytree]
 		elif self._better_cache is None:
-			mytrees = self.porttrees
+			repos = list(self.repositories)
 		else:
-			mytrees = [repo.location for repo in self._better_cache[mycp]]
-		for oroot in mytrees:
+			repos = reversed(self._better_cache[mycp])
+		mylist = []
+		for repo in repos:
+			oroot = repo.location
 			try:
 				file_list = os.listdir(os.path.join(oroot, mycp))
 			except OSError:
@@ -986,16 +1002,17 @@ class portdbapi(dbapi):
 						writemsg(_("\nInvalid ebuild version: %s\n") % \
 							os.path.join(oroot, mycp, x), noiselevel=-1)
 						continue
-					d[_pkg_str(mysplit[0]+"/"+pf, db=self)] = None
-		if invalid_category and d:
+					mylist.append(_pkg_str(mysplit[0]+"/"+pf, db=self, repo=repo.name))
+		if invalid_category and mylist:
 			writemsg(_("\n!!! '%s' has a category that is not listed in " \
 				"%setc/portage/categories\n") % \
 				(mycp, self.settings["PORTAGE_CONFIGROOT"]), noiselevel=-1)
 			mylist = []
-		else:
-			mylist = list(d)
-		# Always sort in ascending order here since it's handy
-		# and the result can be easily cached and reused.
+		# Always sort in ascending order here since it's handy and
+		# the result can be easily cached and reused. Since mylist
+		# is initially in ascending order by repo.priority, stable
+		# sort by version produces results in ascending order by
+		# (pkg.version, repo.priority).
 		self._cpv_sort_ascending(mylist)
 		if self.frozen and mytree is None:
 			cachelist = mylist[:]