Gentoo Archives: gentoo-portage-dev

From: Zac Medico <zmedico@g.o>
To: gentoo-portage-dev@l.g.o
Cc: Zac Medico <zmedico@g.o>
Subject: [gentoo-portage-dev] [PATCH] portdbapi.cp_list: cache repo associations (bug 650814)
Date: Sun, 15 Jul 2018 04:08:13
Message-Id: 20180715040539.2235-1-zmedico@gentoo.org
1 Make portdbapi.cp_list return _pkg_str instances that have a 'repo'
2 attribute (bindbapi.cp_list already does this), with results
3 in ascending order by (pkg.version, repo.priority). Optimize
4 portdbapi.findname2 to use the 'repo' attribute to enable cached
5 results for files previously found by the portdbapi.cp_list
6 method, avoiding filesystem access when possible. Optimize the
7 depgraph._iter_match_pkgs_atom method by elimination of the repo
8 loop, since portdbapi.cp_list now returns separate items when the
9 same package version is found in multiple repos.
10
11 Bug: https://bugs.gentoo.org/650814
12 ---
13 pym/_emerge/depgraph.py | 12 +++---------
14 pym/portage/dbapi/porttree.py | 41 +++++++++++++++++++++++++++++------------
15 2 files changed, 32 insertions(+), 21 deletions(-)
16
17 diff --git a/pym/_emerge/depgraph.py b/pym/_emerge/depgraph.py
18 index 42857c1a5..b63d4f242 100644
19 --- a/pym/_emerge/depgraph.py
20 +++ b/pym/_emerge/depgraph.py
21 @@ -5613,10 +5613,6 @@ class depgraph(object):
22 if cp_list:
23 atom_set = InternalPackageSet(initial_atoms=(atom,),
24 allow_repo=True)
25 - if atom.repo is None and hasattr(db, "getRepositories"):
26 - repo_list = db.getRepositories(catpkg=atom_exp.cp)
27 - else:
28 - repo_list = [atom.repo]
29
30 # descending order
31 cp_list.reverse()
32 @@ -5624,13 +5620,11 @@ class depgraph(object):
33 # Call match_from_list on one cpv at a time, in order
34 # to avoid unnecessary match_from_list comparisons on
35 # versions that are never yielded from this method.
36 - if not match_from_list(atom_exp, [cpv]):
37 - continue
38 - for repo in repo_list:
39 -
40 + if match_from_list(atom_exp, [cpv]):
41 try:
42 pkg = self._pkg(cpv, pkg_type, root_config,
43 - installed=installed, onlydeps=onlydeps, myrepo=repo)
44 + installed=installed, onlydeps=onlydeps,
45 + myrepo=getattr(cpv, 'repo', None))
46 except portage.exception.PackageNotFound:
47 pass
48 else:
49 diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py
50 index 3e36024ff..f6076ee2b 100644
51 --- a/pym/portage/dbapi/porttree.py
52 +++ b/pym/portage/dbapi/porttree.py
53 @@ -459,6 +459,9 @@ class portdbapi(dbapi):
54 mytree = self.treemap.get(myrepo)
55 if mytree is None:
56 return (None, 0)
57 + elif mytree is not None:
58 + # myrepo enables cached results when available
59 + myrepo = self.repositories.location_map.get(mytree)
60
61 mysplit = mycpv.split("/")
62 psplit = pkgsplit(mysplit[1])
63 @@ -495,6 +498,14 @@ class portdbapi(dbapi):
64 relative_path = mysplit[0] + _os.sep + psplit[0] + _os.sep + \
65 mysplit[1] + ".ebuild"
66
67 + # There is no need to access the filesystem when the package
68 + # comes from this db and the package repo attribute corresponds
69 + # to the desired repo, since the file was previously found by
70 + # the cp_list method.
71 + if (myrepo is not None and myrepo == getattr(mycpv, 'repo', None)
72 + and self is getattr(mycpv, '_db', None)):
73 + return (mytree + _os.sep + relative_path, mytree)
74 +
75 for x in mytrees:
76 filename = x + _os.sep + relative_path
77 if _os.access(_unicode_encode(filename,
78 @@ -950,18 +961,23 @@ class portdbapi(dbapi):
79 return cachelist[:]
80 mysplit = mycp.split("/")
81 invalid_category = mysplit[0] not in self._categories
82 - d={}
83 + # Process repos in ascending order by repo.priority, so that
84 + # stable sort by version produces results ordered by
85 + # (pkg.version, repo.priority).
86 if mytree is not None:
87 if isinstance(mytree, basestring):
88 - mytrees = [mytree]
89 + repos = [self.repositories.get_repo_for_location(mytree)]
90 else:
91 # assume it's iterable
92 - mytrees = mytree
93 + repos = [self.repositories.get_repo_for_location(location)
94 + for location in mytree]
95 elif self._better_cache is None:
96 - mytrees = self.porttrees
97 + repos = list(self.repositories)
98 else:
99 - mytrees = [repo.location for repo in self._better_cache[mycp]]
100 - for oroot in mytrees:
101 + repos = reversed(self._better_cache[mycp])
102 + mylist = []
103 + for repo in repos:
104 + oroot = repo.location
105 try:
106 file_list = os.listdir(os.path.join(oroot, mycp))
107 except OSError:
108 @@ -986,16 +1002,17 @@ class portdbapi(dbapi):
109 writemsg(_("\nInvalid ebuild version: %s\n") % \
110 os.path.join(oroot, mycp, x), noiselevel=-1)
111 continue
112 - d[_pkg_str(mysplit[0]+"/"+pf, db=self)] = None
113 - if invalid_category and d:
114 + mylist.append(_pkg_str(mysplit[0]+"/"+pf, db=self, repo=repo.name))
115 + if invalid_category and mylist:
116 writemsg(_("\n!!! '%s' has a category that is not listed in " \
117 "%setc/portage/categories\n") % \
118 (mycp, self.settings["PORTAGE_CONFIGROOT"]), noiselevel=-1)
119 mylist = []
120 - else:
121 - mylist = list(d)
122 - # Always sort in ascending order here since it's handy
123 - # and the result can be easily cached and reused.
124 + # Always sort in ascending order here since it's handy and
125 + # the result can be easily cached and reused. Since mylist
126 + # is initially in ascending order by repo.priority, stable
127 + # sort by version produces results in ascending order by
128 + # (pkg.version, repo.priority).
129 self._cpv_sort_ascending(mylist)
130 if self.frozen and mytree is None:
131 cachelist = mylist[:]
132 --
133 2.13.6

Replies