Gentoo Archives: gentoo-portage-dev

From: Zac Medico <zmedico@g.o>
To: gentoo-portage-dev@l.g.o
Cc: Zac Medico <zmedico@g.o>
Subject: [gentoo-portage-dev] [PATCH] _select_pkg_highest_available: selective cache invalidation (530010)
Date: Fri, 21 Nov 2014 04:06:00
Message-Id: 1416542747-14585-1-git-send-email-zmedico@gentoo.org
1 Implement selective invalidation of cache for the
2 depgraph._select_pkg_highest_available method, so that the entire cache
3 is not discarded whenever a package is added to the graph. On one of my
4 computers with this patch, 'emerge -puvDN @world' takes 15% less time,
5 and results in 58% fewer _select_pkg_highest_available_imp calls.
6
7 X-Gentoo-Bug: 530010
8 X-Gentoo-Url: https://bugs.gentoo.org/show_bug.cgi?id=530010
9 ---
10 pym/_emerge/Package.py | 18 ++++++++++++++++--
11 pym/_emerge/depgraph.py | 14 +++++++++++++-
12 2 files changed, 29 insertions(+), 3 deletions(-)
13
14 diff --git a/pym/_emerge/Package.py b/pym/_emerge/Package.py
15 index a09f73c..8612e8b 100644
16 --- a/pym/_emerge/Package.py
17 +++ b/pym/_emerge/Package.py
18 @@ -35,8 +35,8 @@ class Package(Task):
19 "category", "counter", "cp", "cpv_split",
20 "inherited", "iuse", "mtime",
21 "pf", "root", "slot", "sub_slot", "slot_atom", "version") + \
22 - ("_invalid", "_masks", "_metadata", "_raw_metadata", "_use",
23 - "_validated_atoms", "_visible")
24 + ("_invalid", "_masks", "_metadata", "_provided_cps",
25 + "_raw_metadata", "_use", "_validated_atoms", "_visible")
26
27 metadata_keys = [
28 "BUILD_TIME", "CHOST", "COUNTER", "DEPEND", "EAPI",
29 @@ -128,6 +128,20 @@ class Package(Task):
30 return self._metadata.properties
31
32 @property
33 + def provided_cps(self):
34 +
35 + if self._provided_cps is None:
36 + provided_cps = [self.cp]
37 + for atom in self._metadata["PROVIDE"].split():
38 + try:
39 + provided_cps.append(Atom(atom).cp)
40 + except InvalidAtom:
41 + pass
42 + self._provided_cps = tuple(provided_cps)
43 +
44 + return self._provided_cps
45 +
46 + @property
47 def restrict(self):
48 return self._metadata.restrict
49
50 diff --git a/pym/_emerge/depgraph.py b/pym/_emerge/depgraph.py
51 index 3f4a097..ab79aef 100644
52 --- a/pym/_emerge/depgraph.py
53 +++ b/pym/_emerge/depgraph.py
54 @@ -399,6 +399,7 @@ class _dynamic_depgraph_config(object):
55 self._initially_unsatisfied_deps = []
56 self._ignored_deps = []
57 self._highest_pkg_cache = {}
58 + self._highest_pkg_cache_cp_map = {}
59 self._flatten_atoms_cache = {}
60
61 # Binary packages that have been rejected because their USE
62 @@ -2539,8 +2540,8 @@ class depgraph(object):
63 if not previously_added:
64 self._dynamic_config._package_tracker.add_pkg(pkg)
65 self._dynamic_config._filtered_trees[pkg.root]["porttree"].dbapi._clear_cache()
66 - self._dynamic_config._highest_pkg_cache.clear()
67 self._check_masks(pkg)
68 + self._prune_highest_pkg_cache(pkg)
69
70 if not pkg.installed:
71 # Allow this package to satisfy old-style virtuals in case it
72 @@ -2685,6 +2686,7 @@ class depgraph(object):
73 # Clear caches.
74 self._dynamic_config._filtered_trees[pkg.root]["porttree"].dbapi._clear_cache()
75 self._dynamic_config._highest_pkg_cache.clear()
76 + self._dynamic_config._highest_pkg_cache_cp_map.clear()
77
78
79 def _check_masks(self, pkg):
80 @@ -3967,6 +3969,7 @@ class depgraph(object):
81 # Invalidate the package selection cache, since
82 # arguments influence package selections.
83 self._dynamic_config._highest_pkg_cache.clear()
84 + self._dynamic_config._highest_pkg_cache_cp_map.clear()
85 for trees in self._dynamic_config._filtered_trees.values():
86 trees["porttree"].dbapi._clear_cache()
87
88 @@ -4987,6 +4990,8 @@ class depgraph(object):
89 return ret
90 ret = self._select_pkg_highest_available_imp(root, atom, onlydeps=onlydeps, parent=parent)
91 self._dynamic_config._highest_pkg_cache[cache_key] = ret
92 + self._dynamic_config._highest_pkg_cache_cp_map. \
93 + setdefault(atom.cp, []).append(cache_key)
94 pkg, existing = ret
95 if pkg is not None:
96 if self._pkg_visibility_check(pkg) and \
97 @@ -4994,6 +4999,13 @@ class depgraph(object):
98 self._dynamic_config._visible_pkgs[pkg.root].cpv_inject(pkg)
99 return ret
100
101 + def _prune_highest_pkg_cache(self, pkg):
102 + for cp in pkg.provided_cps:
103 + for cache_key in self._dynamic_config. \
104 + _highest_pkg_cache_cp_map.pop(cp, []):
105 + self._dynamic_config._highest_pkg_cache.pop(
106 + cache_key, None)
107 +
108 def _want_installed_pkg(self, pkg):
109 """
110 Given an installed package returned from select_pkg, return
111 --
112 2.0.4

Replies