1 |
Each of these functions is called repeatedly with the same arguments |
2 |
many times. Cache sizes were selected to minimize memory use increase, |
3 |
while still providing about the same speedup compared to a cache with |
4 |
unbounded size. "emerge -uDvpU --with-bdeps=y @world" runtime decreases |
5 |
from 44.32s -> 29.94s -- a 48% speedup, while the maximum value of the |
6 |
RES column in htop increases from 280 MB -> 290 MB. |
7 |
|
8 |
"emerge -ep @world" time slightly decreases from 18.77s -> 17.93, while |
9 |
max observed RES value actually decreases from 228 MB -> 214 MB (similar |
10 |
values observed across a few before/after runs). |
11 |
--- |
12 |
lib/portage/dep/__init__.py | 106 +++++++++++++++++++++--------------- |
13 |
lib/portage/versions.py | 3 + |
14 |
2 files changed, 66 insertions(+), 43 deletions(-) |
15 |
|
16 |
diff --git a/lib/portage/dep/__init__.py b/lib/portage/dep/__init__.py |
17 |
index 72988357a..4d91a411a 100644 |
18 |
--- a/lib/portage/dep/__init__.py |
19 |
+++ b/lib/portage/dep/__init__.py |
20 |
@@ -23,6 +23,7 @@ portage.proxy.lazyimport.lazyimport(globals(), |
21 |
'portage.util:cmp_sort_key,writemsg', |
22 |
) |
23 |
|
24 |
+from functools import lru_cache |
25 |
from portage import _encodings, _unicode_decode, _unicode_encode |
26 |
from portage.eapi import _get_eapi_attrs |
27 |
from portage.exception import InvalidAtom, InvalidData, InvalidDependString |
28 |
@@ -404,49 +405,9 @@ def paren_enclose(mylist, unevaluated_atom=False, opconvert=False): |
29 |
mystrparts.append(x) |
30 |
return " ".join(mystrparts) |
31 |
|
32 |
-def use_reduce(depstr, uselist=(), masklist=(), matchall=False, excludeall=(), is_src_uri=False, \ |
33 |
- eapi=None, opconvert=False, flat=False, is_valid_flag=None, token_class=None, matchnone=False, |
34 |
- subset=None): |
35 |
- """ |
36 |
- Takes a dep string and reduces the use? conditionals out, leaving an array |
37 |
- with subarrays. All redundant brackets are removed. |
38 |
- |
39 |
- @param depstr: depstring |
40 |
- @type depstr: String |
41 |
- @param uselist: Sequence of use enabled flags |
42 |
- @type uselist: Sequence |
43 |
- @param masklist: Sequence of masked flags (always treated as disabled) |
44 |
- @type masklist: Sequence |
45 |
- @param matchall: Treat all conditionals as active. Used by repoman. |
46 |
- @type matchall: Bool |
47 |
- @param excludeall: Sequence of flags for which negated conditionals are always treated as inactive. |
48 |
- @type excludeall: Sequence |
49 |
- @param is_src_uri: Indicates if depstr represents a SRC_URI |
50 |
- @type is_src_uri: Bool |
51 |
- @param eapi: Indicates the EAPI the dep string has to comply to |
52 |
- @type eapi: String |
53 |
- @param opconvert: Put every operator as first element into it's argument list |
54 |
- @type opconvert: Bool |
55 |
- @param flat: Create a flat list of all tokens |
56 |
- @type flat: Bool |
57 |
- @param is_valid_flag: Function that decides if a given use flag might be used in use conditionals |
58 |
- @type is_valid_flag: Function |
59 |
- @param token_class: Convert all non operator tokens into this class |
60 |
- @type token_class: Class |
61 |
- @param matchnone: Treat all conditionals as inactive. Used by digestgen(). |
62 |
- @type matchnone: Bool |
63 |
- @param subset: Select a subset of dependencies conditional on the given flags |
64 |
- @type subset: Sequence |
65 |
- @rtype: List |
66 |
- @return: The use reduced depend array |
67 |
- """ |
68 |
- if isinstance(depstr, list): |
69 |
- if portage._internal_caller: |
70 |
- warnings.warn(_("Passing paren_reduced dep arrays to %s is deprecated. " + \ |
71 |
- "Pass the original dep string instead.") % \ |
72 |
- ('portage.dep.use_reduce',), DeprecationWarning, stacklevel=2) |
73 |
- depstr = paren_enclose(depstr) |
74 |
- |
75 |
+@lru_cache(1024) |
76 |
+def use_reduce_cached(depstr, uselist, masklist, matchall, excludeall, is_src_uri, eapi, \ |
77 |
+ opconvert, flat, is_valid_flag, token_class, matchnone, subset): |
78 |
if opconvert and flat: |
79 |
raise ValueError("portage.dep.use_reduce: 'opconvert' and 'flat' are mutually exclusive") |
80 |
|
81 |
@@ -769,6 +730,65 @@ def use_reduce(depstr, uselist=(), masklist=(), matchall=False, excludeall=(), i |
82 |
|
83 |
return stack[0] |
84 |
|
85 |
+def use_reduce(depstr, uselist=(), masklist=(), matchall=False, excludeall=(), is_src_uri=False, \ |
86 |
+ eapi=None, opconvert=False, flat=False, is_valid_flag=None, token_class=None, matchnone=False, |
87 |
+ subset=None): |
88 |
+ """ |
89 |
+ Takes a dep string and reduces the use? conditionals out, leaving an array |
90 |
+ with subarrays. All redundant brackets are removed. |
91 |
+ |
92 |
+ @param depstr: depstring |
93 |
+ @type depstr: String |
94 |
+ @param uselist: Sequence of use enabled flags |
95 |
+ @type uselist: Sequence |
96 |
+ @param masklist: Sequence of masked flags (always treated as disabled) |
97 |
+ @type masklist: Sequence |
98 |
+ @param matchall: Treat all conditionals as active. Used by repoman. |
99 |
+ @type matchall: Bool |
100 |
+ @param excludeall: Sequence of flags for which negated conditionals are always treated as inactive. |
101 |
+ @type excludeall: Sequence |
102 |
+ @param is_src_uri: Indicates if depstr represents a SRC_URI |
103 |
+ @type is_src_uri: Bool |
104 |
+ @param eapi: Indicates the EAPI the dep string has to comply to |
105 |
+ @type eapi: String |
106 |
+ @param opconvert: Put every operator as first element into it's argument list |
107 |
+ @type opconvert: Bool |
108 |
+ @param flat: Create a flat list of all tokens |
109 |
+ @type flat: Bool |
110 |
+ @param is_valid_flag: Function that decides if a given use flag might be used in use conditionals |
111 |
+ @type is_valid_flag: Function |
112 |
+ @param token_class: Convert all non operator tokens into this class |
113 |
+ @type token_class: Class |
114 |
+ @param matchnone: Treat all conditionals as inactive. Used by digestgen(). |
115 |
+ @type matchnone: Bool |
116 |
+ @param subset: Select a subset of dependencies conditional on the given flags |
117 |
+ @type subset: Sequence |
118 |
+ @rtype: List |
119 |
+ @return: The use reduced depend array |
120 |
+ """ |
121 |
+ if isinstance(depstr, list): |
122 |
+ if portage._internal_caller: |
123 |
+ warnings.warn(_("Passing paren_reduced dep arrays to %s is deprecated. " + \ |
124 |
+ "Pass the original dep string instead.") % \ |
125 |
+ ('portage.dep.use_reduce',), DeprecationWarning, stacklevel=2) |
126 |
+ depstr = paren_enclose(depstr) |
127 |
+ |
128 |
+ if uselist is not None: |
129 |
+ uselist = tuple(uselist) |
130 |
+ if masklist is not None: |
131 |
+ masklist = tuple(masklist) |
132 |
+ if excludeall is not None: |
133 |
+ excludeall = tuple(excludeall) |
134 |
+ if subset is not None: |
135 |
+ subset = tuple(subset) |
136 |
+ |
137 |
+ result = use_reduce_cached(depstr, uselist, masklist, matchall, excludeall, \ |
138 |
+ is_src_uri, eapi, opconvert, flat, is_valid_flag, token_class, \ |
139 |
+ matchnone, subset) |
140 |
+ |
141 |
+ # The list returned by this function may be modified, so return a copy. |
142 |
+ return result[:] |
143 |
+ |
144 |
def dep_opconvert(deplist): |
145 |
""" |
146 |
Iterate recursively through a list of deps, if the |
147 |
diff --git a/lib/portage/versions.py b/lib/portage/versions.py |
148 |
index 0c21373cc..9ede67230 100644 |
149 |
--- a/lib/portage/versions.py |
150 |
+++ b/lib/portage/versions.py |
151 |
@@ -25,6 +25,7 @@ portage.proxy.lazyimport.lazyimport(globals(), |
152 |
'portage.repository.config:_gen_valid_repo', |
153 |
'portage.util:cmp_sort_key', |
154 |
) |
155 |
+from functools import lru_cache |
156 |
from portage import _unicode_decode |
157 |
from portage.eapi import _get_eapi_attrs |
158 |
from portage.exception import InvalidData |
159 |
@@ -116,6 +117,7 @@ def ververify(myver, silent=1): |
160 |
print(_("!!! syntax error in version: %s") % myver) |
161 |
return False |
162 |
|
163 |
+@lru_cache(1024) |
164 |
def vercmp(ver1, ver2, silent=1): |
165 |
""" |
166 |
Compare two versions |
167 |
@@ -313,6 +315,7 @@ def _pkgsplit(mypkg, eapi=None): |
168 |
_cat_re = re.compile('^%s$' % _cat, re.UNICODE) |
169 |
_missing_cat = 'null' |
170 |
|
171 |
+@lru_cache(10240) |
172 |
def catpkgsplit(mydata, silent=1, eapi=None): |
173 |
""" |
174 |
Takes a Category/Package-Version-Rev and returns a list of each. |
175 |
-- |
176 |
2.27.0.383.g050319c2ae-goog |