1 |
commit: 8267445cf2f8697f12f1424ecdb4f495dc19f27f |
2 |
Author: Zac Medico <zmedico <AT> gentoo <DOT> org> |
3 |
AuthorDate: Tue Nov 21 00:45:18 2017 +0000 |
4 |
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org> |
5 |
CommitDate: Tue Nov 21 01:31:21 2017 +0000 |
6 |
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=8267445c |
7 |
|
8 |
binarytree.populate: avoid lock when possible (bug 607872) |
9 |
|
10 |
In order to avoid unecessary lock contention, do not lock |
11 |
the Packages file unless it needs to be updated. This is |
12 |
useful when PKGDIR is shared via NFS. |
13 |
|
14 |
Bug: https://bugs.gentoo.org/607872 |
15 |
|
16 |
pym/portage/dbapi/bintree.py | 80 +++++++++++++++++++++++++------------------- |
17 |
1 file changed, 46 insertions(+), 34 deletions(-) |
18 |
|
19 |
diff --git a/pym/portage/dbapi/bintree.py b/pym/portage/dbapi/bintree.py |
20 |
index 95bd5dbf8..ffac8d216 100644 |
21 |
--- a/pym/portage/dbapi/bintree.py |
22 |
+++ b/pym/portage/dbapi/bintree.py |
23 |
@@ -523,21 +523,9 @@ class binarytree(object): |
24 |
if self._populating: |
25 |
return |
26 |
|
27 |
- pkgindex_lock = None |
28 |
- try: |
29 |
- if os.access(self.pkgdir, os.W_OK): |
30 |
- pkgindex_lock = lockfile(self._pkgindex_file, |
31 |
- wantnewlockfile=1) |
32 |
- self._populating = True |
33 |
- self._populate(getbinpkgs, getbinpkg_refresh=getbinpkg_refresh) |
34 |
- finally: |
35 |
- if pkgindex_lock: |
36 |
- unlockfile(pkgindex_lock) |
37 |
- self._populating = False |
38 |
- |
39 |
- def _populate(self, getbinpkgs=False, getbinpkg_refresh=True): |
40 |
- if (not os.path.isdir(self.pkgdir) and not getbinpkgs): |
41 |
- return 0 |
42 |
+ if not os.path.isdir(self.pkgdir) and not getbinpkgs: |
43 |
+ self.populated = True |
44 |
+ return |
45 |
|
46 |
# Clear all caches in case populate is called multiple times |
47 |
# as may be the case when _global_updates calls populate() |
48 |
@@ -545,6 +533,41 @@ class binarytree(object): |
49 |
# operate on local packages (getbinpkgs=0). |
50 |
self._remotepkgs = None |
51 |
self.dbapi.clear() |
52 |
+ |
53 |
+ self._populating = True |
54 |
+ try: |
55 |
+ update_pkgindex = self._populate_local() |
56 |
+ |
57 |
+ if update_pkgindex and self.dbapi.writable: |
58 |
+ # If the Packages file needs to be updated, then _populate_local |
59 |
+ # needs to be called once again while the file is locked, so |
60 |
+ # that changes made by a concurrent process cannot be lost. This |
61 |
+ # case is avoided when possible, in order to minimize lock |
62 |
+ # contention. |
63 |
+ pkgindex_lock = None |
64 |
+ try: |
65 |
+ pkgindex_lock = lockfile(self._pkgindex_file, |
66 |
+ wantnewlockfile=True) |
67 |
+ update_pkgindex = self._populate_local() |
68 |
+ if update_pkgindex: |
69 |
+ self._pkgindex_write(update_pkgindex) |
70 |
+ finally: |
71 |
+ if pkgindex_lock: |
72 |
+ unlockfile(pkgindex_lock) |
73 |
+ |
74 |
+ if getbinpkgs: |
75 |
+ if not self.settings.get("PORTAGE_BINHOST"): |
76 |
+ writemsg(_("!!! PORTAGE_BINHOST unset, but use is requested.\n"), |
77 |
+ noiselevel=-1) |
78 |
+ else: |
79 |
+ self._populate_remote(getbinpkg_refresh=getbinpkg_refresh) |
80 |
+ |
81 |
+ finally: |
82 |
+ self._populating = False |
83 |
+ |
84 |
+ self.populated = True |
85 |
+ |
86 |
+ def _populate_local(self): |
87 |
_instance_key = self.dbapi._instance_key |
88 |
if True: |
89 |
pkg_paths = {} |
90 |
@@ -557,7 +580,6 @@ class binarytree(object): |
91 |
pkgindex = self._load_pkgindex() |
92 |
if not self._pkgindex_version_supported(pkgindex): |
93 |
pkgindex = self._new_pkgindex() |
94 |
- header = pkgindex.header |
95 |
metadata = {} |
96 |
basename_index = {} |
97 |
for d in pkgindex.packages: |
98 |
@@ -773,22 +795,16 @@ class binarytree(object): |
99 |
if instance_key not in pkg_paths: |
100 |
del metadata[instance_key] |
101 |
|
102 |
- # Do not bother to write the Packages index if $PKGDIR/All/ exists |
103 |
- # since it will provide no benefit due to the need to read CATEGORY |
104 |
- # from xpak. |
105 |
- if update_pkgindex and os.access(self.pkgdir, os.W_OK): |
106 |
+ if update_pkgindex: |
107 |
del pkgindex.packages[:] |
108 |
pkgindex.packages.extend(iter(metadata.values())) |
109 |
self._update_pkgindex_header(pkgindex.header) |
110 |
- self._pkgindex_write(pkgindex) |
111 |
|
112 |
- if getbinpkgs and not self.settings.get("PORTAGE_BINHOST"): |
113 |
- writemsg(_("!!! PORTAGE_BINHOST unset, but use is requested.\n"), |
114 |
- noiselevel=-1) |
115 |
+ return pkgindex if update_pkgindex else None |
116 |
|
117 |
- if not getbinpkgs or 'PORTAGE_BINHOST' not in self.settings: |
118 |
- self.populated=1 |
119 |
- return |
120 |
+ def _populate_remote(self, getbinpkg_refresh=True): |
121 |
+ |
122 |
+ self._remote_has_index = False |
123 |
self._remotepkgs = {} |
124 |
for base_url in self.settings["PORTAGE_BINHOST"].split(): |
125 |
parsed_url = urlparse(base_url) |
126 |
@@ -802,7 +818,7 @@ class binarytree(object): |
127 |
user_passwd = user + "@" |
128 |
if ":" in user: |
129 |
user, passwd = user.split(":", 1) |
130 |
- port_args = [] |
131 |
+ |
132 |
if port is not None: |
133 |
port_str = ":%s" % (port,) |
134 |
if host.endswith(port_str): |
135 |
@@ -1008,23 +1024,19 @@ class binarytree(object): |
136 |
for d in pkgindex.packages: |
137 |
cpv = _pkg_str(d["CPV"], metadata=d, |
138 |
settings=self.settings) |
139 |
- instance_key = _instance_key(cpv) |
140 |
# Local package instances override remote instances |
141 |
# with the same instance_key. |
142 |
- if instance_key in metadata: |
143 |
+ if self.dbapi.cpv_exists(cpv): |
144 |
continue |
145 |
|
146 |
d["CPV"] = cpv |
147 |
d["BASE_URI"] = remote_base_uri |
148 |
d["PKGINDEX_URI"] = url |
149 |
- self._remotepkgs[instance_key] = d |
150 |
- metadata[instance_key] = d |
151 |
+ self._remotepkgs[self.dbapi._instance_key(cpv)] = d |
152 |
self.dbapi.cpv_inject(cpv) |
153 |
|
154 |
self._remote_has_index = True |
155 |
|
156 |
- self.populated=1 |
157 |
- |
158 |
def inject(self, cpv, filename=None): |
159 |
"""Add a freshly built package to the database. This updates |
160 |
$PKGDIR/Packages with the new package metadata (including MD5). |