Gentoo Archives: gentoo-commits

From: Zac Medico <zmedico@g.o>
To: gentoo-commits@l.g.o
Subject: [gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/
Date: Tue, 21 Nov 2017 01:34:28
Message-Id: 1511227881.8267445cf2f8697f12f1424ecdb4f495dc19f27f.zmedico@gentoo
1 commit: 8267445cf2f8697f12f1424ecdb4f495dc19f27f
2 Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
3 AuthorDate: Tue Nov 21 00:45:18 2017 +0000
4 Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
5 CommitDate: Tue Nov 21 01:31:21 2017 +0000
6 URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=8267445c
7
8 binarytree.populate: avoid lock when possible (bug 607872)
9
10 In order to avoid unecessary lock contention, do not lock
11 the Packages file unless it needs to be updated. This is
12 useful when PKGDIR is shared via NFS.
13
14 Bug: https://bugs.gentoo.org/607872
15
16 pym/portage/dbapi/bintree.py | 80 +++++++++++++++++++++++++-------------------
17 1 file changed, 46 insertions(+), 34 deletions(-)
18
19 diff --git a/pym/portage/dbapi/bintree.py b/pym/portage/dbapi/bintree.py
20 index 95bd5dbf8..ffac8d216 100644
21 --- a/pym/portage/dbapi/bintree.py
22 +++ b/pym/portage/dbapi/bintree.py
23 @@ -523,21 +523,9 @@ class binarytree(object):
24 if self._populating:
25 return
26
27 - pkgindex_lock = None
28 - try:
29 - if os.access(self.pkgdir, os.W_OK):
30 - pkgindex_lock = lockfile(self._pkgindex_file,
31 - wantnewlockfile=1)
32 - self._populating = True
33 - self._populate(getbinpkgs, getbinpkg_refresh=getbinpkg_refresh)
34 - finally:
35 - if pkgindex_lock:
36 - unlockfile(pkgindex_lock)
37 - self._populating = False
38 -
39 - def _populate(self, getbinpkgs=False, getbinpkg_refresh=True):
40 - if (not os.path.isdir(self.pkgdir) and not getbinpkgs):
41 - return 0
42 + if not os.path.isdir(self.pkgdir) and not getbinpkgs:
43 + self.populated = True
44 + return
45
46 # Clear all caches in case populate is called multiple times
47 # as may be the case when _global_updates calls populate()
48 @@ -545,6 +533,41 @@ class binarytree(object):
49 # operate on local packages (getbinpkgs=0).
50 self._remotepkgs = None
51 self.dbapi.clear()
52 +
53 + self._populating = True
54 + try:
55 + update_pkgindex = self._populate_local()
56 +
57 + if update_pkgindex and self.dbapi.writable:
58 + # If the Packages file needs to be updated, then _populate_local
59 + # needs to be called once again while the file is locked, so
60 + # that changes made by a concurrent process cannot be lost. This
61 + # case is avoided when possible, in order to minimize lock
62 + # contention.
63 + pkgindex_lock = None
64 + try:
65 + pkgindex_lock = lockfile(self._pkgindex_file,
66 + wantnewlockfile=True)
67 + update_pkgindex = self._populate_local()
68 + if update_pkgindex:
69 + self._pkgindex_write(update_pkgindex)
70 + finally:
71 + if pkgindex_lock:
72 + unlockfile(pkgindex_lock)
73 +
74 + if getbinpkgs:
75 + if not self.settings.get("PORTAGE_BINHOST"):
76 + writemsg(_("!!! PORTAGE_BINHOST unset, but use is requested.\n"),
77 + noiselevel=-1)
78 + else:
79 + self._populate_remote(getbinpkg_refresh=getbinpkg_refresh)
80 +
81 + finally:
82 + self._populating = False
83 +
84 + self.populated = True
85 +
86 + def _populate_local(self):
87 _instance_key = self.dbapi._instance_key
88 if True:
89 pkg_paths = {}
90 @@ -557,7 +580,6 @@ class binarytree(object):
91 pkgindex = self._load_pkgindex()
92 if not self._pkgindex_version_supported(pkgindex):
93 pkgindex = self._new_pkgindex()
94 - header = pkgindex.header
95 metadata = {}
96 basename_index = {}
97 for d in pkgindex.packages:
98 @@ -773,22 +795,16 @@ class binarytree(object):
99 if instance_key not in pkg_paths:
100 del metadata[instance_key]
101
102 - # Do not bother to write the Packages index if $PKGDIR/All/ exists
103 - # since it will provide no benefit due to the need to read CATEGORY
104 - # from xpak.
105 - if update_pkgindex and os.access(self.pkgdir, os.W_OK):
106 + if update_pkgindex:
107 del pkgindex.packages[:]
108 pkgindex.packages.extend(iter(metadata.values()))
109 self._update_pkgindex_header(pkgindex.header)
110 - self._pkgindex_write(pkgindex)
111
112 - if getbinpkgs and not self.settings.get("PORTAGE_BINHOST"):
113 - writemsg(_("!!! PORTAGE_BINHOST unset, but use is requested.\n"),
114 - noiselevel=-1)
115 + return pkgindex if update_pkgindex else None
116
117 - if not getbinpkgs or 'PORTAGE_BINHOST' not in self.settings:
118 - self.populated=1
119 - return
120 + def _populate_remote(self, getbinpkg_refresh=True):
121 +
122 + self._remote_has_index = False
123 self._remotepkgs = {}
124 for base_url in self.settings["PORTAGE_BINHOST"].split():
125 parsed_url = urlparse(base_url)
126 @@ -802,7 +818,7 @@ class binarytree(object):
127 user_passwd = user + "@"
128 if ":" in user:
129 user, passwd = user.split(":", 1)
130 - port_args = []
131 +
132 if port is not None:
133 port_str = ":%s" % (port,)
134 if host.endswith(port_str):
135 @@ -1008,23 +1024,19 @@ class binarytree(object):
136 for d in pkgindex.packages:
137 cpv = _pkg_str(d["CPV"], metadata=d,
138 settings=self.settings)
139 - instance_key = _instance_key(cpv)
140 # Local package instances override remote instances
141 # with the same instance_key.
142 - if instance_key in metadata:
143 + if self.dbapi.cpv_exists(cpv):
144 continue
145
146 d["CPV"] = cpv
147 d["BASE_URI"] = remote_base_uri
148 d["PKGINDEX_URI"] = url
149 - self._remotepkgs[instance_key] = d
150 - metadata[instance_key] = d
151 + self._remotepkgs[self.dbapi._instance_key(cpv)] = d
152 self.dbapi.cpv_inject(cpv)
153
154 self._remote_has_index = True
155
156 - self.populated=1
157 -
158 def inject(self, cpv, filename=None):
159 """Add a freshly built package to the database. This updates
160 $PKGDIR/Packages with the new package metadata (including MD5).