Gentoo Archives: gentoo-portage-dev

From: "W. Trevor King" <wking@×××××××.us>
To: gentoo-portage-dev@l.g.o
Cc: "W. Trevor King" <wking@×××××××.us>, Zac Medico <zmedico@g.o>, Arfrever Frehtes Taifersar Arahesis <arfrever@g.o>
Subject: [gentoo-portage-dev] [PATCH v2 3/3] pym/portage/package/ebuild/fetch.py: Factor out _get_uris
Date: Sun, 19 Jan 2014 22:15:17
Message-Id: 11f339ff2422304916eee3d9b0eb510baa535e6b.1390169448.git.wking@tremily.us
In Reply to: [gentoo-portage-dev] [PATCH v2 0/3] Initial fetch() refactoring by "W. Trevor King"
1 The current fetch() function is quite long, which makes it hard to
2 know what I can change without adverse side effects. By pulling this
3 logic out of the main function, we get clearer logic in fetch() and
4 more explicit input for the config extraction.
5
6 This block was especially complicated, so I also created the helper
7 functions _get_file_uri_tuples and _expand_mirror. I'd prefer if
8 _expand_mirror iterated through URIs instead of (group, URI) tuples,
9 but we need a distinct marker for third-party URIs to build
10 third_party_mirror_uris which is used to build primaryuri_dict which
11 is used way down in fetch():
12
13 if checksum_failure_count == \
14 checksum_failure_primaryuri:
15 # Switch to "primaryuri" mode in order
16 # to increase the probablility of
17 # of success.
18 primaryuris = \
19 primaryuri_dict.get(myfile)
20 if primaryuris:
21 uri_list.extend(
22 reversed(primaryuris))
23
24 I don't know if this is useful enough to motivate the uglier
25 _expandmirror return values, but I'll kick that can down the road for
26 now.
27 ---
28 pym/portage/package/ebuild/fetch.py | 209 ++++++++++++++++++++++--------------
29 1 file changed, 128 insertions(+), 81 deletions(-)
30
31 diff --git a/pym/portage/package/ebuild/fetch.py b/pym/portage/package/ebuild/fetch.py
32 index de5bf00..a1940f4 100644
33 --- a/pym/portage/package/ebuild/fetch.py
34 +++ b/pym/portage/package/ebuild/fetch.py
35 @@ -15,9 +15,9 @@ import sys
36 import tempfile
37
38 try:
39 - from urllib.parse import urlparse
40 + from urllib.parse import urlparse, urlunparse
41 except ImportError:
42 - from urlparse import urlparse
43 + from urlparse import urlparse, urlunparse
44
45 import portage
46 portage.proxy.lazyimport.lazyimport(globals(),
47 @@ -298,6 +298,129 @@ def _get_fetch_resume_size(settings, default='350K'):
48 return v
49
50
51 +def _get_file_uri_tuples(uris):
52 + """
53 + Return a list of (filename, uri) tuples
54 + """
55 + file_uri_tuples = []
56 + # Check for 'items' attribute since OrderedDict is not a dict.
57 + if hasattr(uris, 'items'):
58 + for filename, uri_set in uris.items():
59 + for uri in uri_set:
60 + file_uri_tuples.append((filename, uri))
61 + if not uri_set:
62 + file_uri_tuples.append((filename, None))
63 + else:
64 + for uri in uris:
65 + if urlparse(uri).scheme:
66 + file_uri_tuples.append(
67 + (os.path.basename(uri), uri))
68 + else:
69 + file_uri_tuples.append(
70 + (os.path.basename(uri), None))
71 + return file_uri_tuples
72 +
73 +
74 +def _expand_mirror(uri, custom_mirrors=(), third_party_mirrors=()):
75 + """
76 + Replace the 'mirror://' scheme in the uri
77 +
78 + Returns an iterable listing expanded (group, URI) tuples,
79 + where the group is either 'custom' or 'third-party'.
80 + """
81 + parsed = urlparse(uri)
82 + mirror = parsed.netloc
83 + path = parsed.path
84 + if path:
85 + # Try user-defined mirrors first
86 + if mirror in custom_mirrors:
87 + for cmirr in custom_mirrors[mirror]:
88 + m_uri = urlparse(cmirr)
89 + yield ('custom', urlunparse((
90 + m_uri.scheme, m_uri.netloc, path) +
91 + parsed[3:]))
92 +
93 + # now try the official mirrors
94 + if mirror in third_party_mirrors:
95 + uris = []
96 + for locmirr in third_party_mirrors[mirror]:
97 + m_uri = urlparse(locmirr)
98 + uris.append(urlunparse((
99 + m_uri.scheme, m_uri.netloc, path) +
100 + parsed[3:]))
101 + random.shuffle(uris)
102 + for uri in uris:
103 + yield ('third-party', uri)
104 +
105 + if (not custom_mirrors.get(mirror, []) and
106 + not third_party_mirrors.get(mirror, [])):
107 + writemsg(
108 + _("No known mirror by the name: %s\n")
109 + % (mirror,))
110 + else:
111 + writemsg(_("Invalid mirror definition in SRC_URI:\n"),
112 + noiselevel=-1)
113 + writemsg(" %s\n" % (uri), noiselevel=-1)
114 +
115 +
116 +def _get_uris(uris, settings, custom_mirrors=(), locations=()):
117 + restrict = settings.get("PORTAGE_RESTRICT", "").split()
118 + restrict_fetch = "fetch" in restrict
119 + restrict_mirror = "mirror" in restrict or "nomirror" in restrict
120 + force_mirror = (
121 + "force-mirror" in settings.features and
122 + not restrict_mirror)
123 +
124 + third_party_mirrors = settings.thirdpartymirrors()
125 + third_party_mirror_uris = {}
126 + filedict = OrderedDict()
127 + primaryuri_dict = {}
128 + for filename, uri in _get_file_uri_tuples(uris=uris):
129 + if filename not in filedict:
130 + filedict[filename] = [
131 + os.path.join(location, 'distfiles', filename)
132 + for location in locations]
133 + if uri is None:
134 + continue
135 + if uri.startswith('mirror://'):
136 + uris = _expand_mirror(
137 + uri=uri, custom_mirrors=custom_mirrors,
138 + third_party_mirrors=third_party_mirrors)
139 + filedict[filename].extend(uri for group, uri in uris)
140 + third_party_mirror_uris.setdefault(filename, []).extend(
141 + uri for group, uri in uris
142 + if group == 'third-party')
143 + else:
144 + if restrict_fetch or force_mirror:
145 + # Only fetch from specific mirrors is allowed.
146 + continue
147 + primaryuris = primaryuri_dict.get(filename)
148 + if primaryuris is None:
149 + primaryuris = []
150 + primaryuri_dict[filename] = primaryuris
151 + primaryuris.append(uri)
152 +
153 + # Order primaryuri_dict values to match that in SRC_URI.
154 + for uris in primaryuri_dict.values():
155 + uris.reverse()
156 +
157 + # Prefer third_party_mirrors over normal mirrors in cases when
158 + # the file does not yet exist on the normal mirrors.
159 + for filename, uris in third_party_mirror_uris.items():
160 + primaryuri_dict.setdefault(filename, []).extend(uris)
161 +
162 + # Now merge primaryuri values into filedict (includes mirrors
163 + # explicitly referenced in SRC_URI).
164 + if "primaryuri" in restrict:
165 + for filename, uris in filedict.items():
166 + filedict[filename] = primaryuri_dict.get(filename, []) + uris
167 + else:
168 + for filename in filedict:
169 + filedict[filename] += primaryuri_dict.get(filename, [])
170 +
171 + return filedict, primaryuri_dict
172 +
173 +
174 def fetch(myuris, mysettings, listonly=0, fetchonly=0,
175 locks_in_subdir=".locks", use_locks=1, try_mirrors=1, digests=None,
176 allow_missing_digests=True):
177 @@ -329,7 +452,6 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0,
178 # couple of checksum failures, to increase the probablility
179 # of success before checksum_failure_max_tries is reached.
180 checksum_failure_primaryuri = 2
181 - thirdpartymirrors = mysettings.thirdpartymirrors()
182
183 # In the background parallel-fetch process, it's safe to skip checksum
184 # verification of pre-existing files in $DISTDIR that have the correct
185 @@ -402,7 +524,6 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0,
186 del mymirrors[x]
187
188 restrict_fetch = "fetch" in restrict
189 - force_mirror = "force-mirror" in features and not restrict_mirror
190 custom_local_mirrors = custommirrors.get("local", [])
191 if restrict_fetch:
192 # With fetch restriction, a normal uri may only be fetched from
193 @@ -413,83 +534,9 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0,
194 else:
195 locations = mymirrors
196
197 - file_uri_tuples = []
198 - # Check for 'items' attribute since OrderedDict is not a dict.
199 - if hasattr(myuris, 'items'):
200 - for myfile, uri_set in myuris.items():
201 - for myuri in uri_set:
202 - file_uri_tuples.append((myfile, myuri))
203 - if not uri_set:
204 - file_uri_tuples.append((myfile, None))
205 - else:
206 - for myuri in myuris:
207 - if urlparse(myuri).scheme:
208 - file_uri_tuples.append((os.path.basename(myuri), myuri))
209 - else:
210 - file_uri_tuples.append((os.path.basename(myuri), None))
211 -
212 - filedict = OrderedDict()
213 - primaryuri_dict = {}
214 - thirdpartymirror_uris = {}
215 - for myfile, myuri in file_uri_tuples:
216 - if myfile not in filedict:
217 - filedict[myfile]=[]
218 - for y in range(0,len(locations)):
219 - filedict[myfile].append(locations[y]+"/distfiles/"+myfile)
220 - if myuri is None:
221 - continue
222 - if myuri[:9]=="mirror://":
223 - eidx = myuri.find("/", 9)
224 - if eidx != -1:
225 - mirrorname = myuri[9:eidx]
226 - path = myuri[eidx+1:]
227 -
228 - # Try user-defined mirrors first
229 - if mirrorname in custommirrors:
230 - for cmirr in custommirrors[mirrorname]:
231 - filedict[myfile].append(
232 - cmirr.rstrip("/") + "/" + path)
233 -
234 - # now try the official mirrors
235 - if mirrorname in thirdpartymirrors:
236 - uris = [locmirr.rstrip("/") + "/" + path \
237 - for locmirr in thirdpartymirrors[mirrorname]]
238 - random.shuffle(uris)
239 - filedict[myfile].extend(uris)
240 - thirdpartymirror_uris.setdefault(myfile, []).extend(uris)
241 -
242 - if not filedict[myfile]:
243 - writemsg(_("No known mirror by the name: %s\n") % (mirrorname))
244 - else:
245 - writemsg(_("Invalid mirror definition in SRC_URI:\n"), noiselevel=-1)
246 - writemsg(" %s\n" % (myuri), noiselevel=-1)
247 - else:
248 - if restrict_fetch or force_mirror:
249 - # Only fetch from specific mirrors is allowed.
250 - continue
251 - primaryuris = primaryuri_dict.get(myfile)
252 - if primaryuris is None:
253 - primaryuris = []
254 - primaryuri_dict[myfile] = primaryuris
255 - primaryuris.append(myuri)
256 -
257 - # Order primaryuri_dict values to match that in SRC_URI.
258 - for uris in primaryuri_dict.values():
259 - uris.reverse()
260 -
261 - # Prefer thirdpartymirrors over normal mirrors in cases when
262 - # the file does not yet exist on the normal mirrors.
263 - for myfile, uris in thirdpartymirror_uris.items():
264 - primaryuri_dict.setdefault(myfile, []).extend(uris)
265 -
266 - # Now merge primaryuri values into filedict (includes mirrors
267 - # explicitly referenced in SRC_URI).
268 - if "primaryuri" in restrict:
269 - for myfile, uris in filedict.items():
270 - filedict[myfile] = primaryuri_dict.get(myfile, []) + uris
271 - else:
272 - for myfile in filedict:
273 - filedict[myfile] += primaryuri_dict.get(myfile, [])
274 + filedict, primaryuri_dict = _get_uris(
275 + uris=myuris, settings=mysettings,
276 + custom_mirrors=custommirrors, locations=locations)
277
278 can_fetch=True
279
280 --
281 1.8.5.2.8.g0f6c0d1