1 |
The current fetch() function is quite long, which makes it hard to |
2 |
know what I can change without adverse side effects. By pulling this |
3 |
logic out of the main function, we get clearer logic in fetch() and |
4 |
more explicit input for the config extraction. |
5 |
|
6 |
This block was especially complicated, so I also created the helper |
7 |
functions _get_file_uri_tuples and _expand_mirror. I'd prefer if |
8 |
_expand_mirror iterated through URIs instead of (group, URI) tuples, |
9 |
but we need a distinct marker for third-party URIs to build |
10 |
third_party_mirror_uris which is used to build primaryuri_dict which |
11 |
is used way down in fetch(): |
12 |
|
13 |
if checksum_failure_count == \ |
14 |
checksum_failure_primaryuri: |
15 |
# Switch to "primaryuri" mode in order |
16 |
# to increase the probablility of |
17 |
# of success. |
18 |
primaryuris = \ |
19 |
primaryuri_dict.get(myfile) |
20 |
if primaryuris: |
21 |
uri_list.extend( |
22 |
reversed(primaryuris)) |
23 |
|
24 |
I don't know if this is useful enough to motivate the uglier |
25 |
_expandmirror return values, but I'll kick that can down the road for |
26 |
now. |
27 |
--- |
28 |
pym/portage/package/ebuild/fetch.py | 209 ++++++++++++++++++++++-------------- |
29 |
1 file changed, 128 insertions(+), 81 deletions(-) |
30 |
|
31 |
diff --git a/pym/portage/package/ebuild/fetch.py b/pym/portage/package/ebuild/fetch.py |
32 |
index de5bf00..a1940f4 100644 |
33 |
--- a/pym/portage/package/ebuild/fetch.py |
34 |
+++ b/pym/portage/package/ebuild/fetch.py |
35 |
@@ -15,9 +15,9 @@ import sys |
36 |
import tempfile |
37 |
|
38 |
try: |
39 |
- from urllib.parse import urlparse |
40 |
+ from urllib.parse import urlparse, urlunparse |
41 |
except ImportError: |
42 |
- from urlparse import urlparse |
43 |
+ from urlparse import urlparse, urlunparse |
44 |
|
45 |
import portage |
46 |
portage.proxy.lazyimport.lazyimport(globals(), |
47 |
@@ -298,6 +298,129 @@ def _get_fetch_resume_size(settings, default='350K'): |
48 |
return v |
49 |
|
50 |
|
51 |
+def _get_file_uri_tuples(uris): |
52 |
+ """ |
53 |
+ Return a list of (filename, uri) tuples |
54 |
+ """ |
55 |
+ file_uri_tuples = [] |
56 |
+ # Check for 'items' attribute since OrderedDict is not a dict. |
57 |
+ if hasattr(uris, 'items'): |
58 |
+ for filename, uri_set in uris.items(): |
59 |
+ for uri in uri_set: |
60 |
+ file_uri_tuples.append((filename, uri)) |
61 |
+ if not uri_set: |
62 |
+ file_uri_tuples.append((filename, None)) |
63 |
+ else: |
64 |
+ for uri in uris: |
65 |
+ if urlparse(uri).scheme: |
66 |
+ file_uri_tuples.append( |
67 |
+ (os.path.basename(uri), uri)) |
68 |
+ else: |
69 |
+ file_uri_tuples.append( |
70 |
+ (os.path.basename(uri), None)) |
71 |
+ return file_uri_tuples |
72 |
+ |
73 |
+ |
74 |
+def _expand_mirror(uri, custom_mirrors=(), third_party_mirrors=()): |
75 |
+ """ |
76 |
+ Replace the 'mirror://' scheme in the uri |
77 |
+ |
78 |
+ Returns an iterable listing expanded (group, URI) tuples, |
79 |
+ where the group is either 'custom' or 'third-party'. |
80 |
+ """ |
81 |
+ parsed = urlparse(uri) |
82 |
+ mirror = parsed.netloc |
83 |
+ path = parsed.path |
84 |
+ if path: |
85 |
+ # Try user-defined mirrors first |
86 |
+ if mirror in custom_mirrors: |
87 |
+ for cmirr in custom_mirrors[mirror]: |
88 |
+ m_uri = urlparse(cmirr) |
89 |
+ yield ('custom', urlunparse(( |
90 |
+ m_uri.scheme, m_uri.netloc, path) + |
91 |
+ parsed[3:])) |
92 |
+ |
93 |
+ # now try the official mirrors |
94 |
+ if mirror in third_party_mirrors: |
95 |
+ uris = [] |
96 |
+ for locmirr in third_party_mirrors[mirror]: |
97 |
+ m_uri = urlparse(locmirr) |
98 |
+ uris.append(urlunparse(( |
99 |
+ m_uri.scheme, m_uri.netloc, path) + |
100 |
+ parsed[3:])) |
101 |
+ random.shuffle(uris) |
102 |
+ for uri in uris: |
103 |
+ yield ('third-party', uri) |
104 |
+ |
105 |
+ if (not custom_mirrors.get(mirror, []) and |
106 |
+ not third_party_mirrors.get(mirror, [])): |
107 |
+ writemsg( |
108 |
+ _("No known mirror by the name: %s\n") |
109 |
+ % (mirror,)) |
110 |
+ else: |
111 |
+ writemsg(_("Invalid mirror definition in SRC_URI:\n"), |
112 |
+ noiselevel=-1) |
113 |
+ writemsg(" %s\n" % (uri), noiselevel=-1) |
114 |
+ |
115 |
+ |
116 |
+def _get_uris(uris, settings, custom_mirrors=(), locations=()): |
117 |
+ restrict = settings.get("PORTAGE_RESTRICT", "").split() |
118 |
+ restrict_fetch = "fetch" in restrict |
119 |
+ restrict_mirror = "mirror" in restrict or "nomirror" in restrict |
120 |
+ force_mirror = ( |
121 |
+ "force-mirror" in settings.features and |
122 |
+ not restrict_mirror) |
123 |
+ |
124 |
+ third_party_mirrors = settings.thirdpartymirrors() |
125 |
+ third_party_mirror_uris = {} |
126 |
+ filedict = OrderedDict() |
127 |
+ primaryuri_dict = {} |
128 |
+ for filename, uri in _get_file_uri_tuples(uris=uris): |
129 |
+ if filename not in filedict: |
130 |
+ filedict[filename] = [ |
131 |
+ os.path.join(location, 'distfiles', filename) |
132 |
+ for location in locations] |
133 |
+ if uri is None: |
134 |
+ continue |
135 |
+ if uri.startswith('mirror://'): |
136 |
+ uris = _expand_mirror( |
137 |
+ uri=uri, custom_mirrors=custom_mirrors, |
138 |
+ third_party_mirrors=third_party_mirrors) |
139 |
+ filedict[filename].extend(uri for group, uri in uris) |
140 |
+ third_party_mirror_uris.setdefault(filename, []).extend( |
141 |
+ uri for group, uri in uris |
142 |
+ if group == 'third-party') |
143 |
+ else: |
144 |
+ if restrict_fetch or force_mirror: |
145 |
+ # Only fetch from specific mirrors is allowed. |
146 |
+ continue |
147 |
+ primaryuris = primaryuri_dict.get(filename) |
148 |
+ if primaryuris is None: |
149 |
+ primaryuris = [] |
150 |
+ primaryuri_dict[filename] = primaryuris |
151 |
+ primaryuris.append(uri) |
152 |
+ |
153 |
+ # Order primaryuri_dict values to match that in SRC_URI. |
154 |
+ for uris in primaryuri_dict.values(): |
155 |
+ uris.reverse() |
156 |
+ |
157 |
+ # Prefer third_party_mirrors over normal mirrors in cases when |
158 |
+ # the file does not yet exist on the normal mirrors. |
159 |
+ for filename, uris in third_party_mirror_uris.items(): |
160 |
+ primaryuri_dict.setdefault(filename, []).extend(uris) |
161 |
+ |
162 |
+ # Now merge primaryuri values into filedict (includes mirrors |
163 |
+ # explicitly referenced in SRC_URI). |
164 |
+ if "primaryuri" in restrict: |
165 |
+ for filename, uris in filedict.items(): |
166 |
+ filedict[filename] = primaryuri_dict.get(filename, []) + uris |
167 |
+ else: |
168 |
+ for filename in filedict: |
169 |
+ filedict[filename] += primaryuri_dict.get(filename, []) |
170 |
+ |
171 |
+ return filedict, primaryuri_dict |
172 |
+ |
173 |
+ |
174 |
def fetch(myuris, mysettings, listonly=0, fetchonly=0, |
175 |
locks_in_subdir=".locks", use_locks=1, try_mirrors=1, digests=None, |
176 |
allow_missing_digests=True): |
177 |
@@ -329,7 +452,6 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, |
178 |
# couple of checksum failures, to increase the probablility |
179 |
# of success before checksum_failure_max_tries is reached. |
180 |
checksum_failure_primaryuri = 2 |
181 |
- thirdpartymirrors = mysettings.thirdpartymirrors() |
182 |
|
183 |
# In the background parallel-fetch process, it's safe to skip checksum |
184 |
# verification of pre-existing files in $DISTDIR that have the correct |
185 |
@@ -402,7 +524,6 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, |
186 |
del mymirrors[x] |
187 |
|
188 |
restrict_fetch = "fetch" in restrict |
189 |
- force_mirror = "force-mirror" in features and not restrict_mirror |
190 |
custom_local_mirrors = custommirrors.get("local", []) |
191 |
if restrict_fetch: |
192 |
# With fetch restriction, a normal uri may only be fetched from |
193 |
@@ -413,83 +534,9 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, |
194 |
else: |
195 |
locations = mymirrors |
196 |
|
197 |
- file_uri_tuples = [] |
198 |
- # Check for 'items' attribute since OrderedDict is not a dict. |
199 |
- if hasattr(myuris, 'items'): |
200 |
- for myfile, uri_set in myuris.items(): |
201 |
- for myuri in uri_set: |
202 |
- file_uri_tuples.append((myfile, myuri)) |
203 |
- if not uri_set: |
204 |
- file_uri_tuples.append((myfile, None)) |
205 |
- else: |
206 |
- for myuri in myuris: |
207 |
- if urlparse(myuri).scheme: |
208 |
- file_uri_tuples.append((os.path.basename(myuri), myuri)) |
209 |
- else: |
210 |
- file_uri_tuples.append((os.path.basename(myuri), None)) |
211 |
- |
212 |
- filedict = OrderedDict() |
213 |
- primaryuri_dict = {} |
214 |
- thirdpartymirror_uris = {} |
215 |
- for myfile, myuri in file_uri_tuples: |
216 |
- if myfile not in filedict: |
217 |
- filedict[myfile]=[] |
218 |
- for y in range(0,len(locations)): |
219 |
- filedict[myfile].append(locations[y]+"/distfiles/"+myfile) |
220 |
- if myuri is None: |
221 |
- continue |
222 |
- if myuri[:9]=="mirror://": |
223 |
- eidx = myuri.find("/", 9) |
224 |
- if eidx != -1: |
225 |
- mirrorname = myuri[9:eidx] |
226 |
- path = myuri[eidx+1:] |
227 |
- |
228 |
- # Try user-defined mirrors first |
229 |
- if mirrorname in custommirrors: |
230 |
- for cmirr in custommirrors[mirrorname]: |
231 |
- filedict[myfile].append( |
232 |
- cmirr.rstrip("/") + "/" + path) |
233 |
- |
234 |
- # now try the official mirrors |
235 |
- if mirrorname in thirdpartymirrors: |
236 |
- uris = [locmirr.rstrip("/") + "/" + path \ |
237 |
- for locmirr in thirdpartymirrors[mirrorname]] |
238 |
- random.shuffle(uris) |
239 |
- filedict[myfile].extend(uris) |
240 |
- thirdpartymirror_uris.setdefault(myfile, []).extend(uris) |
241 |
- |
242 |
- if not filedict[myfile]: |
243 |
- writemsg(_("No known mirror by the name: %s\n") % (mirrorname)) |
244 |
- else: |
245 |
- writemsg(_("Invalid mirror definition in SRC_URI:\n"), noiselevel=-1) |
246 |
- writemsg(" %s\n" % (myuri), noiselevel=-1) |
247 |
- else: |
248 |
- if restrict_fetch or force_mirror: |
249 |
- # Only fetch from specific mirrors is allowed. |
250 |
- continue |
251 |
- primaryuris = primaryuri_dict.get(myfile) |
252 |
- if primaryuris is None: |
253 |
- primaryuris = [] |
254 |
- primaryuri_dict[myfile] = primaryuris |
255 |
- primaryuris.append(myuri) |
256 |
- |
257 |
- # Order primaryuri_dict values to match that in SRC_URI. |
258 |
- for uris in primaryuri_dict.values(): |
259 |
- uris.reverse() |
260 |
- |
261 |
- # Prefer thirdpartymirrors over normal mirrors in cases when |
262 |
- # the file does not yet exist on the normal mirrors. |
263 |
- for myfile, uris in thirdpartymirror_uris.items(): |
264 |
- primaryuri_dict.setdefault(myfile, []).extend(uris) |
265 |
- |
266 |
- # Now merge primaryuri values into filedict (includes mirrors |
267 |
- # explicitly referenced in SRC_URI). |
268 |
- if "primaryuri" in restrict: |
269 |
- for myfile, uris in filedict.items(): |
270 |
- filedict[myfile] = primaryuri_dict.get(myfile, []) + uris |
271 |
- else: |
272 |
- for myfile in filedict: |
273 |
- filedict[myfile] += primaryuri_dict.get(myfile, []) |
274 |
+ filedict, primaryuri_dict = _get_uris( |
275 |
+ uris=myuris, settings=mysettings, |
276 |
+ custom_mirrors=custommirrors, locations=locations) |
277 |
|
278 |
can_fetch=True |
279 |
|
280 |
-- |
281 |
1.8.5.2.8.g0f6c0d1 |