1 |
The current fetch() function is quite long, which makes it hard to |
2 |
know what I can change without adverse side effects. By pulling this |
3 |
logic out of the main function, we get clearer logic in fetch() and |
4 |
more explicit input for the config extraction. |
5 |
|
6 |
This block was especially complicated, so I also created the helper |
7 |
functions _get_file_uri_tuples and _expand_mirror. I'd prefer if |
8 |
_expand_mirror iterated through URIs instead of (group, URI) tuples, |
9 |
but we need a distinct marker for third-party URIs to build |
10 |
third_party_mirror_uris which is used to build primaryuri_dict which |
11 |
is used way down in fetch(): |
12 |
|
13 |
if checksum_failure_count == \ |
14 |
checksum_failure_primaryuri: |
15 |
# Switch to "primaryuri" mode in order |
16 |
# to increase the probablility of |
17 |
# of success. |
18 |
primaryuris = \ |
19 |
primaryuri_dict.get(myfile) |
20 |
if primaryuris: |
21 |
uri_list.extend( |
22 |
reversed(primaryuris)) |
23 |
|
24 |
I don't know if this is useful enough to motivate the uglier |
25 |
_expandmirror return values, but I'll kick that can down the road for |
26 |
now. |
27 |
--- |
28 |
pym/portage/package/ebuild/fetch.py | 197 +++++++++++++++++++++--------------- |
29 |
1 file changed, 117 insertions(+), 80 deletions(-) |
30 |
|
31 |
diff --git a/pym/portage/package/ebuild/fetch.py b/pym/portage/package/ebuild/fetch.py |
32 |
index bd572fa..a617945 100644 |
33 |
--- a/pym/portage/package/ebuild/fetch.py |
34 |
+++ b/pym/portage/package/ebuild/fetch.py |
35 |
@@ -15,9 +15,9 @@ import sys |
36 |
import tempfile |
37 |
|
38 |
try: |
39 |
- from urllib.parse import urlparse |
40 |
+ from urllib.parse import urlparse, urlunparse |
41 |
except ImportError: |
42 |
- from urlparse import urlparse |
43 |
+ from urlparse import urlparse, urlunparse |
44 |
|
45 |
import portage |
46 |
portage.proxy.lazyimport.lazyimport(globals(), |
47 |
@@ -297,6 +297,118 @@ def _get_fetch_resume_size(settings, default='350K'): |
48 |
return v |
49 |
|
50 |
|
51 |
+def _get_file_uri_tuples(uris): |
52 |
+ """Return a list of (filename, uri) tuples |
53 |
+ """ |
54 |
+ file_uri_tuples = [] |
55 |
+ # Check for 'items' attribute since OrderedDict is not a dict. |
56 |
+ if hasattr(uris, 'items'): |
57 |
+ for filename, uri_set in uris.items(): |
58 |
+ for uri in uri_set: |
59 |
+ file_uri_tuples.append((filename, uri)) |
60 |
+ if not uri_set: |
61 |
+ file_uri_tuples.append((filename, None)) |
62 |
+ else: |
63 |
+ for uri in uris: |
64 |
+ if urlparse(uri).scheme: |
65 |
+ file_uri_tuples.append( |
66 |
+ (os.path.basename(myuri), myuri)) |
67 |
+ else: |
68 |
+ file_uri_tuples.append( |
69 |
+ (os.path.basename(myuri), None)) |
70 |
+ return file_uri_tuples |
71 |
+ |
72 |
+ |
73 |
+def _expand_mirror(uri, custom_mirrors=(), third_party_mirrors=()): |
74 |
+ """Replace the 'mirror://' scheme in the uri |
75 |
+ |
76 |
+ Returns an iterable listing expanded (group, URI) tuples, |
77 |
+ where the group is either 'custom' or 'third-party'. |
78 |
+ """ |
79 |
+ parsed = urlparse(uri) |
80 |
+ mirror = parsed.netloc |
81 |
+ path = parsed.path |
82 |
+ if path: |
83 |
+ # Try user-defined mirrors first |
84 |
+ if mirror in custom_mirrors: |
85 |
+ for cmirr in custom_mirrors[mirror]: |
86 |
+ m_uri = urlparse(cmirr) |
87 |
+ yield ('custom', urlunparse(( |
88 |
+ cmirr.scheme, cmirr.netloc, path) + |
89 |
+ parsed[3:])) |
90 |
+ |
91 |
+ # now try the official mirrors |
92 |
+ if mirror in third_party_mirrors: |
93 |
+ uris = [] |
94 |
+ for locmirr in third_party_mirrors[mirror]: |
95 |
+ m_uri = urlparse(cmirr) |
96 |
+ uris.append(urlunparse(( |
97 |
+ cmirr.scheme, cmirr.netloc, path) + |
98 |
+ parsed[3:])) |
99 |
+ random.shuffle(uris) |
100 |
+ for uri in uris: |
101 |
+ yield ('third-party', uri) |
102 |
+ else: |
103 |
+ writemsg(_("Invalid mirror definition in SRC_URI:\n"), |
104 |
+ noiselevel=-1) |
105 |
+ writemsg(" %s\n" % (uri), noiselevel=-1) |
106 |
+ |
107 |
+ |
108 |
+def _get_uris(uris, settings, custom_mirrors=(), locations=()): |
109 |
+ third_party_mirrors = settings.thirdpartymirrors() |
110 |
+ third_party_mirror_uris = {} |
111 |
+ filedict = OrderedDict() |
112 |
+ primaryuri_dict = {} |
113 |
+ for filename, uri in _get_file_uri_tuples(uris=uris): |
114 |
+ if filename not in filedict: |
115 |
+ filedict[filename] = [ |
116 |
+ os.path.join(location, 'distfiles', filename) |
117 |
+ for location in locations] |
118 |
+ if uri is None: |
119 |
+ continue |
120 |
+ if uri.startswith('mirror://'): |
121 |
+ uris = _expand_mirror( |
122 |
+ uri=uri, custom_mirrors=custom_mirrors, |
123 |
+ third_party_mirrors=third_party_mirrors) |
124 |
+ filedict[filename].extend(uri for group, uri in uris) |
125 |
+ third_party_mirror_uris.setdefault(filename, []).extend( |
126 |
+ uri for group, uri in uris |
127 |
+ if group == 'third-party') |
128 |
+ if not filedict[filename]: |
129 |
+ writemsg( |
130 |
+ _("No known mirror by the name: %s\n") |
131 |
+ % (mirror,)) |
132 |
+ else: |
133 |
+ if restrict_fetch or force_mirror: |
134 |
+ # Only fetch from specific mirrors is allowed. |
135 |
+ continue |
136 |
+ primaryuris = primaryuri_dict.get(filename) |
137 |
+ if primaryuris is None: |
138 |
+ primaryuris = [] |
139 |
+ primaryuri_dict[filename] = primaryuris |
140 |
+ primaryuris.append(uri) |
141 |
+ |
142 |
+ # Order primaryuri_dict values to match that in SRC_URI. |
143 |
+ for uris in primaryuri_dict.values(): |
144 |
+ uris.reverse() |
145 |
+ |
146 |
+ # Prefer third_party_mirrors over normal mirrors in cases when |
147 |
+ # the file does not yet exist on the normal mirrors. |
148 |
+ for filename, uris in third_party_mirror_uris.items(): |
149 |
+ primaryuri_dict.setdefault(filename, []).extend(uris) |
150 |
+ |
151 |
+ # Now merge primaryuri values into filedict (includes mirrors |
152 |
+ # explicitly referenced in SRC_URI). |
153 |
+ if "primaryuri" in restrict: |
154 |
+ for filename, uris in filedict.items(): |
155 |
+ filedict[filename] = primaryuri_dict.get(filename, []) + uris |
156 |
+ else: |
157 |
+ for filename in filedict: |
158 |
+ filedict[filename] += primaryuri_dict.get(filename, []) |
159 |
+ |
160 |
+ return filedict, primaryuri_dict |
161 |
+ |
162 |
+ |
163 |
def fetch(myuris, mysettings, listonly=0, fetchonly=0, |
164 |
locks_in_subdir=".locks", use_locks=1, try_mirrors=1, digests=None, |
165 |
allow_missing_digests=True): |
166 |
@@ -328,7 +440,6 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, |
167 |
# couple of checksum failures, to increase the probablility |
168 |
# of success before checksum_failure_max_tries is reached. |
169 |
checksum_failure_primaryuri = 2 |
170 |
- thirdpartymirrors = mysettings.thirdpartymirrors() |
171 |
|
172 |
# In the background parallel-fetch process, it's safe to skip checksum |
173 |
# verification of pre-existing files in $DISTDIR that have the correct |
174 |
@@ -412,83 +523,9 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, |
175 |
else: |
176 |
locations = mymirrors |
177 |
|
178 |
- file_uri_tuples = [] |
179 |
- # Check for 'items' attribute since OrderedDict is not a dict. |
180 |
- if hasattr(myuris, 'items'): |
181 |
- for myfile, uri_set in myuris.items(): |
182 |
- for myuri in uri_set: |
183 |
- file_uri_tuples.append((myfile, myuri)) |
184 |
- if not uri_set: |
185 |
- file_uri_tuples.append((myfile, None)) |
186 |
- else: |
187 |
- for myuri in myuris: |
188 |
- if urlparse(myuri).scheme: |
189 |
- file_uri_tuples.append((os.path.basename(myuri), myuri)) |
190 |
- else: |
191 |
- file_uri_tuples.append((os.path.basename(myuri), None)) |
192 |
- |
193 |
- filedict = OrderedDict() |
194 |
- primaryuri_dict = {} |
195 |
- thirdpartymirror_uris = {} |
196 |
- for myfile, myuri in file_uri_tuples: |
197 |
- if myfile not in filedict: |
198 |
- filedict[myfile]=[] |
199 |
- for y in range(0,len(locations)): |
200 |
- filedict[myfile].append(locations[y]+"/distfiles/"+myfile) |
201 |
- if myuri is None: |
202 |
- continue |
203 |
- if myuri[:9]=="mirror://": |
204 |
- eidx = myuri.find("/", 9) |
205 |
- if eidx != -1: |
206 |
- mirrorname = myuri[9:eidx] |
207 |
- path = myuri[eidx+1:] |
208 |
- |
209 |
- # Try user-defined mirrors first |
210 |
- if mirrorname in custommirrors: |
211 |
- for cmirr in custommirrors[mirrorname]: |
212 |
- filedict[myfile].append( |
213 |
- cmirr.rstrip("/") + "/" + path) |
214 |
- |
215 |
- # now try the official mirrors |
216 |
- if mirrorname in thirdpartymirrors: |
217 |
- uris = [locmirr.rstrip("/") + "/" + path \ |
218 |
- for locmirr in thirdpartymirrors[mirrorname]] |
219 |
- random.shuffle(uris) |
220 |
- filedict[myfile].extend(uris) |
221 |
- thirdpartymirror_uris.setdefault(myfile, []).extend(uris) |
222 |
- |
223 |
- if not filedict[myfile]: |
224 |
- writemsg(_("No known mirror by the name: %s\n") % (mirrorname)) |
225 |
- else: |
226 |
- writemsg(_("Invalid mirror definition in SRC_URI:\n"), noiselevel=-1) |
227 |
- writemsg(" %s\n" % (myuri), noiselevel=-1) |
228 |
- else: |
229 |
- if restrict_fetch or force_mirror: |
230 |
- # Only fetch from specific mirrors is allowed. |
231 |
- continue |
232 |
- primaryuris = primaryuri_dict.get(myfile) |
233 |
- if primaryuris is None: |
234 |
- primaryuris = [] |
235 |
- primaryuri_dict[myfile] = primaryuris |
236 |
- primaryuris.append(myuri) |
237 |
- |
238 |
- # Order primaryuri_dict values to match that in SRC_URI. |
239 |
- for uris in primaryuri_dict.values(): |
240 |
- uris.reverse() |
241 |
- |
242 |
- # Prefer thirdpartymirrors over normal mirrors in cases when |
243 |
- # the file does not yet exist on the normal mirrors. |
244 |
- for myfile, uris in thirdpartymirror_uris.items(): |
245 |
- primaryuri_dict.setdefault(myfile, []).extend(uris) |
246 |
- |
247 |
- # Now merge primaryuri values into filedict (includes mirrors |
248 |
- # explicitly referenced in SRC_URI). |
249 |
- if "primaryuri" in restrict: |
250 |
- for myfile, uris in filedict.items(): |
251 |
- filedict[myfile] = primaryuri_dict.get(myfile, []) + uris |
252 |
- else: |
253 |
- for myfile in filedict: |
254 |
- filedict[myfile] += primaryuri_dict.get(myfile, []) |
255 |
+ filedict, primaryuri_dict = _get_uris( |
256 |
+ uris=myuris, settings=mysettings, |
257 |
+ custom_mirrors=custom_mirrors, locations=locations) |
258 |
|
259 |
can_fetch=True |
260 |
|
261 |
-- |
262 |
1.8.5.2.8.g0f6c0d1 |