1 |
Add a support for the subset of GLEP 75 needed by Gentoo Infra. This |
2 |
includes fetching and parsing layout.conf, and support for flat layout |
3 |
and filename-hash layout with cutoffs being multiplies of 4. |
4 |
|
5 |
Bug: https://bugs.gentoo.org/646898 |
6 |
Signed-off-by: Michał Górny <mgorny@g.o> |
7 |
--- |
8 |
lib/portage/package/ebuild/fetch.py | 139 +++++++++++++++++++++++++++- |
9 |
1 file changed, 135 insertions(+), 4 deletions(-) |
10 |
|
11 |
Changes in v2: switched to a more classy layout to make the code |
12 |
reusable in emirrordist. |
13 |
|
14 |
diff --git a/lib/portage/package/ebuild/fetch.py b/lib/portage/package/ebuild/fetch.py |
15 |
index 227bf45ae..18e3d390a 100644 |
16 |
--- a/lib/portage/package/ebuild/fetch.py |
17 |
+++ b/lib/portage/package/ebuild/fetch.py |
18 |
@@ -7,12 +7,15 @@ __all__ = ['fetch'] |
19 |
|
20 |
import errno |
21 |
import io |
22 |
+import itertools |
23 |
+import json |
24 |
import logging |
25 |
import random |
26 |
import re |
27 |
import stat |
28 |
import sys |
29 |
import tempfile |
30 |
+import time |
31 |
|
32 |
from collections import OrderedDict |
33 |
|
34 |
@@ -27,14 +30,17 @@ portage.proxy.lazyimport.lazyimport(globals(), |
35 |
'portage.package.ebuild.doebuild:doebuild_environment,' + \ |
36 |
'_doebuild_spawn', |
37 |
'portage.package.ebuild.prepare_build_dirs:prepare_build_dirs', |
38 |
+ 'portage.util.configparser:SafeConfigParser,read_configs,NoOptionError', |
39 |
+ 'portage.util._urlopen:urlopen', |
40 |
) |
41 |
|
42 |
from portage import os, selinux, shutil, _encodings, \ |
43 |
_movefile, _shell_quote, _unicode_encode |
44 |
from portage.checksum import (get_valid_checksum_keys, perform_md5, verify_all, |
45 |
- _filter_unaccelarated_hashes, _hash_filter, _apply_hash_filter) |
46 |
+ _filter_unaccelarated_hashes, _hash_filter, _apply_hash_filter, |
47 |
+ checksum_str) |
48 |
from portage.const import BASH_BINARY, CUSTOM_MIRRORS_FILE, \ |
49 |
- GLOBAL_CONFIG_PATH |
50 |
+ GLOBAL_CONFIG_PATH, CACHE_PATH |
51 |
from portage.data import portage_gid, portage_uid, secpass, userpriv_groups |
52 |
from portage.exception import FileNotFound, OperationNotPermitted, \ |
53 |
PortageException, TryAgain |
54 |
@@ -253,6 +259,130 @@ _size_suffix_map = { |
55 |
'Y' : 80, |
56 |
} |
57 |
|
58 |
+ |
59 |
+class FlatLayout(object): |
60 |
+ def get_path(self, filename): |
61 |
+ return filename |
62 |
+ |
63 |
+ |
64 |
+class FilenameHashLayout(object): |
65 |
+ def __init__(self, algo, cutoffs): |
66 |
+ self.algo = algo |
67 |
+ self.cutoffs = [int(x) for x in cutoffs.split(':')] |
68 |
+ |
69 |
+ def get_path(self, filename): |
70 |
+ fnhash = checksum_str(filename.encode('utf8'), self.algo) |
71 |
+ ret = '' |
72 |
+ for c in self.cutoffs: |
73 |
+ assert c % 4 == 0 |
74 |
+ c = c // 4 |
75 |
+ ret += fnhash[:c] + '/' |
76 |
+ fnhash = fnhash[c:] |
77 |
+ return ret + filename |
78 |
+ |
79 |
+ |
80 |
+class MirrorLayoutConfig(object): |
81 |
+ """ |
82 |
+ Class to read layout.conf from a mirror. |
83 |
+ """ |
84 |
+ |
85 |
+ def __init__(self): |
86 |
+ self.structure = () |
87 |
+ |
88 |
+ def read_from_file(self, f): |
89 |
+ cp = SafeConfigParser() |
90 |
+ read_configs(cp, [f]) |
91 |
+ vals = [] |
92 |
+ for i in itertools.count(): |
93 |
+ try: |
94 |
+ vals.append(tuple(cp.get('structure', '%d' % i).split())) |
95 |
+ except NoOptionError: |
96 |
+ break |
97 |
+ self.structure = tuple(vals) |
98 |
+ |
99 |
+ def serialize(self): |
100 |
+ return self.structure |
101 |
+ |
102 |
+ def deserialize(self, data): |
103 |
+ self.structure = data |
104 |
+ |
105 |
+ @staticmethod |
106 |
+ def validate_structure(val): |
107 |
+ if val == ('flat',): |
108 |
+ return True |
109 |
+ if val[0] == 'filename-hash' and len(val) == 3: |
110 |
+ if val[1] not in get_valid_checksum_keys(): |
111 |
+ return False |
112 |
+ # validate cutoffs |
113 |
+ for c in val[2].split(':'): |
114 |
+ try: |
115 |
+ c = int(c) |
116 |
+ except ValueError: |
117 |
+ break |
118 |
+ else: |
119 |
+ if c % 4 != 0: |
120 |
+ break |
121 |
+ else: |
122 |
+ return True |
123 |
+ return False |
124 |
+ return False |
125 |
+ |
126 |
+ def get_best_supported_layout(self): |
127 |
+ for val in self.structure: |
128 |
+ if self.validate_structure(val): |
129 |
+ if val[0] == 'flat': |
130 |
+ return FlatLayout() |
131 |
+ elif val[0] == 'filename-hash': |
132 |
+ return FilenameHashLayout(val[1], val[2]) |
133 |
+ else: |
134 |
+ # fallback |
135 |
+ return FlatLayout() |
136 |
+ |
137 |
+ |
138 |
+def get_mirror_url(mirror_url, filename, eroot): |
139 |
+ """ |
140 |
+ Get correct fetch URL for a given file, accounting for mirror |
141 |
+ layout configuration. |
142 |
+ |
143 |
+ @param mirror_url: Base URL to the mirror (without '/distfiles') |
144 |
+ @param filename: Filename to fetch |
145 |
+ @param eroot: EROOT to use for the cache file |
146 |
+ @return: Full URL to fetch |
147 |
+ """ |
148 |
+ |
149 |
+ mirror_conf = MirrorLayoutConfig() |
150 |
+ |
151 |
+ cache_file = os.path.join(eroot, CACHE_PATH, 'mirror-metadata.json') |
152 |
+ try: |
153 |
+ with open(cache_file, 'r') as f: |
154 |
+ cache = json.load(f) |
155 |
+ except (IOError, ValueError): |
156 |
+ cache = {} |
157 |
+ |
158 |
+ ts, data = cache.get(mirror_url, (0, None)) |
159 |
+ # refresh at least daily |
160 |
+ if ts >= time.time() - 86400: |
161 |
+ mirror_conf.deserialize(data) |
162 |
+ else: |
163 |
+ try: |
164 |
+ f = urlopen(mirror_url + '/distfiles/layout.conf') |
165 |
+ try: |
166 |
+ data = io.StringIO(f.read().decode('utf8')) |
167 |
+ finally: |
168 |
+ f.close() |
169 |
+ |
170 |
+ mirror_conf.read_from_file(data) |
171 |
+ except IOError: |
172 |
+ pass |
173 |
+ |
174 |
+ cache[mirror_url] = (time.time(), mirror_conf.serialize()) |
175 |
+ with open(cache_file, 'w') as f: |
176 |
+ json.dump(cache, f) |
177 |
+ |
178 |
+ return (mirror_url + "/distfiles/" + |
179 |
+ mirror_conf.get_best_supported_layout().get_path(filename)) |
180 |
+ |
181 |
+ |
182 |
def fetch(myuris, mysettings, listonly=0, fetchonly=0, |
183 |
locks_in_subdir=".locks", use_locks=1, try_mirrors=1, digests=None, |
184 |
allow_missing_digests=True): |
185 |
@@ -434,8 +564,9 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, |
186 |
for myfile, myuri in file_uri_tuples: |
187 |
if myfile not in filedict: |
188 |
filedict[myfile]=[] |
189 |
- for y in range(0,len(locations)): |
190 |
- filedict[myfile].append(locations[y]+"/distfiles/"+myfile) |
191 |
+ for l in locations: |
192 |
+ filedict[myfile].append(get_mirror_url(l, myfile, |
193 |
+ mysettings["EROOT"])) |
194 |
if myuri is None: |
195 |
continue |
196 |
if myuri[:9]=="mirror://": |
197 |
-- |
198 |
2.23.0 |