1 |
Add a support for the subset of GLEP 75 needed by Gentoo Infra. This |
2 |
includes fetching and parsing layout.conf, and support for flat layout |
3 |
and filename-hash layout with cutoffs being multiplies of 4. |
4 |
|
5 |
Bug: https://bugs.gentoo.org/646898 |
6 |
Signed-off-by: Michał Górny <mgorny@g.o> |
7 |
--- |
8 |
lib/portage/package/ebuild/fetch.py | 113 +++++++++++++++++++++++++++- |
9 |
1 file changed, 109 insertions(+), 4 deletions(-) |
10 |
|
11 |
diff --git a/lib/portage/package/ebuild/fetch.py b/lib/portage/package/ebuild/fetch.py |
12 |
index 227bf45ae..692efcc01 100644 |
13 |
--- a/lib/portage/package/ebuild/fetch.py |
14 |
+++ b/lib/portage/package/ebuild/fetch.py |
15 |
@@ -7,12 +7,15 @@ __all__ = ['fetch'] |
16 |
|
17 |
import errno |
18 |
import io |
19 |
+import itertools |
20 |
+import json |
21 |
import logging |
22 |
import random |
23 |
import re |
24 |
import stat |
25 |
import sys |
26 |
import tempfile |
27 |
+import time |
28 |
|
29 |
from collections import OrderedDict |
30 |
|
31 |
@@ -27,14 +30,17 @@ portage.proxy.lazyimport.lazyimport(globals(), |
32 |
'portage.package.ebuild.doebuild:doebuild_environment,' + \ |
33 |
'_doebuild_spawn', |
34 |
'portage.package.ebuild.prepare_build_dirs:prepare_build_dirs', |
35 |
+ 'portage.util.configparser:SafeConfigParser,read_configs,NoOptionError', |
36 |
+ 'portage.util._urlopen:urlopen', |
37 |
) |
38 |
|
39 |
from portage import os, selinux, shutil, _encodings, \ |
40 |
_movefile, _shell_quote, _unicode_encode |
41 |
from portage.checksum import (get_valid_checksum_keys, perform_md5, verify_all, |
42 |
- _filter_unaccelarated_hashes, _hash_filter, _apply_hash_filter) |
43 |
+ _filter_unaccelarated_hashes, _hash_filter, _apply_hash_filter, |
44 |
+ checksum_str) |
45 |
from portage.const import BASH_BINARY, CUSTOM_MIRRORS_FILE, \ |
46 |
- GLOBAL_CONFIG_PATH |
47 |
+ GLOBAL_CONFIG_PATH, CACHE_PATH |
48 |
from portage.data import portage_gid, portage_uid, secpass, userpriv_groups |
49 |
from portage.exception import FileNotFound, OperationNotPermitted, \ |
50 |
PortageException, TryAgain |
51 |
@@ -253,6 +259,104 @@ _size_suffix_map = { |
52 |
'Y' : 80, |
53 |
} |
54 |
|
55 |
+ |
56 |
+def filename_hash_path(filename, algo, cutoffs): |
57 |
+ """ |
58 |
+ Get directory path for filename in filename-hash mirror structure. |
59 |
+ |
60 |
+ @param filename: Filename to fetch |
61 |
+ @param algo: Hash algorithm |
62 |
+ @param cutoffs: Cutoff values (n:n...) |
63 |
+ @return: Directory path |
64 |
+ """ |
65 |
+ |
66 |
+ fnhash = checksum_str(filename.encode('utf8'), algo) |
67 |
+ ret = '' |
68 |
+ for c in cutoffs.split(':'): |
69 |
+ c = int(c) // 4 |
70 |
+ ret += fnhash[:c] + '/' |
71 |
+ fnhash = fnhash[c:] |
72 |
+ return ret |
73 |
+ |
74 |
+ |
75 |
+def get_mirror_url(mirror_url, filename, eroot): |
76 |
+ """ |
77 |
+ Get correct fetch URL for a given file, accounting for mirror |
78 |
+ layout configuration. |
79 |
+ |
80 |
+ @param mirror_url: Base URL to the mirror (without '/distfiles') |
81 |
+ @param filename: Filename to fetch |
82 |
+ @param eroot: EROOT to use for the cache file |
83 |
+ @return: Full URL to fetch |
84 |
+ """ |
85 |
+ |
86 |
+ cache_file = os.path.join(eroot, CACHE_PATH, 'mirror-metadata.json') |
87 |
+ try: |
88 |
+ with open(cache_file, 'r') as f: |
89 |
+ cache = json.load(f) |
90 |
+ except (IOError, ValueError): |
91 |
+ cache = {} |
92 |
+ |
93 |
+ ts, layout = cache.get(mirror_url, (0, None)) |
94 |
+ # refresh at least daily |
95 |
+ if ts < time.time() - 86400: |
96 |
+ # the default |
97 |
+ layout = ('flat',) |
98 |
+ |
99 |
+ try: |
100 |
+ f = urlopen(mirror_url + '/distfiles/layout.conf') |
101 |
+ try: |
102 |
+ data = io.StringIO(f.read().decode('utf8')) |
103 |
+ finally: |
104 |
+ f.close() |
105 |
+ cp = SafeConfigParser() |
106 |
+ read_configs(cp, [data]) |
107 |
+ |
108 |
+ for i in itertools.count(): |
109 |
+ try: |
110 |
+ val = tuple(cp.get('structure', '%d' % i).split()) |
111 |
+ if val == ('flat',): |
112 |
+ pass |
113 |
+ elif val[0] == 'filename-hash' and len(val) == 3: |
114 |
+ if val[1] not in get_valid_checksum_keys(): |
115 |
+ continue |
116 |
+ # validate cutoffs |
117 |
+ cutoffs_good = False |
118 |
+ for c in val[2].split(':'): |
119 |
+ try: |
120 |
+ c = int(c) |
121 |
+ except ValueError: |
122 |
+ break |
123 |
+ else: |
124 |
+ if c % 4 != 0: |
125 |
+ break |
126 |
+ else: |
127 |
+ cutoffs_good = True |
128 |
+ if not cutoffs_good: |
129 |
+ continue |
130 |
+ else: |
131 |
+ # (skip unsupported variant) |
132 |
+ continue |
133 |
+ layout = val |
134 |
+ break |
135 |
+ except NoOptionError: |
136 |
+ break |
137 |
+ except IOError: |
138 |
+ pass |
139 |
+ |
140 |
+ cache[mirror_url] = (time.time(), layout) |
141 |
+ with open(cache_file, 'w') as f: |
142 |
+ json.dump(cache, f) |
143 |
+ |
144 |
+ if layout[0] == 'flat': |
145 |
+ return mirror_url + "/distfiles/" + filename |
146 |
+ elif layout[0] == 'filename-hash': |
147 |
+ return (mirror_url + "/distfiles/" + |
148 |
+ filename_hash_path(filename, *layout[1:]) + filename) |
149 |
+ else: |
150 |
+ raise AssertionError("get_mirror_url() got unknown layout type") |
151 |
+ |
152 |
+ |
153 |
def fetch(myuris, mysettings, listonly=0, fetchonly=0, |
154 |
locks_in_subdir=".locks", use_locks=1, try_mirrors=1, digests=None, |
155 |
allow_missing_digests=True): |
156 |
@@ -434,8 +538,9 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, |
157 |
for myfile, myuri in file_uri_tuples: |
158 |
if myfile not in filedict: |
159 |
filedict[myfile]=[] |
160 |
- for y in range(0,len(locations)): |
161 |
- filedict[myfile].append(locations[y]+"/distfiles/"+myfile) |
162 |
+ for l in locations: |
163 |
+ filedict[myfile].append(get_mirror_url(l, myfile, |
164 |
+ mysettings["EROOT"])) |
165 |
if myuri is None: |
166 |
continue |
167 |
if myuri[:9]=="mirror://": |
168 |
-- |
169 |
2.23.0 |