Gentoo Archives: gentoo-portage-dev

From: "Michał Górny" <mgorny@g.o>
To: gentoo-portage-dev@l.g.o
Cc: "Michał Górny" <mgorny@g.o>
Subject: [gentoo-portage-dev] [PATCH] fetch: Support GLEP 75 mirror structure
Date: Thu, 03 Oct 2019 14:52:16
Message-Id: 20191003145136.4613-1-mgorny@gentoo.org
1 Add a support for the subset of GLEP 75 needed by Gentoo Infra. This
2 includes fetching and parsing layout.conf, and support for flat layout
3 and filename-hash layout with cutoffs being multiplies of 4.
4
5 Bug: https://bugs.gentoo.org/646898
6 Signed-off-by: Michał Górny <mgorny@g.o>
7 ---
8 lib/portage/package/ebuild/fetch.py | 113 +++++++++++++++++++++++++++-
9 1 file changed, 109 insertions(+), 4 deletions(-)
10
11 diff --git a/lib/portage/package/ebuild/fetch.py b/lib/portage/package/ebuild/fetch.py
12 index 227bf45ae..692efcc01 100644
13 --- a/lib/portage/package/ebuild/fetch.py
14 +++ b/lib/portage/package/ebuild/fetch.py
15 @@ -7,12 +7,15 @@ __all__ = ['fetch']
16
17 import errno
18 import io
19 +import itertools
20 +import json
21 import logging
22 import random
23 import re
24 import stat
25 import sys
26 import tempfile
27 +import time
28
29 from collections import OrderedDict
30
31 @@ -27,14 +30,17 @@ portage.proxy.lazyimport.lazyimport(globals(),
32 'portage.package.ebuild.doebuild:doebuild_environment,' + \
33 '_doebuild_spawn',
34 'portage.package.ebuild.prepare_build_dirs:prepare_build_dirs',
35 + 'portage.util.configparser:SafeConfigParser,read_configs,NoOptionError',
36 + 'portage.util._urlopen:urlopen',
37 )
38
39 from portage import os, selinux, shutil, _encodings, \
40 _movefile, _shell_quote, _unicode_encode
41 from portage.checksum import (get_valid_checksum_keys, perform_md5, verify_all,
42 - _filter_unaccelarated_hashes, _hash_filter, _apply_hash_filter)
43 + _filter_unaccelarated_hashes, _hash_filter, _apply_hash_filter,
44 + checksum_str)
45 from portage.const import BASH_BINARY, CUSTOM_MIRRORS_FILE, \
46 - GLOBAL_CONFIG_PATH
47 + GLOBAL_CONFIG_PATH, CACHE_PATH
48 from portage.data import portage_gid, portage_uid, secpass, userpriv_groups
49 from portage.exception import FileNotFound, OperationNotPermitted, \
50 PortageException, TryAgain
51 @@ -253,6 +259,104 @@ _size_suffix_map = {
52 'Y' : 80,
53 }
54
55 +
56 +def filename_hash_path(filename, algo, cutoffs):
57 + """
58 + Get directory path for filename in filename-hash mirror structure.
59 +
60 + @param filename: Filename to fetch
61 + @param algo: Hash algorithm
62 + @param cutoffs: Cutoff values (n:n...)
63 + @return: Directory path
64 + """
65 +
66 + fnhash = checksum_str(filename.encode('utf8'), algo)
67 + ret = ''
68 + for c in cutoffs.split(':'):
69 + c = int(c) // 4
70 + ret += fnhash[:c] + '/'
71 + fnhash = fnhash[c:]
72 + return ret
73 +
74 +
75 +def get_mirror_url(mirror_url, filename, eroot):
76 + """
77 + Get correct fetch URL for a given file, accounting for mirror
78 + layout configuration.
79 +
80 + @param mirror_url: Base URL to the mirror (without '/distfiles')
81 + @param filename: Filename to fetch
82 + @param eroot: EROOT to use for the cache file
83 + @return: Full URL to fetch
84 + """
85 +
86 + cache_file = os.path.join(eroot, CACHE_PATH, 'mirror-metadata.json')
87 + try:
88 + with open(cache_file, 'r') as f:
89 + cache = json.load(f)
90 + except (IOError, ValueError):
91 + cache = {}
92 +
93 + ts, layout = cache.get(mirror_url, (0, None))
94 + # refresh at least daily
95 + if ts < time.time() - 86400:
96 + # the default
97 + layout = ('flat',)
98 +
99 + try:
100 + f = urlopen(mirror_url + '/distfiles/layout.conf')
101 + try:
102 + data = io.StringIO(f.read().decode('utf8'))
103 + finally:
104 + f.close()
105 + cp = SafeConfigParser()
106 + read_configs(cp, [data])
107 +
108 + for i in itertools.count():
109 + try:
110 + val = tuple(cp.get('structure', '%d' % i).split())
111 + if val == ('flat',):
112 + pass
113 + elif val[0] == 'filename-hash' and len(val) == 3:
114 + if val[1] not in get_valid_checksum_keys():
115 + continue
116 + # validate cutoffs
117 + cutoffs_good = False
118 + for c in val[2].split(':'):
119 + try:
120 + c = int(c)
121 + except ValueError:
122 + break
123 + else:
124 + if c % 4 != 0:
125 + break
126 + else:
127 + cutoffs_good = True
128 + if not cutoffs_good:
129 + continue
130 + else:
131 + # (skip unsupported variant)
132 + continue
133 + layout = val
134 + break
135 + except NoOptionError:
136 + break
137 + except IOError:
138 + pass
139 +
140 + cache[mirror_url] = (time.time(), layout)
141 + with open(cache_file, 'w') as f:
142 + json.dump(cache, f)
143 +
144 + if layout[0] == 'flat':
145 + return mirror_url + "/distfiles/" + filename
146 + elif layout[0] == 'filename-hash':
147 + return (mirror_url + "/distfiles/" +
148 + filename_hash_path(filename, *layout[1:]) + filename)
149 + else:
150 + raise AssertionError("get_mirror_url() got unknown layout type")
151 +
152 +
153 def fetch(myuris, mysettings, listonly=0, fetchonly=0,
154 locks_in_subdir=".locks", use_locks=1, try_mirrors=1, digests=None,
155 allow_missing_digests=True):
156 @@ -434,8 +538,9 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0,
157 for myfile, myuri in file_uri_tuples:
158 if myfile not in filedict:
159 filedict[myfile]=[]
160 - for y in range(0,len(locations)):
161 - filedict[myfile].append(locations[y]+"/distfiles/"+myfile)
162 + for l in locations:
163 + filedict[myfile].append(get_mirror_url(l, myfile,
164 + mysettings["EROOT"]))
165 if myuri is None:
166 continue
167 if myuri[:9]=="mirror://":
168 --
169 2.23.0

Replies