Gentoo Archives: gentoo-commits

From: Mart Raudsepp <leio@g.o>
To: gentoo-commits@l.g.o
Subject: [gentoo-commits] proj/gentoo-bumpchecker:master commit in: /, modules/
Date: Fri, 03 Mar 2017 23:51:03
Message-Id: 1488584781.033f242da9b9f031d8260aca0bf7b70ce8cd93fa.leio@gentoo
1 commit: 033f242da9b9f031d8260aca0bf7b70ce8cd93fa
2 Author: Mart Raudsepp <leio <AT> gentoo <DOT> org>
3 AuthorDate: Fri Mar 3 23:46:21 2017 +0000
4 Commit: Mart Raudsepp <leio <AT> gentoo <DOT> org>
5 CommitDate: Fri Mar 3 23:46:21 2017 +0000
6 URL: https://gitweb.gentoo.org/proj/gentoo-bumpchecker.git/commit/?id=033f242d
7
8 gnome: Implement latest version retrieval based on GNOME infra cache.json files
9
10 Compared to the no longer working ~vuntz devspace versions file, this is
11 now doing HTTPS retrievals for each package again; not quite full scraping,
12 but almost (considering that if we'd hit up the right subdir directly, it's
13 pretty much the same due to one GET per package again). So it takes a bit
14 to retrieve the files.
15 This can probably be made trivially faster by anyone who knows async io
16 for python-requests or something, but at least it works again.
17
18 gnome-bumpchecker.py | 2 +-
19 modules/gnome_module.py | 45 +++++++++++++++++++++++++++++++++++++++++++--
20 2 files changed, 44 insertions(+), 3 deletions(-)
21
22 diff --git a/gnome-bumpchecker.py b/gnome-bumpchecker.py
23 index e7ceb16..c9656e9 100755
24 --- a/gnome-bumpchecker.py
25 +++ b/gnome-bumpchecker.py
26 @@ -25,7 +25,7 @@ if __name__ == '__main__':
27
28 # Quick versions file parsing
29 release_packages = gnome.generate_data_release()
30 - latest_packages = gnome.generate_data_individual()
31 + latest_packages = gnome.generate_data_individual(release_packages)
32
33 # Old FTP scraping way:
34 # release_packages, latest_packages = gnome.generate_data_ftp()
35
36 diff --git a/modules/gnome_module.py b/modules/gnome_module.py
37 index 7c8f613..06fe6d8 100644
38 --- a/modules/gnome_module.py
39 +++ b/modules/gnome_module.py
40 @@ -1,4 +1,5 @@
41 # Copyright John N. Laliberte <allanonjl@g.o>
42 +# Copyright Mart Raudsepp <leio@g.o>
43 # LICENSE - GPL2
44 # vim: set sts=4 sw=4 et tw=0 :
45
46 @@ -8,6 +9,15 @@ import package_module, clioptions_module
47 DEBUG = False
48
49
50 +# TODO: Figure out some better handling of mappings together with package_module
51 +# TODO: package_module has made the reverse mapping of what we need, because we consume the names from release_packages
52 +# TODO: So this reverses it back again for the modules we care for this until a proper fix
53 +name_mapping = {
54 + "cantarell": "cantarell-fonts",
55 + "nm-applet": "network-manager-applet",
56 + "networkmanager": "NetworkManager",
57 +}
58 +
59 class GNOME:
60 def __init__(self, nextrev=False):
61 options = clioptions_module.Options()
62 @@ -49,8 +59,39 @@ class GNOME:
63 print "Warning: Ignoring package %s because parsing of its name or version string '%s' failed" % (components[1], components[2])
64 return ret
65
66 - def generate_data_individual(self):
67 - return self.generate_data_release()
68 + def generate_data_individual(self, release_packages):
69 + ret = []
70 + for pkg in release_packages:
71 + name = pkg.name.split('/')[-1]
72 + if name in name_mapping:
73 + name = name_mapping[name]
74 + data = self.http.get(self.url_base + '/sources/' + name + '/cache.json')
75 + if not data:
76 + print("Warning: Unable to read cache.json for %s" % pkg.name)
77 + continue
78 + data = data.json()
79 + if data[0] != 4:
80 + print("Warning: unknown cache.json version for package %s" % name)
81 + continue
82 + if pkg.major_minor not in data[3]:
83 + print("Warning: can't find latest version for %s-%s" % (name, pkg.major-minor))
84 + continue
85 + latest = False
86 + # Some modules contain more than LATEST-IS-* for some reason, so we need to iterate and find the correct item instead of [0] (even though it is firsy always, but lets be future-proof)
87 + for tarball in data[3][pkg.major_minor]:
88 + if tarball.startswith('LATEST-IS-'):
89 + latest = tarball[10:] # len('LATEST-IS-') == 10
90 + break
91 + if not latest:
92 + print("Warning: couldn't find latest version from within cache.json[3]")
93 + continue
94 + latest_pkg = package_module.Package(name + "-" + latest)
95 + if pkg.name and pkg.version:
96 + ret.append(latest_pkg)
97 + else:
98 + print("Warning: Ignoring package %s because parsing of its name or version string '%s' failed" % (name, latest))
99 + continue
100 + return ret
101
102 def generate_data_release(self):
103 return self.generate_data_from_versions_markup(self.release_versions_file_path + self.full_release + '/versions')