1 |
commit: 033f242da9b9f031d8260aca0bf7b70ce8cd93fa |
2 |
Author: Mart Raudsepp <leio <AT> gentoo <DOT> org> |
3 |
AuthorDate: Fri Mar 3 23:46:21 2017 +0000 |
4 |
Commit: Mart Raudsepp <leio <AT> gentoo <DOT> org> |
5 |
CommitDate: Fri Mar 3 23:46:21 2017 +0000 |
6 |
URL: https://gitweb.gentoo.org/proj/gentoo-bumpchecker.git/commit/?id=033f242d |
7 |
|
8 |
gnome: Implement latest version retrieval based on GNOME infra cache.json files |
9 |
|
10 |
Compared to the no longer working ~vuntz devspace versions file, this is |
11 |
now doing HTTPS retrievals for each package again; not quite full scraping, |
12 |
but almost (considering that if we'd hit up the right subdir directly, it's |
13 |
pretty much the same due to one GET per package again). So it takes a bit |
14 |
to retrieve the files. |
15 |
This can probably be made trivially faster by anyone who knows async io |
16 |
for python-requests or something, but at least it works again. |
17 |
|
18 |
gnome-bumpchecker.py | 2 +- |
19 |
modules/gnome_module.py | 45 +++++++++++++++++++++++++++++++++++++++++++-- |
20 |
2 files changed, 44 insertions(+), 3 deletions(-) |
21 |
|
22 |
diff --git a/gnome-bumpchecker.py b/gnome-bumpchecker.py |
23 |
index e7ceb16..c9656e9 100755 |
24 |
--- a/gnome-bumpchecker.py |
25 |
+++ b/gnome-bumpchecker.py |
26 |
@@ -25,7 +25,7 @@ if __name__ == '__main__': |
27 |
|
28 |
# Quick versions file parsing |
29 |
release_packages = gnome.generate_data_release() |
30 |
- latest_packages = gnome.generate_data_individual() |
31 |
+ latest_packages = gnome.generate_data_individual(release_packages) |
32 |
|
33 |
# Old FTP scraping way: |
34 |
# release_packages, latest_packages = gnome.generate_data_ftp() |
35 |
|
36 |
diff --git a/modules/gnome_module.py b/modules/gnome_module.py |
37 |
index 7c8f613..06fe6d8 100644 |
38 |
--- a/modules/gnome_module.py |
39 |
+++ b/modules/gnome_module.py |
40 |
@@ -1,4 +1,5 @@ |
41 |
# Copyright John N. Laliberte <allanonjl@g.o> |
42 |
+# Copyright Mart Raudsepp <leio@g.o> |
43 |
# LICENSE - GPL2 |
44 |
# vim: set sts=4 sw=4 et tw=0 : |
45 |
|
46 |
@@ -8,6 +9,15 @@ import package_module, clioptions_module |
47 |
DEBUG = False |
48 |
|
49 |
|
50 |
+# TODO: Figure out some better handling of mappings together with package_module |
51 |
+# TODO: package_module has made the reverse mapping of what we need, because we consume the names from release_packages |
52 |
+# TODO: So this reverses it back again for the modules we care for this until a proper fix |
53 |
+name_mapping = { |
54 |
+ "cantarell": "cantarell-fonts", |
55 |
+ "nm-applet": "network-manager-applet", |
56 |
+ "networkmanager": "NetworkManager", |
57 |
+} |
58 |
+ |
59 |
class GNOME: |
60 |
def __init__(self, nextrev=False): |
61 |
options = clioptions_module.Options() |
62 |
@@ -49,8 +59,39 @@ class GNOME: |
63 |
print "Warning: Ignoring package %s because parsing of its name or version string '%s' failed" % (components[1], components[2]) |
64 |
return ret |
65 |
|
66 |
- def generate_data_individual(self): |
67 |
- return self.generate_data_release() |
68 |
+ def generate_data_individual(self, release_packages): |
69 |
+ ret = [] |
70 |
+ for pkg in release_packages: |
71 |
+ name = pkg.name.split('/')[-1] |
72 |
+ if name in name_mapping: |
73 |
+ name = name_mapping[name] |
74 |
+ data = self.http.get(self.url_base + '/sources/' + name + '/cache.json') |
75 |
+ if not data: |
76 |
+ print("Warning: Unable to read cache.json for %s" % pkg.name) |
77 |
+ continue |
78 |
+ data = data.json() |
79 |
+ if data[0] != 4: |
80 |
+ print("Warning: unknown cache.json version for package %s" % name) |
81 |
+ continue |
82 |
+ if pkg.major_minor not in data[3]: |
83 |
+ print("Warning: can't find latest version for %s-%s" % (name, pkg.major-minor)) |
84 |
+ continue |
85 |
+ latest = False |
86 |
+ # Some modules contain more than LATEST-IS-* for some reason, so we need to iterate and find the correct item instead of [0] (even though it is firsy always, but lets be future-proof) |
87 |
+ for tarball in data[3][pkg.major_minor]: |
88 |
+ if tarball.startswith('LATEST-IS-'): |
89 |
+ latest = tarball[10:] # len('LATEST-IS-') == 10 |
90 |
+ break |
91 |
+ if not latest: |
92 |
+ print("Warning: couldn't find latest version from within cache.json[3]") |
93 |
+ continue |
94 |
+ latest_pkg = package_module.Package(name + "-" + latest) |
95 |
+ if pkg.name and pkg.version: |
96 |
+ ret.append(latest_pkg) |
97 |
+ else: |
98 |
+ print("Warning: Ignoring package %s because parsing of its name or version string '%s' failed" % (name, latest)) |
99 |
+ continue |
100 |
+ return ret |
101 |
|
102 |
def generate_data_release(self): |
103 |
return self.generate_data_from_versions_markup(self.release_versions_file_path + self.full_release + '/versions') |