Gentoo Archives: gentoo-commits

From: Brian Dolbec <brian.dolbec@×××××.com>
To: gentoo-commits@l.g.o
Subject: [gentoo-commits] proj/layman:master commit in: layman/
Date: Sat, 30 Apr 2011 22:37:30
Message-Id: fcd35daad6e84fdaab69f4bd0c969b80daee590d.dol-sen@gentoo
1 commit: fcd35daad6e84fdaab69f4bd0c969b80daee590d
2 Author: dol-sen <brian.dolbec <AT> gmail <DOT> com>
3 AuthorDate: Sat Apr 30 22:36:54 2011 +0000
4 Commit: Brian Dolbec <brian.dolbec <AT> gmail <DOT> com>
5 CommitDate: Sat Apr 30 22:36:54 2011 +0000
6 URL: http://git.overlays.gentoo.org/gitweb/?p=proj/layman.git;a=commit;h=fcd35daa
7
8 Add checks and code to fetch the new list using 'If-Modified-Since' protocol.
9 adds saving the 'last-modified' date of the same filename with a .timestamp extension.
10
11 ---
12 layman/api.py | 39 +++++++++++++++------------
13 layman/db.py | 80 ++++++++++++++++++++++++++++++++++++++++++++-------------
14 2 files changed, 84 insertions(+), 35 deletions(-)
15
16 diff --git a/layman/api.py b/layman/api.py
17 index cbb9d99..b71dd72 100644
18 --- a/layman/api.py
19 +++ b/layman/api.py
20 @@ -120,7 +120,7 @@ class LaymanAPI(object):
21 self._error(ERROR_INTERNAL_ERROR,
22 "Failed to disable repository '"+ovl+"':\n"+str(e))
23 results.append(False)
24 - self.get_installed(reload=True)
25 + self.get_installed(dbreload=True)
26 if False in results:
27 return False
28 return True
29 @@ -151,7 +151,7 @@ class LaymanAPI(object):
30 self._error(ERROR_INTERNAL_ERROR,
31 "Failed to enable repository '"+ovl+"' : "+str(e))
32 results.append(False)
33 - self.get_installed(reload=True)
34 + self.get_installed(dbreload=True)
35 if False in results:
36 return False
37 return True
38 @@ -373,47 +373,52 @@ class LaymanAPI(object):
39 def fetch_remote_list(self):
40 """Fetches the latest remote overlay list"""
41 try:
42 - self._get_remote_db().cache()
43 + dbreload = self._get_remote_db().cache()
44 + self.output.debug(
45 + 'LaymanAPI.fetch_remote_list(); cache updated = %s'
46 + % str(dbreload),8)
47 except Exception as error:
48 self._error('Failed to fetch overlay list!\n Original Error was: '
49 + str(error))
50 return False
51 - self.get_available(reload=True)
52 + self.get_available(dbreload)
53 return True
54
55
56 - def get_available(self, reload=False):
57 + def get_available(self, dbreload=False):
58 """returns the list of available overlays"""
59 - if self._available_ids is None or reload:
60 - self._available_ids = self._get_remote_db(reload).list_ids()
61 + self.output.info('LaymanAPI.get_available() dbreload = %s'
62 + % str(dbreload))
63 + if self._available_ids is None or dbreload:
64 + self._available_ids = self._get_remote_db(dbreload).list_ids()
65 return self._available_ids[:] or ['None']
66
67
68 - def get_installed(self, reload=False):
69 + def get_installed(self, dbreload=False):
70 """returns the list of installed overlays"""
71 - if self._installed_ids is None or reload:
72 - self._installed_ids = self._get_installed_db(reload).list_ids()
73 + if self._installed_ids is None or dbreload:
74 + self._installed_ids = self._get_installed_db(dbreload).list_ids()
75 return self._installed_ids[:]
76
77
78 - def _get_installed_db(self, reload=False):
79 + def _get_installed_db(self, dbreload=False):
80 """returns the list of installed overlays"""
81 - if not self._installed_db or reload:
82 + if not self._installed_db or dbreload:
83 self._installed_db = DB(self.config)
84 return self._installed_db
85
86
87 - def _get_remote_db(self, reload=False):
88 + def _get_remote_db(self, dbreload=False):
89 """returns the list of installed overlays"""
90 - if self._available_db is None or reload:
91 + if self._available_db is None or dbreload:
92 self._available_db = RemoteDB(self.config)
93 return self._available_db
94
95
96 def reload(self):
97 """reloads the installed and remote db's to the data on disk"""
98 - result = self.get_available(reload=True)
99 - result = self.get_installed(reload=True)
100 + result = self.get_available(dbreload=True)
101 + result = self.get_installed(dbreload=True)
102
103
104 def _error(self, message):
105 @@ -423,7 +428,7 @@ class LaymanAPI(object):
106 """
107 self._error_messages.append(message)
108 if self.report_errors:
109 - print(message, file=stderr)
110 + print(message, file=self.config['stderr'])
111
112
113 def get_errors(self):
114
115 diff --git a/layman/db.py b/layman/db.py
116 index d078e46..3498223 100644
117 --- a/layman/db.py
118 +++ b/layman/db.py
119 @@ -24,7 +24,9 @@ __version__ = "$Id: db.py 309 2007-04-09 16:23:38Z wrobel $"
120 #
121 #-------------------------------------------------------------------------------
122
123 -import os, os.path, urllib2, hashlib
124 +import os, os.path
125 +import urllib2
126 +import hashlib
127
128 from layman.utils import path, delete_empty_directory
129 from layman.dbbase import DbBase
130 @@ -243,7 +245,7 @@ class RemoteDB(DbBase):
131
132 self.urls = [i.strip() for i in config['overlays'].split('\n') if len(i)]
133
134 - paths = [self.path(i) for i in self.urls]
135 + paths = [self.filepath(i) + '.xml' for i in self.urls]
136
137 if config['nocheck']:
138 ignore = 2
139 @@ -283,22 +285,45 @@ class RemoteDB(DbBase):
140 >>> a.overlays.keys()
141 [u'wrobel', u'wrobel-stable']
142 '''
143 + has_updates = False
144 for url in self.urls:
145
146 - mpath = self.path(url)
147 + filepath = self.filepath(url)
148 + mpath = filepath + '.xml'
149 + tpath = filepath + '.timestamp'
150
151 - # Check for sufficient privileges
152 - if os.path.exists(mpath) and not os.access(mpath, os.W_OK):
153 - self.output.warn('You do not have permission to update the cache (%s).' % mpath)
154 - import getpass
155 - if getpass.getuser() != 'root':
156 - self.output.warn('Hint: You are not root.\n')
157 + # check when the cache was last updated
158 + # and don't re-fetch it unless it has changed
159 + request = urllib2.Request(url)
160 + opener = urllib2.build_opener()
161 + opener.addheaders = [('User-Agent', 'Layman-2.0-git')]
162 +
163 + if os.path.exists(tpath):
164 + with open(tpath,'r') as previous:
165 + last_time = previous.read()
166 + request.add_header('If-Modified-Since', last_time)
167 +
168 + if not self.check_path([mpath]):
169 continue
170
171 try:
172 -
173 + connection = opener.open(request)
174 + timestamp = connection.headers['last-modified']
175 + except urllib2.HTTPError as e:
176 + if e.getcode() == 304:
177 + self.output.info('Remote list already up to date: %s'
178 + % url)
179 + else:
180 + self.output.info('RemoteDB.cache(); HTTPError was:\n %s'
181 + % str(e))
182 + continue
183 + except IOError as error:
184 + self.output.warn('Failed to update the overlay list from: '
185 + + url + '\nError was:\n' + str(error))
186 + else:
187 + self.output.info('Fetching new list...')
188 # Fetch the remote list
189 - olist = urllib2.urlopen(url).read()
190 + olist = connection.read()
191
192 # Create our storage directory if it is missing
193 if not os.path.exists(os.path.dirname(mpath)):
194 @@ -326,24 +351,43 @@ class RemoteDB(DbBase):
195 out_file.write(olist)
196 out_file.close()
197
198 + out_file = open(tpath, 'w')
199 + out_file.write(timestamp)
200 + out_file.close()
201 +
202 + has_updates = True
203 +
204 except Exception as error:
205 raise IOError('Failed to temporarily cache overlays list in'
206 ' ' + mpath + '\nError was:\n' + str(error))
207 + return has_updates
208
209
210 - except IOError as error:
211 - self.output.warn('Failed to update the overlay list from: '
212 - + url + '\nError was:\n' + str(error))
213 -
214 - def path(self, url):
215 + def filepath(self, url):
216 '''Return a unique file name for the url.'''
217
218 base = self.config['cache']
219
220 self.output.debug('Generating cache path.', 6)
221
222 - return base + '_' + hashlib.md5(url).hexdigest() + '.xml'
223 -
224 + return base + '_' + hashlib.md5(url).hexdigest()
225 +
226 +
227 + def check_path(self, paths, hint=True):
228 + '''Check for sufficient privileges'''
229 + self.output.debug('RemoteDB.check_path; paths = ' + str(paths), 8)
230 + is_ok = True
231 + for path in paths:
232 + if os.path.exists(path) and not os.access(path, os.W_OK):
233 + if hint:
234 + self.output.warn(
235 + 'You do not have permission to update the cache (%s).'
236 + % mpath)
237 + import getpass
238 + if getpass.getuser() != 'root':
239 + self.output.warn('Hint: You are not root.\n')
240 + is_ok = False
241 + return is_ok
242
243 #===============================================================================
244 #