1 |
commit: fcd35daad6e84fdaab69f4bd0c969b80daee590d |
2 |
Author: dol-sen <brian.dolbec <AT> gmail <DOT> com> |
3 |
AuthorDate: Sat Apr 30 22:36:54 2011 +0000 |
4 |
Commit: Brian Dolbec <brian.dolbec <AT> gmail <DOT> com> |
5 |
CommitDate: Sat Apr 30 22:36:54 2011 +0000 |
6 |
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/layman.git;a=commit;h=fcd35daa |
7 |
|
8 |
Add checks and code to fetch the new list using 'If-Modified-Since' protocol. |
9 |
adds saving the 'last-modified' date of the same filename with a .timestamp extension. |
10 |
|
11 |
--- |
12 |
layman/api.py | 39 +++++++++++++++------------ |
13 |
layman/db.py | 80 ++++++++++++++++++++++++++++++++++++++++++++------------- |
14 |
2 files changed, 84 insertions(+), 35 deletions(-) |
15 |
|
16 |
diff --git a/layman/api.py b/layman/api.py |
17 |
index cbb9d99..b71dd72 100644 |
18 |
--- a/layman/api.py |
19 |
+++ b/layman/api.py |
20 |
@@ -120,7 +120,7 @@ class LaymanAPI(object): |
21 |
self._error(ERROR_INTERNAL_ERROR, |
22 |
"Failed to disable repository '"+ovl+"':\n"+str(e)) |
23 |
results.append(False) |
24 |
- self.get_installed(reload=True) |
25 |
+ self.get_installed(dbreload=True) |
26 |
if False in results: |
27 |
return False |
28 |
return True |
29 |
@@ -151,7 +151,7 @@ class LaymanAPI(object): |
30 |
self._error(ERROR_INTERNAL_ERROR, |
31 |
"Failed to enable repository '"+ovl+"' : "+str(e)) |
32 |
results.append(False) |
33 |
- self.get_installed(reload=True) |
34 |
+ self.get_installed(dbreload=True) |
35 |
if False in results: |
36 |
return False |
37 |
return True |
38 |
@@ -373,47 +373,52 @@ class LaymanAPI(object): |
39 |
def fetch_remote_list(self): |
40 |
"""Fetches the latest remote overlay list""" |
41 |
try: |
42 |
- self._get_remote_db().cache() |
43 |
+ dbreload = self._get_remote_db().cache() |
44 |
+ self.output.debug( |
45 |
+ 'LaymanAPI.fetch_remote_list(); cache updated = %s' |
46 |
+ % str(dbreload),8) |
47 |
except Exception as error: |
48 |
self._error('Failed to fetch overlay list!\n Original Error was: ' |
49 |
+ str(error)) |
50 |
return False |
51 |
- self.get_available(reload=True) |
52 |
+ self.get_available(dbreload) |
53 |
return True |
54 |
|
55 |
|
56 |
- def get_available(self, reload=False): |
57 |
+ def get_available(self, dbreload=False): |
58 |
"""returns the list of available overlays""" |
59 |
- if self._available_ids is None or reload: |
60 |
- self._available_ids = self._get_remote_db(reload).list_ids() |
61 |
+ self.output.info('LaymanAPI.get_available() dbreload = %s' |
62 |
+ % str(dbreload)) |
63 |
+ if self._available_ids is None or dbreload: |
64 |
+ self._available_ids = self._get_remote_db(dbreload).list_ids() |
65 |
return self._available_ids[:] or ['None'] |
66 |
|
67 |
|
68 |
- def get_installed(self, reload=False): |
69 |
+ def get_installed(self, dbreload=False): |
70 |
"""returns the list of installed overlays""" |
71 |
- if self._installed_ids is None or reload: |
72 |
- self._installed_ids = self._get_installed_db(reload).list_ids() |
73 |
+ if self._installed_ids is None or dbreload: |
74 |
+ self._installed_ids = self._get_installed_db(dbreload).list_ids() |
75 |
return self._installed_ids[:] |
76 |
|
77 |
|
78 |
- def _get_installed_db(self, reload=False): |
79 |
+ def _get_installed_db(self, dbreload=False): |
80 |
"""returns the list of installed overlays""" |
81 |
- if not self._installed_db or reload: |
82 |
+ if not self._installed_db or dbreload: |
83 |
self._installed_db = DB(self.config) |
84 |
return self._installed_db |
85 |
|
86 |
|
87 |
- def _get_remote_db(self, reload=False): |
88 |
+ def _get_remote_db(self, dbreload=False): |
89 |
"""returns the list of installed overlays""" |
90 |
- if self._available_db is None or reload: |
91 |
+ if self._available_db is None or dbreload: |
92 |
self._available_db = RemoteDB(self.config) |
93 |
return self._available_db |
94 |
|
95 |
|
96 |
def reload(self): |
97 |
"""reloads the installed and remote db's to the data on disk""" |
98 |
- result = self.get_available(reload=True) |
99 |
- result = self.get_installed(reload=True) |
100 |
+ result = self.get_available(dbreload=True) |
101 |
+ result = self.get_installed(dbreload=True) |
102 |
|
103 |
|
104 |
def _error(self, message): |
105 |
@@ -423,7 +428,7 @@ class LaymanAPI(object): |
106 |
""" |
107 |
self._error_messages.append(message) |
108 |
if self.report_errors: |
109 |
- print(message, file=stderr) |
110 |
+ print(message, file=self.config['stderr']) |
111 |
|
112 |
|
113 |
def get_errors(self): |
114 |
|
115 |
diff --git a/layman/db.py b/layman/db.py |
116 |
index d078e46..3498223 100644 |
117 |
--- a/layman/db.py |
118 |
+++ b/layman/db.py |
119 |
@@ -24,7 +24,9 @@ __version__ = "$Id: db.py 309 2007-04-09 16:23:38Z wrobel $" |
120 |
# |
121 |
#------------------------------------------------------------------------------- |
122 |
|
123 |
-import os, os.path, urllib2, hashlib |
124 |
+import os, os.path |
125 |
+import urllib2 |
126 |
+import hashlib |
127 |
|
128 |
from layman.utils import path, delete_empty_directory |
129 |
from layman.dbbase import DbBase |
130 |
@@ -243,7 +245,7 @@ class RemoteDB(DbBase): |
131 |
|
132 |
self.urls = [i.strip() for i in config['overlays'].split('\n') if len(i)] |
133 |
|
134 |
- paths = [self.path(i) for i in self.urls] |
135 |
+ paths = [self.filepath(i) + '.xml' for i in self.urls] |
136 |
|
137 |
if config['nocheck']: |
138 |
ignore = 2 |
139 |
@@ -283,22 +285,45 @@ class RemoteDB(DbBase): |
140 |
>>> a.overlays.keys() |
141 |
[u'wrobel', u'wrobel-stable'] |
142 |
''' |
143 |
+ has_updates = False |
144 |
for url in self.urls: |
145 |
|
146 |
- mpath = self.path(url) |
147 |
+ filepath = self.filepath(url) |
148 |
+ mpath = filepath + '.xml' |
149 |
+ tpath = filepath + '.timestamp' |
150 |
|
151 |
- # Check for sufficient privileges |
152 |
- if os.path.exists(mpath) and not os.access(mpath, os.W_OK): |
153 |
- self.output.warn('You do not have permission to update the cache (%s).' % mpath) |
154 |
- import getpass |
155 |
- if getpass.getuser() != 'root': |
156 |
- self.output.warn('Hint: You are not root.\n') |
157 |
+ # check when the cache was last updated |
158 |
+ # and don't re-fetch it unless it has changed |
159 |
+ request = urllib2.Request(url) |
160 |
+ opener = urllib2.build_opener() |
161 |
+ opener.addheaders = [('User-Agent', 'Layman-2.0-git')] |
162 |
+ |
163 |
+ if os.path.exists(tpath): |
164 |
+ with open(tpath,'r') as previous: |
165 |
+ last_time = previous.read() |
166 |
+ request.add_header('If-Modified-Since', last_time) |
167 |
+ |
168 |
+ if not self.check_path([mpath]): |
169 |
continue |
170 |
|
171 |
try: |
172 |
- |
173 |
+ connection = opener.open(request) |
174 |
+ timestamp = connection.headers['last-modified'] |
175 |
+ except urllib2.HTTPError as e: |
176 |
+ if e.getcode() == 304: |
177 |
+ self.output.info('Remote list already up to date: %s' |
178 |
+ % url) |
179 |
+ else: |
180 |
+ self.output.info('RemoteDB.cache(); HTTPError was:\n %s' |
181 |
+ % str(e)) |
182 |
+ continue |
183 |
+ except IOError as error: |
184 |
+ self.output.warn('Failed to update the overlay list from: ' |
185 |
+ + url + '\nError was:\n' + str(error)) |
186 |
+ else: |
187 |
+ self.output.info('Fetching new list...') |
188 |
# Fetch the remote list |
189 |
- olist = urllib2.urlopen(url).read() |
190 |
+ olist = connection.read() |
191 |
|
192 |
# Create our storage directory if it is missing |
193 |
if not os.path.exists(os.path.dirname(mpath)): |
194 |
@@ -326,24 +351,43 @@ class RemoteDB(DbBase): |
195 |
out_file.write(olist) |
196 |
out_file.close() |
197 |
|
198 |
+ out_file = open(tpath, 'w') |
199 |
+ out_file.write(timestamp) |
200 |
+ out_file.close() |
201 |
+ |
202 |
+ has_updates = True |
203 |
+ |
204 |
except Exception as error: |
205 |
raise IOError('Failed to temporarily cache overlays list in' |
206 |
' ' + mpath + '\nError was:\n' + str(error)) |
207 |
+ return has_updates |
208 |
|
209 |
|
210 |
- except IOError as error: |
211 |
- self.output.warn('Failed to update the overlay list from: ' |
212 |
- + url + '\nError was:\n' + str(error)) |
213 |
- |
214 |
- def path(self, url): |
215 |
+ def filepath(self, url): |
216 |
'''Return a unique file name for the url.''' |
217 |
|
218 |
base = self.config['cache'] |
219 |
|
220 |
self.output.debug('Generating cache path.', 6) |
221 |
|
222 |
- return base + '_' + hashlib.md5(url).hexdigest() + '.xml' |
223 |
- |
224 |
+ return base + '_' + hashlib.md5(url).hexdigest() |
225 |
+ |
226 |
+ |
227 |
+ def check_path(self, paths, hint=True): |
228 |
+ '''Check for sufficient privileges''' |
229 |
+ self.output.debug('RemoteDB.check_path; paths = ' + str(paths), 8) |
230 |
+ is_ok = True |
231 |
+ for path in paths: |
232 |
+ if os.path.exists(path) and not os.access(path, os.W_OK): |
233 |
+ if hint: |
234 |
+ self.output.warn( |
235 |
+ 'You do not have permission to update the cache (%s).' |
236 |
+ % mpath) |
237 |
+ import getpass |
238 |
+ if getpass.getuser() != 'root': |
239 |
+ self.output.warn('Hint: You are not root.\n') |
240 |
+ is_ok = False |
241 |
+ return is_ok |
242 |
|
243 |
#=============================================================================== |
244 |
# |