1 |
The hardlink logic is unused, so start by deleting all of that. |
2 |
If someone wants that (multiple builds on NFS?), we can look at |
3 |
restoring it. |
4 |
--- |
5 |
catalyst/lock.py | 467 ++----------------------------------------------------- |
6 |
1 file changed, 15 insertions(+), 452 deletions(-) |
7 |
|
8 |
diff --git a/catalyst/lock.py b/catalyst/lock.py |
9 |
index 8095a82..39926dd 100644 |
10 |
--- a/catalyst/lock.py |
11 |
+++ b/catalyst/lock.py |
12 |
@@ -1,467 +1,30 @@ |
13 |
|
14 |
- |
15 |
import os |
16 |
-import fcntl |
17 |
-import errno |
18 |
-import sys |
19 |
-import time |
20 |
-from catalyst.support import CatalystError, normpath |
21 |
|
22 |
-def writemsg(mystr): |
23 |
- sys.stderr.write(mystr) |
24 |
- sys.stderr.flush() |
25 |
+from snakeoil import fileutils |
26 |
+from snakeoil import osutils |
27 |
|
28 |
|
29 |
-class LockInUse(Exception): |
30 |
- def __init__(self, message): |
31 |
- if message: |
32 |
- #(type,value)=sys.exc_info()[:2] |
33 |
- #if value!=None: |
34 |
- #print |
35 |
- #kprint traceback.print_exc(file=sys.stdout) |
36 |
- print |
37 |
- print "!!! catalyst lock file in use: "+message |
38 |
- print |
39 |
+LockInUse = osutils.LockException |
40 |
|
41 |
|
42 |
class LockDir(object): |
43 |
- locking_method=fcntl.flock |
44 |
- lock_dirs_in_use=[] |
45 |
- die_on_failed_lock=True |
46 |
- |
47 |
- def __del__(self): |
48 |
- #print "Lock.__del__() 1" |
49 |
- self.clean_my_hardlocks() |
50 |
- #print "Lock.__del__() 2" |
51 |
- self.delete_lock_from_path_list() |
52 |
- #print "Lock.__del__() 3" |
53 |
- if self.islocked(): |
54 |
- #print "Lock.__del__() 4" |
55 |
- self.fcntl_unlock() |
56 |
- #print "Lock.__del__() finnished" |
57 |
- |
58 |
- def __init__(self,lockdir): |
59 |
- self.locked=False |
60 |
- self.myfd=None |
61 |
- self.set_gid(250) |
62 |
- self.locking_method=LockDir.locking_method |
63 |
- self.set_lockdir(lockdir) |
64 |
- self.set_lockfilename(".catalyst_lock") |
65 |
- self.set_lockfile() |
66 |
- |
67 |
- if LockDir.lock_dirs_in_use.count(lockdir)>0: |
68 |
- raise "This directory already associated with a lock object" |
69 |
- else: |
70 |
- LockDir.lock_dirs_in_use.append(lockdir) |
71 |
- |
72 |
- self.myhardlock = None |
73 |
- self.hardlock_paths={} |
74 |
- |
75 |
- def delete_lock_from_path_list(self): |
76 |
- try: |
77 |
- LockDir.lock_dirs_in_use.remove(self.lockdir) |
78 |
- except ValueError: |
79 |
- pass |
80 |
- |
81 |
- def islocked(self): |
82 |
- if self.locked: |
83 |
- return True |
84 |
- else: |
85 |
- return False |
86 |
+ """An object that creates locks inside dirs""" |
87 |
|
88 |
- def set_gid(self,gid): |
89 |
- if not self.islocked(): |
90 |
-# if self.settings["DEBUG"]: |
91 |
-# print "setting gid to", gid |
92 |
- self.gid=gid |
93 |
- |
94 |
- def set_lockdir(self,lockdir): |
95 |
- if not os.path.exists(lockdir): |
96 |
- os.makedirs(lockdir) |
97 |
- if os.path.isdir(lockdir): |
98 |
- if not self.islocked(): |
99 |
- if lockdir[-1] == "/": |
100 |
- lockdir=lockdir[:-1] |
101 |
- self.lockdir=normpath(lockdir) |
102 |
-# if self.settings["DEBUG"]: |
103 |
-# print "setting lockdir to", self.lockdir |
104 |
- else: |
105 |
- raise "the lock object needs a path to a dir" |
106 |
- |
107 |
- def set_lockfilename(self,lockfilename): |
108 |
- if not self.islocked(): |
109 |
- self.lockfilename=lockfilename |
110 |
-# if self.settings["DEBUG"]: |
111 |
-# print "setting lockfilename to", self.lockfilename |
112 |
- |
113 |
- def set_lockfile(self): |
114 |
- if not self.islocked(): |
115 |
- self.lockfile=normpath(self.lockdir+'/'+self.lockfilename) |
116 |
-# if self.settings["DEBUG"]: |
117 |
-# print "setting lockfile to", self.lockfile |
118 |
+ def __init__(self, lockdir): |
119 |
+ self.gid = 250 |
120 |
+ self.lockfile = os.path.join(lockdir, '.catalyst_lock') |
121 |
+ osutils.ensure_dirs(lockdir) |
122 |
+ fileutils.touch(self.lockfile, mode=0o664) |
123 |
+ os.chown(self.lockfile, -1, self.gid) |
124 |
+ self.lock = osutils.FsLock(self.lockfile) |
125 |
|
126 |
def read_lock(self): |
127 |
- if not self.locking_method == "HARDLOCK": |
128 |
- self.fcntl_lock("read") |
129 |
- else: |
130 |
- print "HARDLOCKING doesnt support shared-read locks" |
131 |
- print "using exclusive write locks" |
132 |
- self.hard_lock() |
133 |
+ self.lock.acquire_read_lock() |
134 |
|
135 |
def write_lock(self): |
136 |
- if not self.locking_method == "HARDLOCK": |
137 |
- self.fcntl_lock("write") |
138 |
- else: |
139 |
- self.hard_lock() |
140 |
+ self.lock.acquire_write_lock() |
141 |
|
142 |
def unlock(self): |
143 |
- if not self.locking_method == "HARDLOCK": |
144 |
- self.fcntl_unlock() |
145 |
- else: |
146 |
- self.hard_unlock() |
147 |
- |
148 |
- def fcntl_lock(self,locktype): |
149 |
- if self.myfd==None: |
150 |
- if not os.path.exists(os.path.dirname(self.lockdir)): |
151 |
- raise CatalystError("DirectoryNotFound: %s" |
152 |
- % os.path.dirname(self.lockdir), print_traceback=True) |
153 |
- if not os.path.exists(self.lockfile): |
154 |
- old_mask=os.umask(000) |
155 |
- self.myfd = os.open(self.lockfile, os.O_CREAT|os.O_RDWR,0660) |
156 |
- try: |
157 |
- if os.stat(self.lockfile).st_gid != self.gid: |
158 |
- os.chown(self.lockfile,os.getuid(),self.gid) |
159 |
- except OSError, e: |
160 |
- if e[0] == 2: #XXX: No such file or directory |
161 |
- return self.fcntl_locking(locktype) |
162 |
- else: |
163 |
- writemsg("Cannot chown a lockfile. This could cause inconvenience later.\n") |
164 |
- |
165 |
- os.umask(old_mask) |
166 |
- else: |
167 |
- self.myfd = os.open(self.lockfile, os.O_CREAT|os.O_RDWR,0660) |
168 |
- |
169 |
- try: |
170 |
- if locktype == "read": |
171 |
- self.locking_method(self.myfd,fcntl.LOCK_SH|fcntl.LOCK_NB) |
172 |
- else: |
173 |
- self.locking_method(self.myfd,fcntl.LOCK_EX|fcntl.LOCK_NB) |
174 |
- except IOError, e: |
175 |
- if "errno" not in dir(e): |
176 |
- raise |
177 |
- if e.errno == errno.EAGAIN: |
178 |
- if not LockDir.die_on_failed_lock: |
179 |
- # Resource temp unavailable; eg, someone beat us to the lock. |
180 |
- writemsg("waiting for lock on %s\n" % self.lockfile) |
181 |
- |
182 |
- # Try for the exclusive or shared lock again. |
183 |
- if locktype == "read": |
184 |
- self.locking_method(self.myfd,fcntl.LOCK_SH) |
185 |
- else: |
186 |
- self.locking_method(self.myfd,fcntl.LOCK_EX) |
187 |
- else: |
188 |
- raise LockInUse,self.lockfile |
189 |
- elif e.errno == errno.ENOLCK: |
190 |
- pass |
191 |
- else: |
192 |
- raise |
193 |
- if not os.path.exists(self.lockfile): |
194 |
- os.close(self.myfd) |
195 |
- self.myfd=None |
196 |
- #writemsg("lockfile recurse\n") |
197 |
- self.fcntl_lock(locktype) |
198 |
- else: |
199 |
- self.locked=True |
200 |
- #writemsg("Lockfile obtained\n") |
201 |
- |
202 |
- def fcntl_unlock(self): |
203 |
- unlinkfile = 1 |
204 |
- if not os.path.exists(self.lockfile): |
205 |
- print "lockfile does not exist '%s'" % self.lockfile |
206 |
- #print "fcntl_unlock() , self.myfd:", self.myfd, type(self.myfd) |
207 |
- if self.myfd != None: |
208 |
- #print "fcntl_unlock() trying to close it " |
209 |
- try: |
210 |
- os.close(self.myfd) |
211 |
- self.myfd=None |
212 |
- except Exception: |
213 |
- pass |
214 |
- return False |
215 |
- |
216 |
- try: |
217 |
- if self.myfd == None: |
218 |
- self.myfd = os.open(self.lockfile, os.O_WRONLY,0660) |
219 |
- unlinkfile = 1 |
220 |
- self.locking_method(self.myfd,fcntl.LOCK_UN) |
221 |
- except Exception, e: |
222 |
- #if self.myfd is not None: |
223 |
- #print "fcntl_unlock() trying to close", self.myfd |
224 |
- #os.close(self.myfd) |
225 |
- #self.myfd=None |
226 |
- #raise IOError, "Failed to unlock file '%s'\n%s" % (self.lockfile, str(e)) |
227 |
- try: |
228 |
- # This sleep call was added to allow other processes that are |
229 |
- # waiting for a lock to be able to grab it before it is deleted. |
230 |
- # lockfile() already accounts for this situation, however, and |
231 |
- # the sleep here adds more time than is saved overall, so am |
232 |
- # commenting until it is proved necessary. |
233 |
- #time.sleep(0.0001) |
234 |
- if unlinkfile: |
235 |
- InUse=False |
236 |
- try: |
237 |
- self.locking_method(self.myfd,fcntl.LOCK_EX|fcntl.LOCK_NB) |
238 |
- except Exception: |
239 |
- print "Read lock may be in effect. skipping lockfile delete..." |
240 |
- InUse=True |
241 |
- # We won the lock, so there isn't competition for it. |
242 |
- # We can safely delete the file. |
243 |
- #writemsg("Got the lockfile...\n") |
244 |
- #writemsg("Unlinking...\n") |
245 |
- self.locking_method(self.myfd,fcntl.LOCK_UN) |
246 |
- if not InUse: |
247 |
- os.unlink(self.lockfile) |
248 |
- os.close(self.myfd) |
249 |
- self.myfd=None |
250 |
-# if self.settings["DEBUG"]: |
251 |
-# print "Unlinked lockfile..." |
252 |
- except Exception, e: |
253 |
- # We really don't care... Someone else has the lock. |
254 |
- # So it is their problem now. |
255 |
- print "Failed to get lock... someone took it." |
256 |
- print str(e) |
257 |
- |
258 |
- # Why test lockfilename? Because we may have been handed an |
259 |
- # fd originally, and the caller might not like having their |
260 |
- # open fd closed automatically on them. |
261 |
- #if type(lockfilename) == types.StringType: |
262 |
- # os.close(myfd) |
263 |
- #print "fcntl_unlock() trying a last ditch close", self.myfd |
264 |
- if self.myfd != None: |
265 |
- os.close(self.myfd) |
266 |
- self.myfd=None |
267 |
- self.locked=False |
268 |
- time.sleep(.0001) |
269 |
- |
270 |
- def hard_lock(self,max_wait=14400): |
271 |
- """Does the NFS, hardlink shuffle to ensure locking on the disk. |
272 |
- We create a PRIVATE lockfile, that is just a placeholder on the disk. |
273 |
- Then we HARDLINK the real lockfile to that private file. |
274 |
- If our file can 2 references, then we have the lock. :) |
275 |
- Otherwise we lather, rise, and repeat. |
276 |
- We default to a 4 hour timeout. |
277 |
- """ |
278 |
- |
279 |
- self.myhardlock = self.hardlock_name(self.lockdir) |
280 |
- |
281 |
- start_time = time.time() |
282 |
- reported_waiting = False |
283 |
- |
284 |
- while time.time() < (start_time + max_wait): |
285 |
- # We only need it to exist. |
286 |
- self.myfd = os.open(self.myhardlock, os.O_CREAT|os.O_RDWR,0660) |
287 |
- os.close(self.myfd) |
288 |
- |
289 |
- self.add_hardlock_file_to_cleanup() |
290 |
- if not os.path.exists(self.myhardlock): |
291 |
- raise CatalystError("FileNotFound: Created lockfile is missing: " |
292 |
- "%(filename)s" % {"filename":self.myhardlock}, |
293 |
- print_traceback=True) |
294 |
- try: |
295 |
- os.link(self.myhardlock, self.lockfile) |
296 |
- except Exception: |
297 |
-# if self.settings["DEBUG"]: |
298 |
-# print "lockfile(): Hardlink: Link failed." |
299 |
-# print "Exception: ",e |
300 |
- pass |
301 |
- |
302 |
- if self.hardlink_is_mine(self.myhardlock, self.lockfile): |
303 |
- # We have the lock. |
304 |
- if reported_waiting: |
305 |
- print |
306 |
- return True |
307 |
- |
308 |
- if reported_waiting: |
309 |
- writemsg(".") |
310 |
- else: |
311 |
- reported_waiting = True |
312 |
- print |
313 |
- print "Waiting on (hardlink) lockfile: (one '.' per 3 seconds)" |
314 |
- print "Lockfile: " + self.lockfile |
315 |
- time.sleep(3) |
316 |
- |
317 |
- os.unlink(self.myhardlock) |
318 |
- return False |
319 |
- |
320 |
- def hard_unlock(self): |
321 |
- try: |
322 |
- if os.path.exists(self.myhardlock): |
323 |
- os.unlink(self.myhardlock) |
324 |
- if os.path.exists(self.lockfile): |
325 |
- os.unlink(self.lockfile) |
326 |
- except Exception: |
327 |
- writemsg("Something strange happened to our hardlink locks.\n") |
328 |
- |
329 |
- def add_hardlock_file_to_cleanup(self): |
330 |
- #mypath = self.normpath(path) |
331 |
- if os.path.isdir(self.lockdir) and os.path.isfile(self.myhardlock): |
332 |
- self.hardlock_paths[self.lockdir]=self.myhardlock |
333 |
- |
334 |
- def remove_hardlock_file_from_cleanup(self): |
335 |
- if self.lockdir in self.hardlock_paths: |
336 |
- del self.hardlock_paths[self.lockdir] |
337 |
- print self.hardlock_paths |
338 |
- |
339 |
- @staticmethod |
340 |
- def hardlock_name(path): |
341 |
- mypath=path+"/.hardlock-"+os.uname()[1]+"-"+str(os.getpid()) |
342 |
- newpath = os.path.normpath(mypath) |
343 |
- if len(newpath) > 1: |
344 |
- if newpath[1] == "/": |
345 |
- newpath = "/"+newpath.lstrip("/") |
346 |
- return newpath |
347 |
- |
348 |
- @staticmethod |
349 |
- def hardlink_is_mine(link, lock): |
350 |
- import stat |
351 |
- try: |
352 |
- myhls = os.stat(link) |
353 |
- mylfs = os.stat(lock) |
354 |
- except Exception: |
355 |
- myhls = None |
356 |
- mylfs = None |
357 |
- |
358 |
- if myhls: |
359 |
- if myhls[stat.ST_NLINK] == 2: |
360 |
- return True |
361 |
- if mylfs: |
362 |
- if mylfs[stat.ST_INO] == myhls[stat.ST_INO]: |
363 |
- return True |
364 |
- return False |
365 |
- |
366 |
- @staticmethod |
367 |
- def hardlink_active(lock): |
368 |
- if not os.path.exists(lock): |
369 |
- return False |
370 |
- |
371 |
- def clean_my_hardlocks(self): |
372 |
- try: |
373 |
- for x in self.hardlock_paths.keys(): |
374 |
- self.hardlock_cleanup(x) |
375 |
- except AttributeError: |
376 |
- pass |
377 |
- |
378 |
- def hardlock_cleanup(self,path): |
379 |
- #mypid = str(os.getpid()) |
380 |
- myhost = os.uname()[1] |
381 |
- mydl = os.listdir(path) |
382 |
- results = [] |
383 |
- mycount = 0 |
384 |
- |
385 |
- mylist = {} |
386 |
- for x in mydl: |
387 |
- filepath=path+"/"+x |
388 |
- if os.path.isfile(filepath): |
389 |
- parts = filepath.split(".hardlock-") |
390 |
- if len(parts) == 2: |
391 |
- filename = parts[0] |
392 |
- hostpid = parts[1].split("-") |
393 |
- host = "-".join(hostpid[:-1]) |
394 |
- pid = hostpid[-1] |
395 |
- if filename not in mylist: |
396 |
- mylist[filename] = {} |
397 |
- |
398 |
- if host not in mylist[filename]: |
399 |
- mylist[filename][host] = [] |
400 |
- mylist[filename][host].append(pid) |
401 |
- mycount += 1 |
402 |
- else: |
403 |
- mylist[filename][host].append(pid) |
404 |
- mycount += 1 |
405 |
- |
406 |
- |
407 |
- results.append("Found %(count)s locks" % {"count":mycount}) |
408 |
- for x in mylist.keys(): |
409 |
- if myhost in mylist[x]: |
410 |
- mylockname = self.hardlock_name(x) |
411 |
- if self.hardlink_is_mine(mylockname, self.lockfile) or \ |
412 |
- not os.path.exists(self.lockfile): |
413 |
- for y in mylist[x].keys(): |
414 |
- for z in mylist[x][y]: |
415 |
- filename = x+".hardlock-"+y+"-"+z |
416 |
- if filename == mylockname: |
417 |
- self.hard_unlock() |
418 |
- continue |
419 |
- try: |
420 |
- # We're sweeping through, unlinking everyone's locks. |
421 |
- os.unlink(filename) |
422 |
- results.append("Unlinked: " + filename) |
423 |
- except Exception: |
424 |
- pass |
425 |
- try: |
426 |
- os.unlink(x) |
427 |
- results.append("Unlinked: " + x) |
428 |
- os.unlink(mylockname) |
429 |
- results.append("Unlinked: " + mylockname) |
430 |
- except Exception: |
431 |
- pass |
432 |
- else: |
433 |
- try: |
434 |
- os.unlink(mylockname) |
435 |
- results.append("Unlinked: " + mylockname) |
436 |
- except Exception: |
437 |
- pass |
438 |
- return results |
439 |
- |
440 |
- |
441 |
-if __name__ == "__main__": |
442 |
- |
443 |
- def lock_work(): |
444 |
- print |
445 |
- for i in range(1,6): |
446 |
- print i,time.time() |
447 |
- time.sleep(1) |
448 |
- print |
449 |
- |
450 |
- print "Lock 5 starting" |
451 |
- Lock1=LockDir("/tmp/lock_path") |
452 |
- Lock1.write_lock() |
453 |
- print "Lock1 write lock" |
454 |
- |
455 |
- lock_work() |
456 |
- |
457 |
- Lock1.unlock() |
458 |
- print "Lock1 unlock" |
459 |
- |
460 |
- Lock1.read_lock() |
461 |
- print "Lock1 read lock" |
462 |
- |
463 |
- lock_work() |
464 |
- |
465 |
- Lock1.unlock() |
466 |
- print "Lock1 unlock" |
467 |
- |
468 |
- Lock1.read_lock() |
469 |
- print "Lock1 read lock" |
470 |
- |
471 |
- Lock1.write_lock() |
472 |
- print "Lock1 write lock" |
473 |
- |
474 |
- lock_work() |
475 |
- |
476 |
- Lock1.unlock() |
477 |
- print "Lock1 unlock" |
478 |
- |
479 |
- Lock1.read_lock() |
480 |
- print "Lock1 read lock" |
481 |
- |
482 |
- lock_work() |
483 |
- |
484 |
- Lock1.unlock() |
485 |
- print "Lock1 unlock" |
486 |
- |
487 |
-#Lock1.write_lock() |
488 |
-#time.sleep(2) |
489 |
-#Lock1.unlock() |
490 |
- ##Lock1.write_lock() |
491 |
- #time.sleep(2) |
492 |
- #Lock1.unlock() |
493 |
+ # Releasing a write lock is the same as a read lock. |
494 |
+ self.lock.release_write_lock() |
495 |
-- |
496 |
2.5.2 |