| 1 |
On Wednesday 16 November 2005 16:16, Alec Warner wrote:
|
| 2 |
> Brian asked me to split this up, and the first patch had some |
| 3 |
> cruft...and I broke things, both from old messing around. So I started |
| 4 |
> with a clean installed of rc7, hopefully these are a bit better. |
| 5 |
> |
| 6 |
> One patch is for the backend stuff, portage_modules, portage_plugins, |
| 7 |
> portage_file additions, portage_lock additions, portage_const additions |
| 8 |
> and portage_data additions. |
| 9 |
> |
| 10 |
> The second patch is the addition of register_plugin.py. |
| 11 |
|
| 12 |
Err.. That's not very split up. ;)
|
| 13 |
|
| 14 |
|
| 15 |
+++ ./pym/portage_file.py 2005-11-16 01:56:19.157073160 -0500
|
| 16 |
|
| 17 |
+ # if the dir perms would lack +wx, we have to force it
|
| 18 |
+ force_temp_perms = ((mode & 0300) != 0300)
|
| 19 |
+ resets = []
|
| 20 |
+ apath = normpath(os.path.abspath(path))
|
| 21 |
+ sticky_parent = False
|
| 22 |
+ creating = False
|
| 23 |
+
|
| 24 |
+ for dir in apath.split(os.path.sep):
|
| 25 |
+ base = os.path.join(base,dir)
|
| 26 |
+ try:
|
| 27 |
+ st = os.stat(base)
|
| 28 |
+ # something exists. why are we doing isdir?
|
| 29 |
+ if not stat.S_ISDIR(st.st_mode):
|
| 30 |
+ return False
|
| 31 |
|
| 32 |
What's with the comment here?
|
| 33 |
|
| 34 |
+ # if it's a subdir, we need +wx at least
|
| 35 |
+ if apath != base:
|
| 36 |
+ if ((st.st_mode & 0300) != 0300):
|
| 37 |
+ try: os.chmod(base, (st.st_mode | 0300))
|
| 38 |
+ except OSError: return False
|
| 39 |
+ resets.append((base, st.st_mode))
|
| 40 |
+ sticky_parent = (st.st_gid & stat.S_ISGID)
|
| 41 |
|
| 42 |
<snip>
|
| 43 |
|
| 44 |
+ try:
|
| 45 |
+ for base, m in reversed(resets):
|
| 46 |
+ os.chmod(base, m)
|
| 47 |
+ if uid != -1 or gid != -1:
|
| 48 |
+ os.chown(base, uid, gid)
|
| 49 |
+ except OSError:
|
| 50 |
+ return False
|
| 51 |
|
| 52 |
If for some reason the parent dir originally has perms 0000, this code will
|
| 53 |
incorrectly change its uid and gid.
|
| 54 |
|
| 55 |
+++ ./pym/portage_locks.py 2005-11-16 01:56:25.152161768 -0500
|
| 56 |
@@ -358,3 +359,91 @@
|
| 57 |
|
| 58 |
return results
|
| 59 |
|
| 60 |
+class LockException(Exception):
|
| 61 |
+ """Base lock exception class"""
|
| 62 |
+ def __init__(self, path, reason):
|
| 63 |
+ self.path, self.reason = path, reason
|
| 64 |
+
|
| 65 |
+class NonExistant(LockException):
|
| 66 |
+ """Missing file/dir exception"""
|
| 67 |
+ def __init__(self, path, reason=None):
|
| 68 |
+ LockException.__init__(self, path, reason)
|
| 69 |
+ def __str__(self):
|
| 70 |
+ return "Lock action for '%s' failed due to not being a valid dir/file %s" % (self.path, self.reason)
|
| 71 |
+
|
| 72 |
+class GenericFailed(LockException):
|
| 73 |
+ """the fallback lock exception class- covers perms, IOError's, and general whackyness"""
|
| 74 |
+ def __str__(self):
|
| 75 |
+ return "Lock action for '%s' failed due to '%s'" % (self.path, self.reason)
|
| 76 |
|
| 77 |
This should really be moved to portage_exception.
|
| 78 |
|
| 79 |
+# should the fd be left open indefinitely?
|
| 80 |
+# IMO, it shouldn't, but opening/closing everytime around is expensive
|
| 81 |
|
| 82 |
It should be closed when locks are released, I think. Otherwise the file may
|
| 83 |
be deleted and recreated while one FsLock instance has it open which could
|
| 84 |
create situations where two FsLocks are trying to lock the same path but
|
| 85 |
using different fds.
|
| 86 |
|
| 87 |
+class FsLock(object):
|
| 88 |
+ __slots__ = ["path", "fd", "create"]
|
| 89 |
+ def __init__(self, path, create=False):
|
| 90 |
+ """path specifies the fs path for the lock
|
| 91 |
+ create controls whether the file will be create if the file doesn't exist
|
| 92 |
+ if create is true, the base dir must exist, and it will create a file
|
| 93 |
+
|
| 94 |
+ If you want a directory yourself, create it.
|
| 95 |
+ """
|
| 96 |
+ self.path = path
|
| 97 |
+ self.fd = None
|
| 98 |
+ self.create = create
|
| 99 |
+ if not create:
|
| 100 |
+ if not os.path.exists(path):
|
| 101 |
+ raise NonExistant(path)
|
| 102 |
+
|
| 103 |
+ def _acquire_fd(self):
|
| 104 |
+ if self.create:
|
| 105 |
+ try: self.fd = os.open(self.path, os.R_OK|os.O_CREAT)
|
| 106 |
+ except OSError, oe:
|
| 107 |
+ raise GenericFailed(self.path, oe)
|
| 108 |
+ else:
|
| 109 |
+ try: self.fd = os.open(self.path, os.R_OK)
|
| 110 |
+ except OSError, oe: raise NonExistant(self.path, oe)
|
| 111 |
+
|
| 112 |
+ def _enact_change(self, flags, blocking):
|
| 113 |
+ if self.fd == None:
|
| 114 |
+ self._acquire_fd()
|
| 115 |
+ # we do it this way, due to the fact try/except is a bit of a hit
|
| 116 |
|
| 117 |
Instead of?
|
| 118 |
|
| 119 |
+ if not blocking:
|
| 120 |
+ try: fcntl.flock(self.fd, flags|fcntl.LOCK_NB)
|
| 121 |
+ except IOError, ie:
|
| 122 |
+ if ie.errno == 11:
|
| 123 |
|
| 124 |
11 ?
|
| 125 |
|
| 126 |
+ return False
|
| 127 |
+ raise GenericFailed(self.path, ie)
|
| 128 |
+ else:
|
| 129 |
+ fcntl.flock(self.fd, flags)
|
| 130 |
+ return True
|
| 131 |
+
|
| 132 |
+ def acquire_write_lock(self, blocking=True):
|
| 133 |
+ """Acquire an exclusive lock
|
| 134 |
+ Returns True if lock is acquired, False if not.
|
| 135 |
+ Note if you have a read lock, it implicitly upgrades atomically"""
|
| 136 |
+ return self._enact_change(fcntl.LOCK_EX, blocking)
|
| 137 |
+
|
| 138 |
+ def acquire_read_lock(self, blocking=True):
|
| 139 |
+ """Acquire a shared lock
|
| 140 |
+ Returns True if lock is acquired, False if not.
|
| 141 |
+ Note, if you have a write_lock already, it'll implicitly downgrade atomically"""
|
| 142 |
+ return self._enact_change(fcntl.LOCK_SH, blocking)
|
| 143 |
+
|
| 144 |
+ def release_write_lock(self):
|
| 145 |
+ """Release an exclusive lock if held"""
|
| 146 |
+ self._enact_change(fcntl.LOCK_UN, False)
|
| 147 |
+
|
| 148 |
+ def release_read_lock(self):
|
| 149 |
+ self._enact_change(fcntl.LOCK_UN, False)
|
| 150 |
+
|
| 151 |
+ def __del__(self):
|
| 152 |
+ # alright, it's 5:45am, yes this is weird code.
|
| 153 |
+ try:
|
| 154 |
+ if self.fd != None:
|
| 155 |
+ self.release_read_lock()
|
| 156 |
+ finally:
|
| 157 |
+ if self.fd != None:
|
| 158 |
+ os.close(self.fd)
|
| 159 |
|
| 160 |
Allowing an exception to be raised during __del__ is not a good thing...
|
| 161 |
Any held lock should be cleaned up too otherwise misbehaving callers could
|
| 162 |
deadlock.
|
| 163 |
|
| 164 |
Locking should probably be separated out from this patch though. If this
|
| 165 |
needs to be added, the existing locking should really be removed.
|
| 166 |
|
| 167 |
+++ ./pym/portage_modules.py 2005-11-16 01:56:28.269687832 -0500
|
| 168 |
|
| 169 |
+class FailedImport(ImportError):
|
| 170 |
+ def __init__(self, trg, e): self.trg, self.e = trg, e
|
| 171 |
+ def __str__(self): return "Failed importing target '%s': '%s'" % (self.trg, self.e)
|
| 172 |
|
| 173 |
As above. Should really stick with the python convention of suffixing with
|
| 174 |
Error or Exception as well. "from portage_modules import FailedImport" is
|
| 175 |
a bit unclear...
|
| 176 |
|
| 177 |
+__import_lock = threading.Lock()
|
| 178 |
+
|
| 179 |
+def load_module(name):
|
| 180 |
+ """load a module, throwing a FailedImport if __import__ fails"""
|
| 181 |
+ __import_lock.acquire()
|
| 182 |
+ try:
|
| 183 |
+ if name in sys.modules:
|
| 184 |
+ return sys.modules[name]
|
| 185 |
+ try:
|
| 186 |
+ m = __import__(name)
|
| 187 |
+ nl = name.split('.')
|
| 188 |
+ # __import__ returns nl[0]... so.
|
| 189 |
+ nl.pop(0)
|
| 190 |
+ while len(nl):
|
| 191 |
+ m = getattr(m, nl[0])
|
| 192 |
+ nl.pop(0)
|
| 193 |
+ return m
|
| 194 |
+ # * ferringb|afk hit it at some point, sure of that
|
| 195 |
+ # but no idea how, so commenting out to see if things break...
|
| 196 |
+ # except AttributeError, e:
|
| 197 |
+ # raise FailedImport(name, e)
|
| 198 |
|
| 199 |
I can't see how either, but comments like these don't really help. Please
|
| 200 |
leave them out unless they describe how the code works.
|
| 201 |
|
| 202 |
+++ ./pym/portage_plugins.py 2005-11-16 01:56:51.419168576 -0500
|
| 203 |
@@ -0,0 +1,174 @@
|
| 204 |
+# Copyright: 2005 Gentoo Foundation
|
| 205 |
+# License: GPL2
|
| 206 |
+# $Id:$
|
| 207 |
+
|
| 208 |
+# I don't like this.
|
| 209 |
+# doesn't seem clean/right.
|
| 210 |
|
| 211 |
???
|
| 212 |
|
| 213 |
+PLUGINS_EXTENSION = ".plugins"
|
| 214 |
|
| 215 |
Constants should go in portage_const.
|
| 216 |
|
| 217 |
+class RegistrationException(Exception):
|
| 218 |
+ def __init__(self, reason): self.reason = reason
|
| 219 |
+ def __str__(self): return "failed action due to %s" % self.reason
|
| 220 |
+
|
| 221 |
+class FailedDir(RegistrationException):
|
| 222 |
+ pass
|
| 223 |
+
|
| 224 |
+class PluginExistsAlready(RegistrationException):
|
| 225 |
+ def __init__(self):
|
| 226 |
+ RegistrationException.__init__(self, "plugin exists aleady, magic found")
|
| 227 |
+
|
| 228 |
+class FailedUpdating(RegistrationException):
|
| 229 |
+ def __str__(self):
|
| 230 |
+ return "failed updating plugin_type due error: %s" % (self.reason)
|
| 231 |
+
|
| 232 |
+class PluginNotFound(RegistrationException):
|
| 233 |
+ def __init__(self, plugin, reason="unknown"):
|
| 234 |
+ self.plugin, self.reason = plugin, reason
|
| 235 |
+ def __str__(self):
|
| 236 |
+ return "Plugin '%s' wasn't found; reason: %s" % (self.plugin, self.reason)
|
| 237 |
|
| 238 |
Suffixes and portage_exception.
|
| 239 |
|
| 240 |
+ # this could be fine grained down to per plugin_type
|
| 241 |
+ plug_lock = FsLock(plugins_dir)
|
| 242 |
|
| 243 |
Then it probably should be. ;)
|
| 244 |
|
| 245 |
+ def query_plugins(self, plugin_type=None, locking=True, raw=False):
|
| 246 |
|
| 247 |
Not sure about allowing the external callers to disable locking... That
|
| 248 |
should only be used by register and deregister, right? Perhaps breaking
|
| 249 |
the unlocked version into a "protected" method?
|
| 250 |
|
| 251 |
+ # designed this way to minimize lock holding time.
|
| 252 |
+ if plugin_type != None:
|
| 253 |
+ ptypes = [plugin_type + PLUGINS_EXTENSION]
|
| 254 |
+ d = {}
|
| 255 |
+ if locking:
|
| 256 |
+ try:
|
| 257 |
+ plug_lock = FsLock(plugins_dir)
|
| 258 |
+ except NonExistant:
|
| 259 |
+ return {}
|
| 260 |
+ plug_lock.acquire_read_lock()
|
| 261 |
+ try:
|
| 262 |
+ if plugin_type == None:
|
| 263 |
+ ptypes = [x for x in os.listdir(plugins_dir) if x.endswith(PLUGINS_EXTENSION) ]
|
| 264 |
|
| 265 |
The plugin_type if/else could be combined as ptypes is not used before this.
|
| 266 |
Is [x for x in list] 2.2 safe by the way?
|
| 267 |
|
| 268 |
+ len_exten = len(PLUGINS_EXTENSION)
|
| 269 |
+ for x in ptypes:
|
| 270 |
+ c = RawConfigParser()
|
| 271 |
+ c.read(os.path.join(plugins_dir, x.lstrip(os.path.sep)))
|
| 272 |
+ if raw:
|
| 273 |
+ d[x[:-len_exten]] = c
|
| 274 |
+ else:
|
| 275 |
+ d[x[:-len_exten]] = dict([(y, dict(c.items(y))) for y in c.sections()])
|
| 276 |
|
| 277 |
Same here on the 2.2 bit... Although that's pretty hard to read either way.
|
| 278 |
|
| 279 |
+ finally:
|
| 280 |
+ if locking:
|
| 281 |
+ plug_lock.release_read_lock()
|
| 282 |
+ if plugin_type != None:
|
| 283 |
+ return d[plugin_type]
|
| 284 |
+ return d
|
| 285 |
|
| 286 |
Not sure about having different return types...
|
| 287 |
|
| 288 |
+registry = None
|
| 289 |
+from portage.util.currying import pre_curry, pretty_docs
|
| 290 |
+
|
| 291 |
+def proxy_it(method, *a, **kw):
|
| 292 |
+ global registry
|
| 293 |
+ if registry == None:
|
| 294 |
+ registry = GlobalPluginRegistry()
|
| 295 |
+ return getattr(registry, method)(*a, **kw)
|
| 296 |
+
|
| 297 |
+for x in ["register", "deregister", "query_plugins", "get_plugin"]:
|
| 298 |
+ v = pre_curry(proxy_it, x)
|
| 299 |
+ doc = getattr(GlobalPluginRegistry, x).__doc__
|
| 300 |
+ if doc == None:
|
| 301 |
+ doc = ''
|
| 302 |
+ else:
|
| 303 |
+ # do this so indentation on pydoc __doc__ is sane
|
| 304 |
+ doc = "\n".join(map(lambda x:x.lstrip(), doc.split("\n"))) +"\n"
|
| 305 |
+ doc += "proxied call to module level registry instances %s method" % x
|
| 306 |
+ globals()[x] = pretty_docs(v, doc)
|
| 307 |
+
|
| 308 |
+del x, v, proxy_it, doc
|
| 309 |
|
| 310 |
What's this currying and pretty_docs stuff?
|
| 311 |
|
| 312 |
--
|
| 313 |
Jason Stubbs
|
| 314 |
|
| 315 |
--
|
| 316 |
gentoo-portage-dev@g.o mailing list |
Archives