OLD | NEW |
| (Empty) |
1 # -*- test-case-name: twisted.test.test_paths -*- | |
2 # Copyright (c) 2001-2008 Twisted Matrix Laboratories. | |
3 # See LICENSE for details. | |
4 | |
5 """ | |
6 Object-oriented filesystem path representation. | |
7 """ | |
8 | |
9 import os | |
10 import errno | |
11 import hashlib | |
12 import random | |
13 import base64 | |
14 | |
15 from os.path import isabs, exists, normpath, abspath, splitext | |
16 from os.path import basename, dirname | |
17 from os.path import join as joinpath | |
18 from os import sep as slash | |
19 from os import listdir, utime, stat | |
20 | |
21 from stat import S_ISREG, S_ISDIR | |
22 | |
23 # Please keep this as light as possible on other Twisted imports; many, many | |
24 # things import this module, and it would be good if it could easily be | |
25 # modified for inclusion in the standard library. --glyph | |
26 | |
27 from twisted.python.runtime import platform | |
28 | |
29 from twisted.python.win32 import ERROR_FILE_NOT_FOUND, ERROR_PATH_NOT_FOUND | |
30 from twisted.python.win32 import ERROR_INVALID_NAME, ERROR_DIRECTORY | |
31 from twisted.python.win32 import WindowsError | |
32 | |
33 def _stub_islink(path): | |
34 """ | |
35 Always return 'false' if the operating system does not support symlinks. | |
36 | |
37 @param path: a path string. | |
38 @type path: L{str} | |
39 @return: false | |
40 """ | |
41 return False | |
42 | |
43 | |
44 def _stub_urandom(n): | |
45 """ | |
46 Provide random data in versions of Python prior to 2.4. This is an | |
47 effectively compatible replacement for 'os.urandom'. | |
48 | |
49 @type n: L{int} | |
50 @param n: the number of bytes of data to return | |
51 @return: C{n} bytes of random data. | |
52 @rtype: str | |
53 """ | |
54 randomData = [random.randrange(256) for n in xrange(n)] | |
55 return ''.join(map(chr, randomData)) | |
56 | |
57 | |
58 def _stub_armor(s): | |
59 """ | |
60 ASCII-armor for random data. This uses a hex encoding, although we will | |
61 prefer url-safe base64 encoding for features in this module if it is | |
62 available. | |
63 """ | |
64 return s.encode('hex') | |
65 | |
66 islink = getattr(os.path, 'islink', _stub_islink) | |
67 randomBytes = getattr(os, 'urandom', _stub_urandom) | |
68 armor = getattr(base64, 'urlsafe_b64encode', _stub_armor) | |
69 | |
70 class InsecurePath(Exception): | |
71 pass | |
72 | |
73 | |
74 class UnlistableError(OSError): | |
75 """ | |
76 An exception which is used to distinguish between errors which mean 'this | |
77 is not a directory you can list' and other, more catastrophic errors. | |
78 | |
79 This error will try to look as much like the original error as possible, | |
80 while still being catchable as an independent type. | |
81 | |
82 @ivar originalException: the actual original exception instance, either an | |
83 L{OSError} or a L{WindowsError}. | |
84 """ | |
85 def __init__(self, originalException): | |
86 """ | |
87 Create an UnlistableError exception. | |
88 | |
89 @param originalException: an instance of OSError. | |
90 """ | |
91 self.__dict__.update(originalException.__dict__) | |
92 self.originalException = originalException | |
93 | |
94 | |
95 | |
96 class _WindowsUnlistableError(UnlistableError, WindowsError): | |
97 """ | |
98 This exception is raised on Windows, for compatibility with previous | |
99 releases of FilePath where unportable programs may have done "except | |
100 WindowsError:" around a call to children(). | |
101 | |
102 It is private because all application code may portably catch | |
103 L{UnlistableError} instead. | |
104 """ | |
105 | |
106 | |
107 | |
108 def _secureEnoughString(): | |
109 """ | |
110 Create a pseudorandom, 16-character string for use in secure filenames. | |
111 """ | |
112 return armor(hashlib.sha1(randomBytes(64)).digest())[:16] | |
113 | |
114 class _PathHelper: | |
115 """ | |
116 Abstract helper class also used by ZipPath; implements certain utility metho
ds. | |
117 """ | |
118 | |
119 def getContent(self): | |
120 return self.open().read() | |
121 | |
122 def children(self): | |
123 """ | |
124 List the chilren of this path object. | |
125 | |
126 @raise OSError: If an error occurs while listing the directory. If the | |
127 error is 'serious', meaning that the operation failed due to an access | |
128 violation, exhaustion of some kind of resource (file descriptors or | |
129 memory), OSError or a platform-specific variant will be raised. | |
130 | |
131 @raise UnlistableError: If the inability to list the directory is due | |
132 to this path not existing or not being a directory, the more specific | |
133 OSError subclass L{UnlistableError} is raised instead. | |
134 | |
135 @return: an iterable of all currently-existing children of this object | |
136 accessible with L{_PathHelper.child}. | |
137 """ | |
138 try: | |
139 subnames = self.listdir() | |
140 except WindowsError, winErrObj: | |
141 # WindowsError is an OSError subclass, so if not for this clause | |
142 # the OSError clause below would be handling these. Windows error | |
143 # codes aren't the same as POSIX error codes, so we need to handle | |
144 # them differently. | |
145 | |
146 # Under Python 2.5 on Windows, WindowsError has a winerror | |
147 # attribute and an errno attribute. The winerror attribute is | |
148 # bound to the Windows error code while the errno attribute is | |
149 # bound to a translation of that code to a perhaps equivalent POSIX | |
150 # error number. | |
151 | |
152 # Under Python 2.4 on Windows, WindowsError only has an errno | |
153 # attribute. It is bound to the Windows error code. | |
154 | |
155 # For simplicity of code and to keep the number of paths through | |
156 # this suite minimal, we grab the Windows error code under either | |
157 # version. | |
158 | |
159 # Furthermore, attempting to use os.listdir on a non-existent path | |
160 # in Python 2.4 will result in a Windows error code of | |
161 # ERROR_PATH_NOT_FOUND. However, in Python 2.5, | |
162 # ERROR_FILE_NOT_FOUND results instead. -exarkun | |
163 winerror = getattr(winErrObj, 'winerror', winErrObj.errno) | |
164 if winerror not in (ERROR_PATH_NOT_FOUND, | |
165 ERROR_FILE_NOT_FOUND, | |
166 ERROR_INVALID_NAME, | |
167 ERROR_DIRECTORY): | |
168 raise | |
169 raise _WindowsUnlistableError(winErrObj) | |
170 except OSError, ose: | |
171 if ose.errno not in (errno.ENOENT, errno.ENOTDIR): | |
172 # Other possible errors here, according to linux manpages: | |
173 # EACCES, EMIFLE, ENFILE, ENOMEM. None of these seem like the | |
174 # sort of thing which should be handled normally. -glyph | |
175 raise | |
176 raise UnlistableError(ose) | |
177 return map(self.child, subnames) | |
178 | |
179 def walk(self): | |
180 """ | |
181 Yield myself, then each of my children, and each of those children's | |
182 children in turn. | |
183 | |
184 @return: a generator yielding FilePath-like objects. | |
185 """ | |
186 yield self | |
187 if self.isdir(): | |
188 for c in self.children(): | |
189 for subc in c.walk(): | |
190 yield subc | |
191 | |
192 def sibling(self, path): | |
193 return self.parent().child(path) | |
194 | |
195 def segmentsFrom(self, ancestor): | |
196 """ | |
197 Return a list of segments between a child and its ancestor. | |
198 | |
199 For example, in the case of a path X representing /a/b/c/d and a path Y | |
200 representing /a/b, C{Y.segmentsFrom(X)} will return C{['c', | |
201 'd']}. | |
202 | |
203 @param ancestor: an instance of the same class as self, ostensibly an | |
204 ancestor of self. | |
205 | |
206 @raise: ValueError if the 'ancestor' parameter is not actually an | |
207 ancestor, i.e. a path for /x/y/z is passed as an ancestor for /a/b/c/d. | |
208 | |
209 @return: a list of strs | |
210 """ | |
211 # this might be an unnecessarily inefficient implementation but it will | |
212 # work on win32 and for zipfiles; later I will deterimine if the | |
213 # obvious fast implemenation does the right thing too | |
214 f = self | |
215 p = f.parent() | |
216 segments = [] | |
217 while f != ancestor and p != f: | |
218 segments[0:0] = [f.basename()] | |
219 f = p | |
220 p = p.parent() | |
221 if f == ancestor and segments: | |
222 return segments | |
223 raise ValueError("%r not parent of %r" % (ancestor, self)) | |
224 | |
225 | |
226 # new in 8.0 | |
227 def __hash__(self): | |
228 """ | |
229 Hash the same as another FilePath with the same path as mine. | |
230 """ | |
231 return hash((self.__class__, self.path)) | |
232 | |
233 | |
234 # pending deprecation in 8.0 | |
235 def getmtime(self): | |
236 """ | |
237 Deprecated. Use getModificationTime instead. | |
238 """ | |
239 return int(self.getModificationTime()) | |
240 | |
241 | |
242 def getatime(self): | |
243 """ | |
244 Deprecated. Use getAccessTime instead. | |
245 """ | |
246 return int(self.getAccessTime()) | |
247 | |
248 | |
249 def getctime(self): | |
250 """ | |
251 Deprecated. Use getStatusChangeTime instead. | |
252 """ | |
253 return int(self.getStatusChangeTime()) | |
254 | |
255 | |
256 | |
257 class FilePath(_PathHelper): | |
258 """ | |
259 I am a path on the filesystem that only permits 'downwards' access. | |
260 | |
261 Instantiate me with a pathname (for example, | |
262 FilePath('/home/myuser/public_html')) and I will attempt to only provide | |
263 access to files which reside inside that path. I may be a path to a file, | |
264 a directory, or a file which does not exist. | |
265 | |
266 The correct way to use me is to instantiate me, and then do ALL filesystem | |
267 access through me. In other words, do not import the 'os' module; if you | |
268 need to open a file, call my 'open' method. If you need to list a | |
269 directory, call my 'path' method. | |
270 | |
271 Even if you pass me a relative path, I will convert that to an absolute | |
272 path internally. | |
273 | |
274 Note: although time-related methods do return floating-point results, they | |
275 may still be only second resolution depending on the platform and the last | |
276 value passed to L{os.stat_float_times}. If you want greater-than-second | |
277 precision, call C{os.stat_float_times(True)}, or use Python 2.5. | |
278 Greater-than-second precision is only available in Windows on Python2.5 and | |
279 later. | |
280 | |
281 @type alwaysCreate: C{bool} | |
282 @ivar alwaysCreate: When opening this file, only succeed if the file does no
t | |
283 already exist. | |
284 """ | |
285 | |
286 statinfo = None | |
287 path = None | |
288 | |
289 def __init__(self, path, alwaysCreate=False): | |
290 self.path = abspath(path) | |
291 self.alwaysCreate = alwaysCreate | |
292 | |
293 def __getstate__(self): | |
294 d = self.__dict__.copy() | |
295 if d.has_key('statinfo'): | |
296 del d['statinfo'] | |
297 return d | |
298 | |
299 def child(self, path): | |
300 if platform.isWindows() and path.count(":"): | |
301 # Catch paths like C:blah that don't have a slash | |
302 raise InsecurePath("%r contains a colon." % (path,)) | |
303 norm = normpath(path) | |
304 if slash in norm: | |
305 raise InsecurePath("%r contains one or more directory separators" %
(path,)) | |
306 newpath = abspath(joinpath(self.path, norm)) | |
307 if not newpath.startswith(self.path): | |
308 raise InsecurePath("%r is not a child of %s" % (newpath, self.path)) | |
309 return self.clonePath(newpath) | |
310 | |
311 def preauthChild(self, path): | |
312 """ | |
313 Use me if `path' might have slashes in it, but you know they're safe. | |
314 | |
315 (NOT slashes at the beginning. It still needs to be a _child_). | |
316 """ | |
317 newpath = abspath(joinpath(self.path, normpath(path))) | |
318 if not newpath.startswith(self.path): | |
319 raise InsecurePath("%s is not a child of %s" % (newpath, self.path)) | |
320 return self.clonePath(newpath) | |
321 | |
322 def childSearchPreauth(self, *paths): | |
323 """Return my first existing child with a name in 'paths'. | |
324 | |
325 paths is expected to be a list of *pre-secured* path fragments; in most | |
326 cases this will be specified by a system administrator and not an | |
327 arbitrary user. | |
328 | |
329 If no appropriately-named children exist, this will return None. | |
330 """ | |
331 p = self.path | |
332 for child in paths: | |
333 jp = joinpath(p, child) | |
334 if exists(jp): | |
335 return self.clonePath(jp) | |
336 | |
337 def siblingExtensionSearch(self, *exts): | |
338 """Attempt to return a path with my name, given multiple possible | |
339 extensions. | |
340 | |
341 Each extension in exts will be tested and the first path which exists | |
342 will be returned. If no path exists, None will be returned. If '' is | |
343 in exts, then if the file referred to by this path exists, 'self' will | |
344 be returned. | |
345 | |
346 The extension '*' has a magic meaning, which means "any path that | |
347 begins with self.path+'.' is acceptable". | |
348 """ | |
349 p = self.path | |
350 for ext in exts: | |
351 if not ext and self.exists(): | |
352 return self | |
353 if ext == '*': | |
354 basedot = basename(p)+'.' | |
355 for fn in listdir(dirname(p)): | |
356 if fn.startswith(basedot): | |
357 return self.clonePath(joinpath(dirname(p), fn)) | |
358 p2 = p + ext | |
359 if exists(p2): | |
360 return self.clonePath(p2) | |
361 | |
362 def siblingExtension(self, ext): | |
363 return self.clonePath(self.path+ext) | |
364 | |
365 | |
366 def linkTo(self, linkFilePath): | |
367 """ | |
368 Creates a symlink to self to at the path in the L{FilePath} | |
369 C{linkFilePath}. Only works on posix systems due to its dependence on | |
370 C{os.symlink}. Propagates C{OSError}s up from C{os.symlink} if | |
371 C{linkFilePath.parent()} does not exist, or C{linkFilePath} already | |
372 exists. | |
373 | |
374 @param linkFilePath: a FilePath representing the link to be created | |
375 @type linkFilePath: L{FilePath} | |
376 """ | |
377 os.symlink(self.path, linkFilePath.path) | |
378 | |
379 | |
380 def open(self, mode='r'): | |
381 if self.alwaysCreate: | |
382 assert 'a' not in mode, "Appending not supported when alwaysCreate =
= True" | |
383 return self.create() | |
384 return open(self.path, mode+'b') | |
385 | |
386 # stat methods below | |
387 | |
388 def restat(self, reraise=True): | |
389 """ | |
390 Re-calculate cached effects of 'stat'. To refresh information on this p
ath | |
391 after you know the filesystem may have changed, call this method. | |
392 | |
393 @param reraise: a boolean. If true, re-raise exceptions from | |
394 L{os.stat}; otherwise, mark this path as not existing, and remove any | |
395 cached stat information. | |
396 """ | |
397 try: | |
398 self.statinfo = stat(self.path) | |
399 except OSError: | |
400 self.statinfo = 0 | |
401 if reraise: | |
402 raise | |
403 | |
404 | |
405 def chmod(self, mode): | |
406 """ | |
407 Changes the permissions on self, if possible. Propagates errors from | |
408 C{os.chmod} up. | |
409 | |
410 @param mode: integer representing the new permissions desired (same as | |
411 the command line chmod) | |
412 @type mode: C{int} | |
413 """ | |
414 os.chmod(self.path, mode) | |
415 | |
416 | |
417 def getsize(self): | |
418 st = self.statinfo | |
419 if not st: | |
420 self.restat() | |
421 st = self.statinfo | |
422 return st.st_size | |
423 | |
424 | |
425 def getModificationTime(self): | |
426 """ | |
427 Retrieve the time of last access from this file. | |
428 | |
429 @return: a number of seconds from the epoch. | |
430 @rtype: float | |
431 """ | |
432 st = self.statinfo | |
433 if not st: | |
434 self.restat() | |
435 st = self.statinfo | |
436 return float(st.st_mtime) | |
437 | |
438 | |
439 def getStatusChangeTime(self): | |
440 """ | |
441 Retrieve the time of the last status change for this file. | |
442 | |
443 @return: a number of seconds from the epoch. | |
444 @rtype: float | |
445 """ | |
446 st = self.statinfo | |
447 if not st: | |
448 self.restat() | |
449 st = self.statinfo | |
450 return float(st.st_ctime) | |
451 | |
452 | |
453 def getAccessTime(self): | |
454 """ | |
455 Retrieve the time that this file was last accessed. | |
456 | |
457 @return: a number of seconds from the epoch. | |
458 @rtype: float | |
459 """ | |
460 st = self.statinfo | |
461 if not st: | |
462 self.restat() | |
463 st = self.statinfo | |
464 return float(st.st_atime) | |
465 | |
466 | |
467 def exists(self): | |
468 """ | |
469 Check if the C{path} exists. | |
470 | |
471 @return: C{True} if the stats of C{path} can be retrieved successfully, | |
472 C{False} in the other cases. | |
473 @rtype: C{bool} | |
474 """ | |
475 if self.statinfo: | |
476 return True | |
477 else: | |
478 self.restat(False) | |
479 if self.statinfo: | |
480 return True | |
481 else: | |
482 return False | |
483 | |
484 | |
485 def isdir(self): | |
486 st = self.statinfo | |
487 if not st: | |
488 self.restat(False) | |
489 st = self.statinfo | |
490 if not st: | |
491 return False | |
492 return S_ISDIR(st.st_mode) | |
493 | |
494 def isfile(self): | |
495 st = self.statinfo | |
496 if not st: | |
497 self.restat(False) | |
498 st = self.statinfo | |
499 if not st: | |
500 return False | |
501 return S_ISREG(st.st_mode) | |
502 | |
503 def islink(self): | |
504 # We can't use cached stat results here, because that is the stat of | |
505 # the destination - (see #1773) which in *every case* but this one is | |
506 # the right thing to use. We could call lstat here and use that, but | |
507 # it seems unlikely we'd actually save any work that way. -glyph | |
508 return islink(self.path) | |
509 | |
510 def isabs(self): | |
511 return isabs(self.path) | |
512 | |
513 def listdir(self): | |
514 return listdir(self.path) | |
515 | |
516 def splitext(self): | |
517 return splitext(self.path) | |
518 | |
519 def __repr__(self): | |
520 return 'FilePath(%r)' % (self.path,) | |
521 | |
522 def touch(self): | |
523 try: | |
524 self.open('a').close() | |
525 except IOError: | |
526 pass | |
527 utime(self.path, None) | |
528 | |
529 def remove(self): | |
530 """ | |
531 Removes the file or directory that is represented by self. If | |
532 C{self.path} is a directory, recursively remove all its children | |
533 before removing the directory. If it's a file or link, just delete | |
534 it. | |
535 """ | |
536 if self.isdir() and not self.islink(): | |
537 for child in self.children(): | |
538 child.remove() | |
539 os.rmdir(self.path) | |
540 else: | |
541 os.remove(self.path) | |
542 self.restat(False) | |
543 | |
544 | |
545 def makedirs(self): | |
546 """ | |
547 Create all directories not yet existing in C{path} segments, using | |
548 C{os.makedirs}. | |
549 """ | |
550 return os.makedirs(self.path) | |
551 | |
552 | |
553 def globChildren(self, pattern): | |
554 """ | |
555 Assuming I am representing a directory, return a list of | |
556 FilePaths representing my children that match the given | |
557 pattern. | |
558 """ | |
559 import glob | |
560 path = self.path[-1] == '/' and self.path + pattern or slash.join([self.
path, pattern]) | |
561 return map(self.clonePath, glob.glob(path)) | |
562 | |
563 def basename(self): | |
564 return basename(self.path) | |
565 | |
566 def dirname(self): | |
567 return dirname(self.path) | |
568 | |
569 def parent(self): | |
570 return self.clonePath(self.dirname()) | |
571 | |
572 def setContent(self, content, ext='.new'): | |
573 sib = self.siblingExtension(ext) | |
574 sib.open('w').write(content) | |
575 if platform.isWindows() and exists(self.path): | |
576 os.unlink(self.path) | |
577 os.rename(sib.path, self.path) | |
578 | |
579 # new in 2.2.0 | |
580 | |
581 def __cmp__(self, other): | |
582 if not isinstance(other, FilePath): | |
583 return NotImplemented | |
584 return cmp(self.path, other.path) | |
585 | |
586 def createDirectory(self): | |
587 os.mkdir(self.path) | |
588 | |
589 def requireCreate(self, val=1): | |
590 self.alwaysCreate = val | |
591 | |
592 def create(self): | |
593 """Exclusively create a file, only if this file previously did not exist
. | |
594 """ | |
595 fdint = os.open(self.path, (os.O_EXCL | | |
596 os.O_CREAT | | |
597 os.O_RDWR)) | |
598 | |
599 # XXX TODO: 'name' attribute of returned files is not mutable or | |
600 # settable via fdopen, so this file is slighly less functional than the | |
601 # one returned from 'open' by default. send a patch to Python... | |
602 | |
603 return os.fdopen(fdint, 'w+b') | |
604 | |
605 def temporarySibling(self): | |
606 """ | |
607 Create a path naming a temporary sibling of this path in a secure fashio
n. | |
608 """ | |
609 sib = self.sibling(_secureEnoughString() + self.basename()) | |
610 sib.requireCreate() | |
611 return sib | |
612 | |
613 _chunkSize = 2 ** 2 ** 2 ** 2 | |
614 | |
615 def copyTo(self, destination): | |
616 # XXX TODO: *thorough* audit and documentation of the exact desired | |
617 # semantics of this code. Right now the behavior of existent | |
618 # destination symlinks is convenient, and quite possibly correct, but | |
619 # its security properties need to be explained. | |
620 if self.isdir(): | |
621 if not destination.exists(): | |
622 destination.createDirectory() | |
623 for child in self.children(): | |
624 destChild = destination.child(child.basename()) | |
625 child.copyTo(destChild) | |
626 elif self.isfile(): | |
627 writefile = destination.open('w') | |
628 readfile = self.open() | |
629 while 1: | |
630 # XXX TODO: optionally use os.open, os.read and O_DIRECT and | |
631 # use os.fstatvfs to determine chunk sizes and make | |
632 # *****sure**** copy is page-atomic; the following is good | |
633 # enough for 99.9% of everybody and won't take a week to audit | |
634 # though. | |
635 chunk = readfile.read(self._chunkSize) | |
636 writefile.write(chunk) | |
637 if len(chunk) < self._chunkSize: | |
638 break | |
639 writefile.close() | |
640 readfile.close() | |
641 else: | |
642 # If you see the following message because you want to copy | |
643 # symlinks, fifos, block devices, character devices, or unix | |
644 # sockets, please feel free to add support to do sensible things in | |
645 # reaction to those types! | |
646 raise NotImplementedError( | |
647 "Only copying of files and directories supported") | |
648 | |
649 def moveTo(self, destination): | |
650 try: | |
651 os.rename(self.path, destination.path) | |
652 self.restat(False) | |
653 except OSError, ose: | |
654 if ose.errno == errno.EXDEV: | |
655 # man 2 rename, ubuntu linux 5.10 "breezy": | |
656 | |
657 # oldpath and newpath are not on the same mounted filesystem. | |
658 # (Linux permits a filesystem to be mounted at multiple | |
659 # points, but rename(2) does not work across different mount | |
660 # points, even if the same filesystem is mounted on both.) | |
661 | |
662 # that means it's time to copy trees of directories! | |
663 secsib = destination.temporarySibling() | |
664 self.copyTo(secsib) # slow | |
665 secsib.moveTo(destination) # visible | |
666 | |
667 # done creating new stuff. let's clean me up. | |
668 mysecsib = self.temporarySibling() | |
669 self.moveTo(mysecsib) # visible | |
670 mysecsib.remove() # slow | |
671 else: | |
672 raise | |
673 | |
674 | |
675 FilePath.clonePath = FilePath | |
OLD | NEW |