1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22 """Wrapper class around a file like "/usr/bin/env"
23
24 This class makes certain file operations more convenient and
25 associates stat information with filenames
26
27 """
28
29 import stat, errno, socket, time, re, gzip, pwd, grp
30
31 from duplicity import tarfile
32 from duplicity import file_naming
33 from duplicity import globals
34 from duplicity import gpg
35 from duplicity import util
36 from duplicity import librsync
37 from duplicity import log
38 from duplicity import dup_time
39 from duplicity.lazy import *
40
41 _copy_blocksize = 64 * 1024
42 _tmp_path_counter = 1
43
45 """Used to emulate the output of os.stat() and related"""
46
47
48 st_mode = 0
49
50
53
55 """Read only Path
56
57 Objects of this class doesn't represent real files, so they don't
58 have a name. They are required to be indexed though.
59
60 """
62 """ROPath initializer"""
63 self.opened, self.fileobj = None, None
64 self.index = index
65 self.stat, self.type = None, None
66 self.mode, self.devnums = None, None
67
69 """Set the value of self.type, self.mode from self.stat"""
70 if not self.stat:
71 self.type = None
72
73 st_mode = self.stat.st_mode
74 if stat.S_ISREG(st_mode):
75 self.type = "reg"
76 elif stat.S_ISDIR(st_mode):
77 self.type = "dir"
78 elif stat.S_ISLNK(st_mode):
79 self.type = "sym"
80 elif stat.S_ISFIFO(st_mode):
81 self.type = "fifo"
82 elif stat.S_ISSOCK(st_mode):
83 raise PathException(self.get_relative_path() +
84 "is a socket, unsupported by tar")
85 self.type = "sock"
86 elif stat.S_ISCHR(st_mode):
87 self.type = "chr"
88 elif stat.S_ISBLK(st_mode):
89 self.type = "blk"
90 else:
91 raise PathException("Unknown type")
92
93 self.mode = stat.S_IMODE(st_mode)
94 if self.type in ("chr", "blk"):
95 self.devnums = (os.major(self.stat.st_rdev),
96 os.minor(self.stat.st_rdev))
97
99 """Black out self - set type and stat to None"""
100 self.type, self.stat = None, None
101
103 """True if corresponding file exists"""
104 return self.type
105
107 """True if self corresponds to regular file"""
108 return self.type == "reg"
109
111 """True if self is dir"""
112 return self.type == "dir"
113
115 """True if self is sym"""
116 return self.type == "sym"
117
119 """True if self is fifo"""
120 return self.type == "fifo"
121
123 """True is self is socket"""
124 return self.type == "sock"
125
127 """True is self is a device file"""
128 return self.type == "chr" or self.type == "blk"
129
131 """Return device number path resides on"""
132 return self.stat.st_dev
133
135 """Return length in bytes from stat object"""
136 return self.stat.st_size
137
139 """Return mod time of path in seconds"""
140 return int(self.stat.st_mtime)
141
143 """Return relative path, created from index"""
144 if self.index:
145 return "/".join(self.index)
146 else:
147 return "."
148
150 """Return permissions mode, owner and group"""
151 s1 = self.stat
152 return '%s:%s %o' % (s1.st_uid, s1.st_gid, self.mode)
153
154 - def open(self, mode):
155 """Return fileobj associated with self"""
156 assert mode == "rb" and self.fileobj and not self.opened, \
157 "%s %s %s" % (mode, self.fileobj, self.opened)
158 self.opened = 1
159 return self.fileobj
160
162 """Return contents of associated fileobj in string"""
163 fin = self.open("rb")
164 buf = fin.read()
165 assert not fin.close()
166 return buf
167
169 """Set file object returned by open()"""
170 assert not self.fileobj
171 self.fileobj = fileobj
172 self.opened = None
173
175 """Set data from tarinfo object (part of tarfile module)"""
176
177 type = tarinfo.type
178 if type == tarfile.REGTYPE or type == tarfile.AREGTYPE:
179 self.type = "reg"
180 elif type == tarfile.LNKTYPE:
181 raise PathException("Hard links not supported yet")
182 elif type == tarfile.SYMTYPE:
183 self.type = "sym"
184 self.symtext = tarinfo.linkname
185 elif type == tarfile.CHRTYPE:
186 self.type = "chr"
187 self.devnums = (tarinfo.devmajor, tarinfo.devminor)
188 elif type == tarfile.BLKTYPE:
189 self.type = "blk"
190 self.devnums = (tarinfo.devmajor, tarinfo.devminor)
191 elif type == tarfile.DIRTYPE:
192 self.type = "dir"
193 elif type == tarfile.FIFOTYPE:
194 self.type = "fifo"
195 else:
196 raise PathException("Unknown tarinfo type %s" % (type,))
197
198 self.mode = tarinfo.mode
199 self.stat = StatResult()
200
201 """ Set user and group id
202 use numeric id if name lookup fails
203 OR
204 --numeric-owner is set
205 """
206 try:
207 if globals.numeric_owner:
208 raise KeyError
209 self.stat.st_uid = pwd.getpwnam(tarinfo.uname)[2]
210 except KeyError:
211 self.stat.st_uid = tarinfo.uid
212 try:
213 if globals.numeric_owner:
214 raise KeyError
215 self.stat.st_gid = grp.getgrnam(tarinfo.gname)[2]
216 except KeyError:
217 self.stat.st_gid = tarinfo.gid
218
219 self.stat.st_mtime = int(tarinfo.mtime)
220 if self.stat.st_mtime < 0:
221 log.Warn(_("Warning: %s has negative mtime, treating as 0.")
222 % (tarinfo.name,))
223 self.stat.st_mtime = 0
224 self.stat.st_size = tarinfo.size
225
227 """Return ropath copy of self"""
228 new_ropath = ROPath(self.index, self.stat)
229 new_ropath.type, new_ropath.mode = self.type, self.mode
230 if self.issym():
231 new_ropath.symtext = self.symtext
232 elif self.isdev():
233 new_ropath.devnums = self.devnums
234 if self.exists():
235 new_ropath.stat = self.stat
236 return new_ropath
237
239 """Generate a tarfile.TarInfo object based on self
240
241 Doesn't set size based on stat, because we may want to replace
242 data wiht other stream. Size should be set separately by
243 calling function.
244
245 """
246 ti = tarfile.TarInfo()
247 if self.index:
248 ti.name = "/".join(self.index)
249 else:
250 ti.name = "."
251 if self.isdir():
252 ti.name += "/"
253
254 ti.size = 0
255 if self.type:
256
257
258 if self.isreg():
259 ti.type = tarfile.REGTYPE
260 ti.size = self.stat.st_size
261 elif self.isdir():
262 ti.type = tarfile.DIRTYPE
263 elif self.isfifo():
264 ti.type = tarfile.FIFOTYPE
265 elif self.issym():
266 ti.type = tarfile.SYMTYPE
267 ti.linkname = self.symtext
268 elif self.isdev():
269 if self.type == "chr":
270 ti.type = tarfile.CHRTYPE
271 else:
272 ti.type = tarfile.BLKTYPE
273 ti.devmajor, ti.devminor = self.devnums
274 else:
275 raise PathException("Unrecognized type " + str(self.type))
276
277 ti.mode = self.mode
278 ti.uid, ti.gid = self.stat.st_uid, self.stat.st_gid
279 if self.stat.st_mtime < 0:
280 log.Warn(_("Warning: %s has negative mtime, treating as 0.")
281 % (self.get_relative_path(),))
282 ti.mtime = 0
283 else:
284 ti.mtime = int(self.stat.st_mtime)
285
286 try:
287 ti.uname = pwd.getpwuid(ti.uid)[0]
288 except KeyError:
289 ti.uname = ''
290 try:
291 ti.gname = grp.getgrgid(ti.gid)[0]
292 except KeyError:
293 ti.gname = ''
294
295 if ti.type in (tarfile.CHRTYPE, tarfile.BLKTYPE):
296 if hasattr(os, "major") and hasattr(os, "minor"):
297 ti.devmajor, ti.devminor = self.devnums
298 else:
299
300
301
302 ti.mode, ti.size = 0, 0
303 return ti
304
306 """Used to compare two ROPaths. Doesn't look at fileobjs"""
307 if not self.type and not other.type:
308 return 1
309 if not self.stat and other.stat or not other.stat and self.stat:
310 return 0
311 if self.type != other.type:
312 return 0
313
314 if self.isreg() or self.isdir() or self.isfifo():
315
316
317 if not self.perms_equal(other):
318 return 0
319 if int(self.stat.st_mtime) == int(other.stat.st_mtime):
320 return 1
321
322 return self.stat.st_mtime <= 0 and other.stat.st_mtime <= 0
323 elif self.issym():
324
325 return self.symtext == other.symtext
326 elif self.isdev():
327 return self.perms_equal(other) and self.devnums == other.devnums
328 assert 0
329
331 return not self.__eq__(other)
332
334 """Compare ROPaths like __eq__, but log reason if different
335
336 This is placed in a separate function from __eq__ because
337 __eq__ should be very time sensitive, and logging statements
338 would slow it down. Used when verifying.
339
340 If include_data is true, also read all the data of regular
341 files and see if they differ.
342
343 """
344 def log_diff(log_string):
345 log_str = _("Difference found:") + " " + log_string
346 log.Notice(log_str % (self.get_relative_path(),))
347
348 if not self.type and not other.type:
349 return 1
350 if not self.stat and other.stat:
351 log_diff(_("New file %s"))
352 return 0
353 if not other.stat and self.stat:
354 log_diff(_("File %s is missing"))
355 return 0
356 if self.type != other.type:
357 log_diff(_("File %%s has type %s, expected %s") %
358 (other.type, self.type))
359 return 0
360
361 if self.isreg() or self.isdir() or self.isfifo():
362 if not self.perms_equal(other):
363 log_diff(_("File %%s has permissions %s, expected %s") %
364 (other.getperms(), self.getperms()))
365 return 0
366 if ((int(self.stat.st_mtime) != int(other.stat.st_mtime)) and
367 (self.stat.st_mtime > 0 or other.stat.st_mtime > 0)):
368 log_diff(_("File %%s has mtime %s, expected %s") %
369 (dup_time.timetopretty(int(other.stat.st_mtime)),
370 dup_time.timetopretty(int(self.stat.st_mtime))))
371 return 0
372 if self.isreg() and include_data:
373 if self.compare_data(other):
374 return 1
375 else:
376 log_diff(_("Data for file %s is different"))
377 return 0
378 else:
379 return 1
380 elif self.issym():
381 if self.symtext == other.symtext:
382 return 1
383 else:
384 log_diff(_("Symlink %%s points to %s, expected %s") %
385 (other.symtext, self.symtext))
386 return 0
387 elif self.isdev():
388 if not self.perms_equal(other):
389 log_diff(_("File %%s has permissions %s, expected %s") %
390 (other.getperms(), self.getperms()))
391 return 0
392 if self.devnums != other.devnums:
393 log_diff(_("Device file %%s has numbers %s, expected %s")
394 % (other.devnums, self.devnums))
395 return 0
396 return 1
397 assert 0
398
400 """Compare data from two regular files, return true if same"""
401 f1 = self.open("rb")
402 f2 = other.open("rb")
403 def close():
404 assert not f1.close()
405 assert not f2.close()
406 while 1:
407 buf1 = f1.read(_copy_blocksize)
408 buf2 = f2.read(_copy_blocksize)
409 if buf1 != buf2:
410 close()
411 return 0
412 if not buf1:
413 close()
414 return 1
415
417 """True if self and other have same permissions and ownership"""
418 s1, s2 = self.stat, other.stat
419 return (self.mode == other.mode and
420 s1.st_gid == s2.st_gid and s1.st_uid == s2.st_uid)
421
422 - def copy(self, other):
423 """Copy self to other. Also copies data. Other must be Path"""
424 if self.isreg():
425 other.writefileobj(self.open("rb"))
426 elif self.isdir():
427 os.mkdir(other.name)
428 elif self.issym():
429 os.symlink(self.symtext, other.name)
430 os.lchown(other.name, self.stat.st_uid, self.stat.st_gid)
431 other.setdata()
432 return
433 elif self.isfifo():
434 os.mkfifo(other.name)
435 elif self.issock():
436 socket.socket(socket.AF_UNIX).bind(other.name)
437 elif self.isdev():
438 if self.type == "chr":
439 devtype = "c"
440 else:
441 devtype = "b"
442 other.makedev(devtype, *self.devnums)
443 self.copy_attribs(other)
444
460
462 """Return string representation"""
463 return "(%s %s)" % (self.index, self.type)
464
465
467 """
468 Path class - wrapper around ordinary local files
469
470 Besides caching stat() results, this class organizes various file
471 code.
472 """
473 regex_chars_to_quote = re.compile("[\\\\\\\"\\$`]")
474
487
497
499 """Refresh stat cache"""
500 try:
501 self.stat = os.lstat(self.name)
502 except OSError, e:
503 err_string = errno.errorcode[e[0]]
504 if err_string in ["ENOENT", "ENOTDIR", "ELOOP", "ENOTCONN"]:
505 self.stat, self.type = None, None
506 self.mode = None
507 else:
508 raise
509 else:
510 self.set_from_stat()
511 if self.issym():
512 self.symtext = os.readlink(self.name)
513
515 """Return new Path with ext added to index"""
516 return self.__class__(self.base, self.index + (ext,))
517
519 """Return new Path with index index"""
520 return self.__class__(self.base, index)
521
523 """Return list generated by os.listdir"""
524 return os.listdir(self.name)
525
527 """Return true if path is a directory and is empty"""
528 return self.isdir() and not self.listdir()
529
530 - def open(self, mode = "rb"):
531 """
532 Return fileobj associated with self
533
534 Usually this is just the file data on disk, but can be
535 replaced with arbitrary data using the setfileobj method.
536 """
537 assert not self.opened
538 if self.fileobj:
539 result = self.fileobj
540 else:
541 result = open(self.name, mode)
542 return result
543
544 - def makedev(self, type, major, minor):
545 """Make a device file with specified type, major/minor nums"""
546 cmdlist = ['mknod', self.name, type, str(major), str(minor)]
547 if os.spawnvp(os.P_WAIT, 'mknod', cmdlist) != 0:
548 raise PathException("Error running %s" % cmdlist)
549 self.setdata()
550
552 """Make directory(s) at specified path"""
553 log.Info(_("Making directory %s") % (self.name,))
554 try:
555 os.makedirs(self.name)
556 except OSError:
557 if (not globals.force):
558 raise PathException("Error creating directory %s" % (self.name,), 7)
559 self.setdata()
560
569
571 """Open the file, write 0 bytes, close"""
572 log.Info(_("Touching %s") % (self.name,))
573 fp = self.open("wb")
574 fp.close()
575
585
587 """Return directory that self is in"""
588 if self.index:
589 return Path(self.base, self.index[:-1])
590 else:
591 components = self.base.split("/")
592 if len(components) == 2 and not components[0]:
593 return Path("/")
594 else:
595 return Path("/".join(components[:-1]))
596
598 """Copy file object fin to self. Close both when done."""
599 fout = self.open("wb")
600 while 1:
601 buf = fin.read(_copy_blocksize)
602 if not buf:
603 break
604 fout.write(buf)
605 if fin.close() or fout.close():
606 raise PathException("Error closing file object")
607 self.setdata()
608
614
615 - def move(self, new_path):
616 """Like rename but destination may be on different file system"""
617 self.copy(new_path)
618 self.delete()
619
624
634
647
654
656 """Return string representation"""
657 return "(%s %s %s)" % (self.index, self.name, self.type)
658
659 - def quote(self, s = None):
660 """
661 Return quoted version of s (defaults to self.name)
662
663 The output is meant to be interpreted with shells, so can be
664 used with os.system.
665 """
666 if not s:
667 s = self.name
668 return '"%s"' % self.regex_chars_to_quote.sub(lambda m: "\\"+m.group(0), s)
669
671 """Return unquoted version of string s, as quoted by above quote()"""
672 assert s[0] == s[-1] == "\""
673 result = ""; i = 1
674 while i < len(s)-1:
675 if s[i] == "\\":
676 result += s[i+1]
677 i += 2
678 else:
679 result += s[i]
680 i += 1
681 return result
682
684 """Return filename of last component"""
685 components = self.name.split("/")
686 assert components and components[-1]
687 return components[-1]
688
690 """
691 Return string of canonical version of path
692
693 Remove ".", and trailing slashes where possible. Note that
694 it's harder to remove "..", as "foo/bar/.." is not necessarily
695 "foo", so we can't use path.normpath()
696 """
697 newpath = "/".join(filter(lambda x: x and x != ".",
698 self.name.split("/")))
699 if self.name[0] == "/":
700 return "/" + newpath
701 elif newpath:
702 return newpath
703 else:
704 return "."
705
706
708 """
709 Represent duplicity data files
710
711 Based on the file name, files that are compressed or encrypted
712 will have different open() methods.
713 """
714 - def __init__(self, base, index = (), parseresults = None):
715 """
716 DupPath initializer
717
718 The actual filename (no directory) must be the single element
719 of the index, unless parseresults is given.
720
721 """
722 if parseresults:
723 self.pr = parseresults
724 else:
725 assert len(index) == 1
726 self.pr = file_naming.parse(index[0])
727 assert self.pr, "must be a recognizable duplicity file"
728
729 Path.__init__(self, base, index)
730
732 """
733 Return fileobj with appropriate encryption/compression
734
735 If encryption is specified but no gpg_profile, use
736 globals.default_profile.
737 """
738 assert not self.opened and not self.fileobj
739 assert not (self.pr.encrypted and self.pr.compressed)
740 if gpg_profile:
741 assert self.pr.encrypted
742
743 if self.pr.compressed:
744 return gzip.GzipFile(self.name, mode)
745 elif self.pr.encrypted:
746 if not gpg_profile:
747 gpg_profile = globals.gpg_profile
748 if mode == "rb":
749 return gpg.GPGFile(False, self, gpg_profile)
750 elif mode == "wb":
751 return gpg.GPGFile(True, self, gpg_profile)
752 else:
753 return self.open(mode)
754
755
757 """Delete a directory. Called by Path.deltree"""
766