duplicity.statistics

1 # -*- Mode:Python; indent-tabs-mode:nil; tab-width:4 -*- 2 # 3 # Copyright 2002 Ben Escoto <ben@emerose.org> 4 # Copyright 2007 Kenneth Loafman <kenneth@loafman.com> 5 # 6 # This file is part of duplicity. 7 # 8 # Duplicity is free software; you can redistribute it and/or modify it 9 # under the terms of the GNU General Public License as published by the 10 # Free Software Foundation; either version 2 of the License, or (at your 11 # option) any later version. 12 # 13 # Duplicity is distributed in the hope that it will be useful, but 14 # WITHOUT ANY WARRANTY; without even the implied warranty of 15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 16 # General Public License for more details. 17 # 18 # You should have received a copy of the GNU General Public License 19 # along with duplicity; if not, write to the Free Software Foundation, 20 # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 21 22 """Generate and process backup statistics""" 23 24 import re, time, os 25 26 from duplicity import dup_time 27 28

29 -class StatsException(Exception):

30 pass

31

32 -class StatsObj:

33 """Contains various statistics, provide string conversion functions""" 34 # used when quoting files in get_stats_line 35 space_regex = re.compile(" ") 36 37 stat_file_attrs = ('SourceFiles', 38 'SourceFileSize', 39 'NewFiles', 40 'NewFileSize', 41 'DeletedFiles', 42 'ChangedFiles', 43 'ChangedFileSize', 44 'ChangedDeltaSize', 45 'DeltaEntries', 46 'RawDeltaSize') 47 stat_misc_attrs = ('Errors', 48 'TotalDestinationSizeChange') 49 stat_time_attrs = ('StartTime', 50 'EndTime', 51 'ElapsedTime') 52 stat_attrs = (('Filename',) + stat_time_attrs + 53 stat_misc_attrs + stat_file_attrs) 54 55 # Below, the second value in each pair is true iff the value 56 # indicates a number of bytes 57 stat_file_pairs = (('SourceFiles', False), 58 ('SourceFileSize', True), 59 ('NewFiles', False), 60 ('NewFileSize', True), 61 ('DeletedFiles', False), 62 ('ChangedFiles', False), 63 ('ChangedFileSize', True), 64 ('ChangedDeltaSize', True), 65 ('DeltaEntries', False), 66 ('RawDeltaSize', True)) 67 68 # This is used in get_byte_summary_string below 69 byte_abbrev_list = ((1024*1024*1024*1024, "TB"), 70 (1024*1024*1024, "GB"), 71 (1024*1024, "MB"), 72 (1024, "KB")) 73

74 - def __init__(self):

75 """Set attributes to None""" 76 for attr in self.stat_attrs: 77 self.__dict__[attr] = None

78

79 - def get_stat(self, attribute):

80 """Get a statistic""" 81 return self.__dict__[attribute]

82

83 - def set_stat(self, attr, value):

84 """Set attribute to given value""" 85 self.__dict__[attr] = value

86

87 - def increment_stat(self, attr):

88 """Add 1 to value of attribute""" 89 self.__dict__[attr] += 1

90

91 - def get_total_dest_size_change(self):

92 """Return total destination size change 93 94 This represents the total increase in the size of the 95 duplicity destination directory, or None if not available. 96 97 """ 98 return 0 # this needs to be re-done for duplicity

99

100 - def get_stats_line(self, index, use_repr = 1):

101 """Return one line abbreviated version of full stats string""" 102 file_attrs = map(lambda attr: str(self.get_stat(attr)), 103 self.stat_file_attrs) 104 if not index: 105 filename = "." 106 else: 107 filename = apply(os.path.join, index) 108 if use_repr: 109 # use repr to quote newlines in relative filename, then 110 # take of leading and trailing quote and quote spaces. 111 filename = self.space_regex.sub("\\x20", repr(filename)[1:-1]) 112 return " ".join([filename,] + file_attrs)

113

114 - def set_stats_from_line(self, line):

115 """Set statistics from given line""" 116 def error(): 117 raise StatsException("Bad line '%s'" % line)

118 if line[-1] == "\n": 119 line = line[:-1] 120 lineparts = line.split(" ") 121 if len(lineparts) < len(self.stat_file_attrs): 122 error() 123 for attr, val_string in zip(self.stat_file_attrs, 124 lineparts[-len(self.stat_file_attrs):]): 125 try: 126 val = long(val_string) 127 except ValueError: 128 try: 129 val = float(val_string) 130 except ValueError: 131 error() 132 self.set_stat(attr, val) 133 return self

134

135 - def get_stats_string(self):

136 """Return extended string printing out statistics""" 137 return "%s%s%s" % (self.get_timestats_string(), 138 self.get_filestats_string(), 139 self.get_miscstats_string())

140

141 - def get_timestats_string(self):

142 """Return portion of statistics string dealing with time""" 143 timelist = [] 144 if self.StartTime is not None: 145 timelist.append("StartTime %.2f (%s)\n" % 146 (self.StartTime, dup_time.timetopretty(self.StartTime))) 147 if self.EndTime is not None: 148 timelist.append("EndTime %.2f (%s)\n" % 149 (self.EndTime, dup_time.timetopretty(self.EndTime))) 150 if self.ElapsedTime or (self.StartTime is not None and 151 self.EndTime is not None): 152 if self.ElapsedTime is None: 153 self.ElapsedTime = self.EndTime - self.StartTime 154 timelist.append("ElapsedTime %.2f (%s)\n" % 155 (self.ElapsedTime, dup_time.inttopretty(self.ElapsedTime))) 156 return "".join(timelist)

157

158 - def get_filestats_string(self):

159 """Return portion of statistics string about files and bytes""" 160 def fileline(stat_file_pair): 161 """Return zero or one line of the string""" 162 attr, in_bytes = stat_file_pair 163 val = self.get_stat(attr) 164 if val is None: 165 return "" 166 if in_bytes: 167 return "%s %s (%s)\n" % (attr, val, 168 self.get_byte_summary_string(val)) 169 else: 170 return "%s %s\n" % (attr, val)

171 172 return "".join(map(fileline, self.stat_file_pairs)) 173

174 - def get_miscstats_string(self):

175 """Return portion of extended stat string about misc attributes""" 176 misc_string = "" 177 tdsc = self.TotalDestinationSizeChange 178 if tdsc is not None: 179 misc_string += ("TotalDestinationSizeChange %s (%s)\n" % 180 (tdsc, self.get_byte_summary_string(tdsc))) 181 if self.Errors is not None: 182 misc_string += "Errors %d\n" % self.Errors 183 return misc_string

184

185 - def get_byte_summary_string(self, byte_count):

186 """Turn byte count into human readable string like "7.23GB" """ 187 if byte_count < 0: 188 sign = "-" 189 byte_count = -byte_count 190 else: 191 sign = "" 192 193 for abbrev_bytes, abbrev_string in self.byte_abbrev_list: 194 if byte_count >= abbrev_bytes: 195 # Now get 3 significant figures 196 abbrev_count = float(byte_count)/abbrev_bytes 197 if abbrev_count >= 100: 198 precision = 0 199 elif abbrev_count >= 10: 200 precision = 1 201 else: 202 precision = 2 203 return "%s%%.%df %s" % (sign, precision, abbrev_string) \ 204 % (abbrev_count,) 205 byte_count = round(byte_count) 206 if byte_count == 1: 207 return sign + "1 byte" 208 else: 209 return "%s%d bytes" % (sign, byte_count)

210

211 - def get_stats_logstring(self, title):

212 """Like get_stats_string, but add header and footer""" 213 header = "--------------[ %s ]--------------" % title 214 footer = "-" * len(header) 215 return "%s\n%s%s\n" % (header, self.get_stats_string(), footer)

216

217 - def set_stats_from_string(self, s):

218 """Initialize attributes from string, return self for convenience""" 219 def error(line): 220 raise StatsException("Bad line '%s'" % line)

221 222 for line in s.split("\n"): 223 if not line: 224 continue 225 line_parts = line.split() 226 if len(line_parts) < 2: 227 error(line) 228 attr, value_string = line_parts[:2] 229 if not attr in self.stat_attrs: 230 error(line) 231 try: 232 try: 233 val1 = long(value_string) 234 except ValueError: 235 val1 = None 236 val2 = float(value_string) 237 if val1 == val2: 238 self.set_stat(attr, val1) # use integer val 239 else: 240 self.set_stat(attr, val2) # use float 241 except ValueError: 242 error(line) 243 return self 244

245 - def write_stats_to_path(self, path):

246 """Write statistics string to given path""" 247 fin = path.open("w") 248 fin.write(self.get_stats_string()) 249 assert not fin.close()

250

251 - def read_stats_from_path(self, path):

252 """Set statistics from path, return self for convenience""" 253 fp = path.open("r") 254 self.set_stats_from_string(fp.read()) 255 assert not fp.close() 256 return self

257

258 - def stats_equal(self, s):

259 """Return true if s has same statistics as self""" 260 assert isinstance(s, StatsObj) 261 for attr in self.stat_file_attrs: 262 if self.get_stat(attr) != s.get_stat(attr): 263 return None 264 return 1

265

266 - def set_to_average(self, statobj_list):

267 """Set self's attributes to average of those in statobj_list""" 268 for attr in self.stat_attrs: 269 self.set_stat(attr, 0) 270 for statobj in statobj_list: 271 for attr in self.stat_attrs: 272 if statobj.get_stat(attr) is None: 273 self.set_stat(attr, None) 274 elif self.get_stat(attr) is not None: 275 self.set_stat(attr, statobj.get_stat(attr) + 276 self.get_stat(attr)) 277 278 # Don't compute average starting/stopping time 279 self.StartTime = None 280 self.EndTime = None 281 282 for attr in self.stat_attrs: 283 if self.get_stat(attr) is not None: 284 self.set_stat(attr, 285 self.get_stat(attr)/float(len(statobj_list))) 286 return self

287

288 - def get_statsobj_copy(self):

289 """Return new StatsObj object with same stats as self""" 290 s = StatsObj() 291 for attr in self.stat_attrs: 292 s.set_stat(attr, self.get_stat(attr)) 293 return s

294 295

296 -class StatsDeltaProcess(StatsObj):

297 """Keep track of statistics during DirDelta process"""

298 - def __init__(self):

299 """StatsDeltaProcess initializer - zero file attributes""" 300 StatsObj.__init__(self) 301 for attr in StatsObj.stat_file_attrs: 302 self.__dict__[attr] = 0 303 self.Errors = 0 304 self.StartTime = time.time()

305

306 - def add_new_file(self, path):

307 """Add stats of new file path to statistics""" 308 filesize = path.getsize() 309 self.SourceFiles += 1 310 # SourceFileSize is added-to incrementally as read 311 self.NewFiles += 1 312 self.NewFileSize += filesize 313 self.DeltaEntries += 1

314

315 - def add_changed_file(self, path):

316 """Add stats of file that has changed since last backup""" 317 filesize = path.getsize() 318 self.SourceFiles += 1 319 # SourceFileSize is added-to incrementally as read 320 self.ChangedFiles += 1 321 self.ChangedFileSize += filesize 322 self.DeltaEntries += 1

323

324 - def add_deleted_file(self):

325 """Add stats of file no longer in source directory""" 326 self.DeletedFiles += 1 # can't add size since not available 327 self.DeltaEntries += 1

328

329 - def add_unchanged_file(self, path):

330 """Add stats of file that hasn't changed since last backup""" 331 filesize = path.getsize() 332 self.SourceFiles += 1 333 self.SourceFileSize += filesize

334

335 - def close(self):

336 """End collection of data, set EndTime""" 337 self.EndTime = time.time()

338

Source Code for Module duplicity.statistics