1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39 """
40 Provides an extension to back up mbox email files.
41
42 Backing up email
43 ================
44
45 Email folders (often stored as mbox flatfiles) are not well-suited being backed
46 up with an incremental backup like the one offered by Cedar Backup. This is
47 because mbox files often change on a daily basis, forcing the incremental
48 backup process to back them up every day in order to avoid losing data. This
49 can result in quite a bit of wasted space when backing up large folders. (Note
50 that the alternative maildir format does not share this problem, since it
51 typically uses one file per message.)
52
53 One solution to this problem is to design a smarter incremental backup process,
54 which backs up baseline content on the first day of the week, and then backs up
55 only new messages added to that folder on every other day of the week. This way,
56 the backup for any single day is only as large as the messages placed into the
57 folder on that day. The backup isn't as "perfect" as the incremental backup
58 process, because it doesn't preserve information about messages deleted from
59 the backed-up folder. However, it should be much more space-efficient, and
60 in a recovery situation, it seems better to restore too much data rather
61 than too little.
62
63 What is this extension?
64 =======================
65
66 This is a Cedar Backup extension used to back up mbox email files via the Cedar
67 Backup command line. Individual mbox files or directories containing mbox
68 files can be backed up using the same collect modes allowed for filesystems in
69 the standard Cedar Backup collect action: weekly, daily, incremental. It
70 implements the "smart" incremental backup process discussed above, using
71 functionality provided by the C{grepmail} utility.
72
73 This extension requires a new configuration section <mbox> and is intended to
74 be run either immediately before or immediately after the standard collect
75 action. Aside from its own configuration, it requires the options and collect
76 configuration sections in the standard Cedar Backup configuration file.
77
78 The mbox action is conceptually similar to the standard collect action,
79 except that mbox directories are not collected recursively. This implies
80 some configuration changes (i.e. there's no need for global exclusions or an
81 ignore file). If you back up a directory, all of the mbox files in that
82 directory are backed up into a single tar file using the indicated
83 compression method.
84
85 @author: Kenneth J. Pronovici <pronovic@ieee.org>
86 """
87
88
89
90
91
92
93 import os
94 import logging
95 import datetime
96 import pickle
97 import tempfile
98 from bz2 import BZ2File
99 from gzip import GzipFile
100
101
102 from CedarBackup2.filesystem import FilesystemList, BackupFileList
103 from CedarBackup2.xmlutil import createInputDom, addContainerNode, addStringNode
104 from CedarBackup2.xmlutil import isElement, readChildren, readFirstChild, readString, readStringList
105 from CedarBackup2.config import VALID_COLLECT_MODES, VALID_COMPRESS_MODES
106 from CedarBackup2.util import isStartOfWeek, buildNormalizedPath
107 from CedarBackup2.util import resolveCommand, executeCommand
108 from CedarBackup2.util import ObjectTypeList, UnorderedList, RegexList, encodePath, changeOwnership
109
110
111
112
113
114
115 logger = logging.getLogger("CedarBackup2.log.extend.mbox")
116
117 GREPMAIL_COMMAND = [ "grepmail", ]
118 REVISION_PATH_EXTENSION = "mboxlast"
119
120
121
122
123
124
125 -class MboxFile(object):
126
127 """
128 Class representing mbox file configuration..
129
130 The following restrictions exist on data in this class:
131
132 - The absolute path must be absolute.
133 - The collect mode must be one of the values in L{VALID_COLLECT_MODES}.
134 - The compress mode must be one of the values in L{VALID_COMPRESS_MODES}.
135
136 @sort: __init__, __repr__, __str__, __cmp__, absolutePath, collectMode, compressMode
137 """
138
139 - def __init__(self, absolutePath=None, collectMode=None, compressMode=None):
140 """
141 Constructor for the C{MboxFile} class.
142
143 You should never directly instantiate this class.
144
145 @param absolutePath: Absolute path to an mbox file on disk.
146 @param collectMode: Overridden collect mode for this directory.
147 @param compressMode: Overridden compression mode for this directory.
148 """
149 self._absolutePath = None
150 self._collectMode = None
151 self._compressMode = None
152 self.absolutePath = absolutePath
153 self.collectMode = collectMode
154 self.compressMode = compressMode
155
161
163 """
164 Informal string representation for class instance.
165 """
166 return self.__repr__()
167
192
194 """
195 Property target used to set the absolute path.
196 The value must be an absolute path if it is not C{None}.
197 It does not have to exist on disk at the time of assignment.
198 @raise ValueError: If the value is not an absolute path.
199 @raise ValueError: If the value cannot be encoded properly.
200 """
201 if value is not None:
202 if not os.path.isabs(value):
203 raise ValueError("Absolute path must be, er, an absolute path.")
204 self._absolutePath = encodePath(value)
205
207 """
208 Property target used to get the absolute path.
209 """
210 return self._absolutePath
211
213 """
214 Property target used to set the collect mode.
215 If not C{None}, the mode must be one of the values in L{VALID_COLLECT_MODES}.
216 @raise ValueError: If the value is not valid.
217 """
218 if value is not None:
219 if value not in VALID_COLLECT_MODES:
220 raise ValueError("Collect mode must be one of %s." % VALID_COLLECT_MODES)
221 self._collectMode = value
222
224 """
225 Property target used to get the collect mode.
226 """
227 return self._collectMode
228
230 """
231 Property target used to set the compress mode.
232 If not C{None}, the mode must be one of the values in L{VALID_COMPRESS_MODES}.
233 @raise ValueError: If the value is not valid.
234 """
235 if value is not None:
236 if value not in VALID_COMPRESS_MODES:
237 raise ValueError("Compress mode must be one of %s." % VALID_COMPRESS_MODES)
238 self._compressMode = value
239
241 """
242 Property target used to get the compress mode.
243 """
244 return self._compressMode
245
246 absolutePath = property(_getAbsolutePath, _setAbsolutePath, None, doc="Absolute path to the mbox file.")
247 collectMode = property(_getCollectMode, _setCollectMode, None, doc="Overridden collect mode for this mbox file.")
248 compressMode = property(_getCompressMode, _setCompressMode, None, doc="Overridden compress mode for this mbox file.")
249
250
251
252
253
254
255 -class MboxDir(object):
256
257 """
258 Class representing mbox directory configuration..
259
260 The following restrictions exist on data in this class:
261
262 - The absolute path must be absolute.
263 - The collect mode must be one of the values in L{VALID_COLLECT_MODES}.
264 - The compress mode must be one of the values in L{VALID_COMPRESS_MODES}.
265
266 Unlike collect directory configuration, this is the only place exclusions
267 are allowed (no global exclusions at the <mbox> configuration level). Also,
268 we only allow relative exclusions and there is no configured ignore file.
269 This is because mbox directory backups are not recursive.
270
271 @sort: __init__, __repr__, __str__, __cmp__, absolutePath, collectMode,
272 compressMode, relativeExcludePaths, excludePatterns
273 """
274
275 - def __init__(self, absolutePath=None, collectMode=None, compressMode=None,
276 relativeExcludePaths=None, excludePatterns=None):
277 """
278 Constructor for the C{MboxDir} class.
279
280 You should never directly instantiate this class.
281
282 @param absolutePath: Absolute path to a mbox file on disk.
283 @param collectMode: Overridden collect mode for this directory.
284 @param compressMode: Overridden compression mode for this directory.
285 @param relativeExcludePaths: List of relative paths to exclude.
286 @param excludePatterns: List of regular expression patterns to exclude
287 """
288 self._absolutePath = None
289 self._collectMode = None
290 self._compressMode = None
291 self._relativeExcludePaths = None
292 self._excludePatterns = None
293 self.absolutePath = absolutePath
294 self.collectMode = collectMode
295 self.compressMode = compressMode
296 self.relativeExcludePaths = relativeExcludePaths
297 self.excludePatterns = excludePatterns
298
305
307 """
308 Informal string representation for class instance.
309 """
310 return self.__repr__()
311
346
348 """
349 Property target used to set the absolute path.
350 The value must be an absolute path if it is not C{None}.
351 It does not have to exist on disk at the time of assignment.
352 @raise ValueError: If the value is not an absolute path.
353 @raise ValueError: If the value cannot be encoded properly.
354 """
355 if value is not None:
356 if not os.path.isabs(value):
357 raise ValueError("Absolute path must be, er, an absolute path.")
358 self._absolutePath = encodePath(value)
359
361 """
362 Property target used to get the absolute path.
363 """
364 return self._absolutePath
365
367 """
368 Property target used to set the collect mode.
369 If not C{None}, the mode must be one of the values in L{VALID_COLLECT_MODES}.
370 @raise ValueError: If the value is not valid.
371 """
372 if value is not None:
373 if value not in VALID_COLLECT_MODES:
374 raise ValueError("Collect mode must be one of %s." % VALID_COLLECT_MODES)
375 self._collectMode = value
376
378 """
379 Property target used to get the collect mode.
380 """
381 return self._collectMode
382
384 """
385 Property target used to set the compress mode.
386 If not C{None}, the mode must be one of the values in L{VALID_COMPRESS_MODES}.
387 @raise ValueError: If the value is not valid.
388 """
389 if value is not None:
390 if value not in VALID_COMPRESS_MODES:
391 raise ValueError("Compress mode must be one of %s." % VALID_COMPRESS_MODES)
392 self._compressMode = value
393
395 """
396 Property target used to get the compress mode.
397 """
398 return self._compressMode
399
401 """
402 Property target used to set the relative exclude paths list.
403 Elements do not have to exist on disk at the time of assignment.
404 """
405 if value is None:
406 self._relativeExcludePaths = None
407 else:
408 try:
409 saved = self._relativeExcludePaths
410 self._relativeExcludePaths = UnorderedList()
411 self._relativeExcludePaths.extend(value)
412 except Exception, e:
413 self._relativeExcludePaths = saved
414 raise e
415
417 """
418 Property target used to get the relative exclude paths list.
419 """
420 return self._relativeExcludePaths
421
423 """
424 Property target used to set the exclude patterns list.
425 """
426 if value is None:
427 self._excludePatterns = None
428 else:
429 try:
430 saved = self._excludePatterns
431 self._excludePatterns = RegexList()
432 self._excludePatterns.extend(value)
433 except Exception, e:
434 self._excludePatterns = saved
435 raise e
436
438 """
439 Property target used to get the exclude patterns list.
440 """
441 return self._excludePatterns
442
443 absolutePath = property(_getAbsolutePath, _setAbsolutePath, None, doc="Absolute path to the mbox directory.")
444 collectMode = property(_getCollectMode, _setCollectMode, None, doc="Overridden collect mode for this mbox directory.")
445 compressMode = property(_getCompressMode, _setCompressMode, None, doc="Overridden compress mode for this mbox directory.")
446 relativeExcludePaths = property(_getRelativeExcludePaths, _setRelativeExcludePaths, None, "List of relative paths to exclude.")
447 excludePatterns = property(_getExcludePatterns, _setExcludePatterns, None, "List of regular expression patterns to exclude.")
448
455
456 """
457 Class representing mbox configuration.
458
459 Mbox configuration is used for backing up mbox email files.
460
461 The following restrictions exist on data in this class:
462
463 - The collect mode must be one of the values in L{VALID_COLLECT_MODES}.
464 - The compress mode must be one of the values in L{VALID_COMPRESS_MODES}.
465 - The C{mboxFiles} list must be a list of C{MboxFile} objects
466 - The C{mboxDirs} list must be a list of C{MboxDir} objects
467
468 For the C{mboxFiles} and C{mboxDirs} lists, validation is accomplished
469 through the L{util.ObjectTypeList} list implementation that overrides common
470 list methods and transparently ensures that each element is of the proper
471 type.
472
473 Unlike collect configuration, no global exclusions are allowed on this
474 level. We only allow relative exclusions at the mbox directory level.
475 Also, there is no configured ignore file. This is because mbox directory
476 backups are not recursive.
477
478 @note: Lists within this class are "unordered" for equality comparisons.
479
480 @sort: __init__, __repr__, __str__, __cmp__, collectMode, compressMode, mboxFiles, mboxDirs
481 """
482
483 - def __init__(self, collectMode=None, compressMode=None, mboxFiles=None, mboxDirs=None):
484 """
485 Constructor for the C{MboxConfig} class.
486
487 @param collectMode: Default collect mode.
488 @param compressMode: Default compress mode.
489 @param mboxFiles: List of mbox files to back up
490 @param mboxDirs: List of mbox directories to back up
491
492 @raise ValueError: If one of the values is invalid.
493 """
494 self._collectMode = None
495 self._compressMode = None
496 self._mboxFiles = None
497 self._mboxDirs = None
498 self.collectMode = collectMode
499 self.compressMode = compressMode
500 self.mboxFiles = mboxFiles
501 self.mboxDirs = mboxDirs
502
508
510 """
511 Informal string representation for class instance.
512 """
513 return self.__repr__()
514
516 """
517 Definition of equals operator for this class.
518 Lists within this class are "unordered" for equality comparisons.
519 @param other: Other object to compare to.
520 @return: -1/0/1 depending on whether self is C{<}, C{=} or C{>} other.
521 """
522 if other is None:
523 return 1
524 if self.collectMode != other.collectMode:
525 if self.collectMode < other.collectMode:
526 return -1
527 else:
528 return 1
529 if self.compressMode != other.compressMode:
530 if self.compressMode < other.compressMode:
531 return -1
532 else:
533 return 1
534 if self.mboxFiles != other.mboxFiles:
535 if self.mboxFiles < other.mboxFiles:
536 return -1
537 else:
538 return 1
539 if self.mboxDirs != other.mboxDirs:
540 if self.mboxDirs < other.mboxDirs:
541 return -1
542 else:
543 return 1
544 return 0
545
547 """
548 Property target used to set the collect mode.
549 If not C{None}, the mode must be one of the values in L{VALID_COLLECT_MODES}.
550 @raise ValueError: If the value is not valid.
551 """
552 if value is not None:
553 if value not in VALID_COLLECT_MODES:
554 raise ValueError("Collect mode must be one of %s." % VALID_COLLECT_MODES)
555 self._collectMode = value
556
558 """
559 Property target used to get the collect mode.
560 """
561 return self._collectMode
562
564 """
565 Property target used to set the compress mode.
566 If not C{None}, the mode must be one of the values in L{VALID_COMPRESS_MODES}.
567 @raise ValueError: If the value is not valid.
568 """
569 if value is not None:
570 if value not in VALID_COMPRESS_MODES:
571 raise ValueError("Compress mode must be one of %s." % VALID_COMPRESS_MODES)
572 self._compressMode = value
573
575 """
576 Property target used to get the compress mode.
577 """
578 return self._compressMode
579
581 """
582 Property target used to set the mboxFiles list.
583 Either the value must be C{None} or each element must be an C{MboxFile}.
584 @raise ValueError: If the value is not an C{MboxFile}
585 """
586 if value is None:
587 self._mboxFiles = None
588 else:
589 try:
590 saved = self._mboxFiles
591 self._mboxFiles = ObjectTypeList(MboxFile, "MboxFile")
592 self._mboxFiles.extend(value)
593 except Exception, e:
594 self._mboxFiles = saved
595 raise e
596
598 """
599 Property target used to get the mboxFiles list.
600 """
601 return self._mboxFiles
602
604 """
605 Property target used to set the mboxDirs list.
606 Either the value must be C{None} or each element must be an C{MboxDir}.
607 @raise ValueError: If the value is not an C{MboxDir}
608 """
609 if value is None:
610 self._mboxDirs = None
611 else:
612 try:
613 saved = self._mboxDirs
614 self._mboxDirs = ObjectTypeList(MboxDir, "MboxDir")
615 self._mboxDirs.extend(value)
616 except Exception, e:
617 self._mboxDirs = saved
618 raise e
619
621 """
622 Property target used to get the mboxDirs list.
623 """
624 return self._mboxDirs
625
626 collectMode = property(_getCollectMode, _setCollectMode, None, doc="Default collect mode.")
627 compressMode = property(_getCompressMode, _setCompressMode, None, doc="Default compress mode.")
628 mboxFiles = property(_getMboxFiles, _setMboxFiles, None, doc="List of mbox files to back up.")
629 mboxDirs = property(_getMboxDirs, _setMboxDirs, None, doc="List of mbox directories to back up.")
630
637
638 """
639 Class representing this extension's configuration document.
640
641 This is not a general-purpose configuration object like the main Cedar
642 Backup configuration object. Instead, it just knows how to parse and emit
643 Mbox-specific configuration values. Third parties who need to read and
644 write configuration related to this extension should access it through the
645 constructor, C{validate} and C{addConfig} methods.
646
647 @note: Lists within this class are "unordered" for equality comparisons.
648
649 @sort: __init__, __repr__, __str__, __cmp__, mbox, validate, addConfig
650 """
651
652 - def __init__(self, xmlData=None, xmlPath=None, validate=True):
653 """
654 Initializes a configuration object.
655
656 If you initialize the object without passing either C{xmlData} or
657 C{xmlPath} then configuration will be empty and will be invalid until it
658 is filled in properly.
659
660 No reference to the original XML data or original path is saved off by
661 this class. Once the data has been parsed (successfully or not) this
662 original information is discarded.
663
664 Unless the C{validate} argument is C{False}, the L{LocalConfig.validate}
665 method will be called (with its default arguments) against configuration
666 after successfully parsing any passed-in XML. Keep in mind that even if
667 C{validate} is C{False}, it might not be possible to parse the passed-in
668 XML document if lower-level validations fail.
669
670 @note: It is strongly suggested that the C{validate} option always be set
671 to C{True} (the default) unless there is a specific need to read in
672 invalid configuration from disk.
673
674 @param xmlData: XML data representing configuration.
675 @type xmlData: String data.
676
677 @param xmlPath: Path to an XML file on disk.
678 @type xmlPath: Absolute path to a file on disk.
679
680 @param validate: Validate the document after parsing it.
681 @type validate: Boolean true/false.
682
683 @raise ValueError: If both C{xmlData} and C{xmlPath} are passed-in.
684 @raise ValueError: If the XML data in C{xmlData} or C{xmlPath} cannot be parsed.
685 @raise ValueError: If the parsed configuration document is not valid.
686 """
687 self._mbox = None
688 self.mbox = None
689 if xmlData is not None and xmlPath is not None:
690 raise ValueError("Use either xmlData or xmlPath, but not both.")
691 if xmlData is not None:
692 self._parseXmlData(xmlData)
693 if validate:
694 self.validate()
695 elif xmlPath is not None:
696 xmlData = open(xmlPath).read()
697 self._parseXmlData(xmlData)
698 if validate:
699 self.validate()
700
702 """
703 Official string representation for class instance.
704 """
705 return "LocalConfig(%s)" % (self.mbox)
706
708 """
709 Informal string representation for class instance.
710 """
711 return self.__repr__()
712
714 """
715 Definition of equals operator for this class.
716 Lists within this class are "unordered" for equality comparisons.
717 @param other: Other object to compare to.
718 @return: -1/0/1 depending on whether self is C{<}, C{=} or C{>} other.
719 """
720 if other is None:
721 return 1
722 if self.mbox != other.mbox:
723 if self.mbox < other.mbox:
724 return -1
725 else:
726 return 1
727 return 0
728
730 """
731 Property target used to set the mbox configuration value.
732 If not C{None}, the value must be a C{MboxConfig} object.
733 @raise ValueError: If the value is not a C{MboxConfig}
734 """
735 if value is None:
736 self._mbox = None
737 else:
738 if not isinstance(value, MboxConfig):
739 raise ValueError("Value must be a C{MboxConfig} object.")
740 self._mbox = value
741
743 """
744 Property target used to get the mbox configuration value.
745 """
746 return self._mbox
747
748 mbox = property(_getMbox, _setMbox, None, "Mbox configuration in terms of a C{MboxConfig} object.")
749
751 """
752 Validates configuration represented by the object.
753
754 Mbox configuration must be filled in. Within that, the collect mode and
755 compress mode are both optional, but the list of repositories must
756 contain at least one entry.
757
758 Each configured file or directory must contain an absolute path, and then
759 must be either able to take collect mode and compress mode configuration
760 from the parent C{MboxConfig} object, or must set each value on its own.
761
762 @raise ValueError: If one of the validations fails.
763 """
764 if self.mbox is None:
765 raise ValueError("Mbox section is required.")
766 if ((self.mbox.mboxFiles is None or len(self.mbox.mboxFiles) < 1) and \
767 (self.mbox.mboxDirs is None or len(self.mbox.mboxDirs) < 1)):
768 raise ValueError("At least one mbox file or directory must be configured.")
769 if self.mbox.mboxFiles is not None:
770 for mboxFile in self.mbox.mboxFiles:
771 if mboxFile.absolutePath is None:
772 raise ValueError("Each mbox file must set an absolute path.")
773 if self.mbox.collectMode is None and mboxFile.collectMode is None:
774 raise ValueError("Collect mode must either be set in parent mbox section or individual mbox file.")
775 if self.mbox.compressMode is None and mboxFile.compressMode is None:
776 raise ValueError("Compress mode must either be set in parent mbox section or individual mbox file.")
777 if self.mbox.mboxDirs is not None:
778 for mboxDir in self.mbox.mboxDirs:
779 if mboxDir.absolutePath is None:
780 raise ValueError("Each mbox directory must set an absolute path.")
781 if self.mbox.collectMode is None and mboxDir.collectMode is None:
782 raise ValueError("Collect mode must either be set in parent mbox section or individual mbox directory.")
783 if self.mbox.compressMode is None and mboxDir.compressMode is None:
784 raise ValueError("Compress mode must either be set in parent mbox section or individual mbox directory.")
785
787 """
788 Adds an <mbox> configuration section as the next child of a parent.
789
790 Third parties should use this function to write configuration related to
791 this extension.
792
793 We add the following fields to the document::
794
795 collectMode //cb_config/mbox/collectMode
796 compressMode //cb_config/mbox/compressMode
797
798 We also add groups of the following items, one list element per
799 item::
800
801 mboxFiles //cb_config/mbox/file
802 mboxDirs //cb_config/mbox/dir
803
804 The mbox files and mbox directories are added by L{_addMboxFile} and
805 L{_addMboxDir}.
806
807 @param xmlDom: DOM tree as from C{impl.createDocument()}.
808 @param parentNode: Parent that the section should be appended to.
809 """
810 if self.mbox is not None:
811 sectionNode = addContainerNode(xmlDom, parentNode, "mbox")
812 addStringNode(xmlDom, sectionNode, "collect_mode", self.mbox.collectMode)
813 addStringNode(xmlDom, sectionNode, "compress_mode", self.mbox.compressMode)
814 if self.mbox.mboxFiles is not None:
815 for mboxFile in self.mbox.mboxFiles:
816 LocalConfig._addMboxFile(xmlDom, sectionNode, mboxFile)
817 if self.mbox.mboxDirs is not None:
818 for mboxDir in self.mbox.mboxDirs:
819 LocalConfig._addMboxDir(xmlDom, sectionNode, mboxDir)
820
822 """
823 Internal method to parse an XML string into the object.
824
825 This method parses the XML document into a DOM tree (C{xmlDom}) and then
826 calls a static method to parse the mbox configuration section.
827
828 @param xmlData: XML data to be parsed
829 @type xmlData: String data
830
831 @raise ValueError: If the XML cannot be successfully parsed.
832 """
833 (xmlDom, parentNode) = createInputDom(xmlData)
834 self._mbox = LocalConfig._parseMbox(parentNode)
835
836 @staticmethod
838 """
839 Parses an mbox configuration section.
840
841 We read the following individual fields::
842
843 collectMode //cb_config/mbox/collect_mode
844 compressMode //cb_config/mbox/compress_mode
845
846 We also read groups of the following item, one list element per
847 item::
848
849 mboxFiles //cb_config/mbox/file
850 mboxDirs //cb_config/mbox/dir
851
852 The mbox files are parsed by L{_parseMboxFiles} and the mbox
853 directories are parsed by L{_parseMboxDirs}.
854
855 @param parent: Parent node to search beneath.
856
857 @return: C{MboxConfig} object or C{None} if the section does not exist.
858 @raise ValueError: If some filled-in value is invalid.
859 """
860 mbox = None
861 section = readFirstChild(parent, "mbox")
862 if section is not None:
863 mbox = MboxConfig()
864 mbox.collectMode = readString(section, "collect_mode")
865 mbox.compressMode = readString(section, "compress_mode")
866 mbox.mboxFiles = LocalConfig._parseMboxFiles(section)
867 mbox.mboxDirs = LocalConfig._parseMboxDirs(section)
868 return mbox
869
870 @staticmethod
872 """
873 Reads a list of C{MboxFile} objects from immediately beneath the parent.
874
875 We read the following individual fields::
876
877 absolutePath abs_path
878 collectMode collect_mode
879 compressMode compess_mode
880
881 @param parent: Parent node to search beneath.
882
883 @return: List of C{MboxFile} objects or C{None} if none are found.
884 @raise ValueError: If some filled-in value is invalid.
885 """
886 lst = []
887 for entry in readChildren(parent, "file"):
888 if isElement(entry):
889 mboxFile = MboxFile()
890 mboxFile.absolutePath = readString(entry, "abs_path")
891 mboxFile.collectMode = readString(entry, "collect_mode")
892 mboxFile.compressMode = readString(entry, "compress_mode")
893 lst.append(mboxFile)
894 if lst == []:
895 lst = None
896 return lst
897
898 @staticmethod
900 """
901 Reads a list of C{MboxDir} objects from immediately beneath the parent.
902
903 We read the following individual fields::
904
905 absolutePath abs_path
906 collectMode collect_mode
907 compressMode compess_mode
908
909 We also read groups of the following items, one list element per
910 item::
911
912 relativeExcludePaths exclude/rel_path
913 excludePatterns exclude/pattern
914
915 The exclusions are parsed by L{_parseExclusions}.
916
917 @param parent: Parent node to search beneath.
918
919 @return: List of C{MboxDir} objects or C{None} if none are found.
920 @raise ValueError: If some filled-in value is invalid.
921 """
922 lst = []
923 for entry in readChildren(parent, "dir"):
924 if isElement(entry):
925 mboxDir = MboxDir()
926 mboxDir.absolutePath = readString(entry, "abs_path")
927 mboxDir.collectMode = readString(entry, "collect_mode")
928 mboxDir.compressMode = readString(entry, "compress_mode")
929 (mboxDir.relativeExcludePaths, mboxDir.excludePatterns) = LocalConfig._parseExclusions(entry)
930 lst.append(mboxDir)
931 if lst == []:
932 lst = None
933 return lst
934
935 @staticmethod
937 """
938 Reads exclusions data from immediately beneath the parent.
939
940 We read groups of the following items, one list element per item::
941
942 relative exclude/rel_path
943 patterns exclude/pattern
944
945 If there are none of some pattern (i.e. no relative path items) then
946 C{None} will be returned for that item in the tuple.
947
948 @param parentNode: Parent node to search beneath.
949
950 @return: Tuple of (relative, patterns) exclusions.
951 """
952 section = readFirstChild(parentNode, "exclude")
953 if section is None:
954 return (None, None)
955 else:
956 relative = readStringList(section, "rel_path")
957 patterns = readStringList(section, "pattern")
958 return (relative, patterns)
959
960 @staticmethod
962 """
963 Adds an mbox file container as the next child of a parent.
964
965 We add the following fields to the document::
966
967 absolutePath file/abs_path
968 collectMode file/collect_mode
969 compressMode file/compress_mode
970
971 The <file> node itself is created as the next child of the parent node.
972 This method only adds one mbox file node. The parent must loop for each
973 mbox file in the C{MboxConfig} object.
974
975 If C{mboxFile} is C{None}, this method call will be a no-op.
976
977 @param xmlDom: DOM tree as from C{impl.createDocument()}.
978 @param parentNode: Parent that the section should be appended to.
979 @param mboxFile: MboxFile to be added to the document.
980 """
981 if mboxFile is not None:
982 sectionNode = addContainerNode(xmlDom, parentNode, "file")
983 addStringNode(xmlDom, sectionNode, "abs_path", mboxFile.absolutePath)
984 addStringNode(xmlDom, sectionNode, "collect_mode", mboxFile.collectMode)
985 addStringNode(xmlDom, sectionNode, "compress_mode", mboxFile.compressMode)
986
987 @staticmethod
989 """
990 Adds an mbox directory container as the next child of a parent.
991
992 We add the following fields to the document::
993
994 absolutePath dir/abs_path
995 collectMode dir/collect_mode
996 compressMode dir/compress_mode
997
998 We also add groups of the following items, one list element per item::
999
1000 relativeExcludePaths dir/exclude/rel_path
1001 excludePatterns dir/exclude/pattern
1002
1003 The <dir> node itself is created as the next child of the parent node.
1004 This method only adds one mbox directory node. The parent must loop for
1005 each mbox directory in the C{MboxConfig} object.
1006
1007 If C{mboxDir} is C{None}, this method call will be a no-op.
1008
1009 @param xmlDom: DOM tree as from C{impl.createDocument()}.
1010 @param parentNode: Parent that the section should be appended to.
1011 @param mboxDir: MboxDir to be added to the document.
1012 """
1013 if mboxDir is not None:
1014 sectionNode = addContainerNode(xmlDom, parentNode, "dir")
1015 addStringNode(xmlDom, sectionNode, "abs_path", mboxDir.absolutePath)
1016 addStringNode(xmlDom, sectionNode, "collect_mode", mboxDir.collectMode)
1017 addStringNode(xmlDom, sectionNode, "compress_mode", mboxDir.compressMode)
1018 if ((mboxDir.relativeExcludePaths is not None and mboxDir.relativeExcludePaths != []) or
1019 (mboxDir.excludePatterns is not None and mboxDir.excludePatterns != [])):
1020 excludeNode = addContainerNode(xmlDom, sectionNode, "exclude")
1021 if mboxDir.relativeExcludePaths is not None:
1022 for relativePath in mboxDir.relativeExcludePaths:
1023 addStringNode(xmlDom, excludeNode, "rel_path", relativePath)
1024 if mboxDir.excludePatterns is not None:
1025 for pattern in mboxDir.excludePatterns:
1026 addStringNode(xmlDom, excludeNode, "pattern", pattern)
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037 -def executeAction(configPath, options, config):
1038 """
1039 Executes the mbox backup action.
1040
1041 @param configPath: Path to configuration file on disk.
1042 @type configPath: String representing a path on disk.
1043
1044 @param options: Program command-line options.
1045 @type options: Options object.
1046
1047 @param config: Program configuration.
1048 @type config: Config object.
1049
1050 @raise ValueError: Under many generic error conditions
1051 @raise IOError: If a backup could not be written for some reason.
1052 """
1053 logger.debug("Executing mbox extended action.")
1054 newRevision = datetime.datetime.today()
1055 if config.options is None or config.collect is None:
1056 raise ValueError("Cedar Backup configuration is not properly filled in.")
1057 local = LocalConfig(xmlPath=configPath)
1058 todayIsStart = isStartOfWeek(config.options.startingDay)
1059 fullBackup = options.full or todayIsStart
1060 logger.debug("Full backup flag is [%s]" % fullBackup)
1061 if local.mbox.mboxFiles is not None:
1062 for mboxFile in local.mbox.mboxFiles:
1063 logger.debug("Working with mbox file [%s]" % mboxFile.absolutePath)
1064 collectMode = _getCollectMode(local, mboxFile)
1065 compressMode = _getCompressMode(local, mboxFile)
1066 lastRevision = _loadLastRevision(config, mboxFile, fullBackup, collectMode)
1067 if fullBackup or (collectMode in ['daily', 'incr', ]) or (collectMode == 'weekly' and todayIsStart):
1068 logger.debug("Mbox file meets criteria to be backed up today.")
1069 _backupMboxFile(config, mboxFile.absolutePath, fullBackup,
1070 collectMode, compressMode, lastRevision, newRevision)
1071 else:
1072 logger.debug("Mbox file will not be backed up, per collect mode.")
1073 if collectMode == 'incr':
1074 _writeNewRevision(config, mboxFile, newRevision)
1075 if local.mbox.mboxDirs is not None:
1076 for mboxDir in local.mbox.mboxDirs:
1077 logger.debug("Working with mbox directory [%s]" % mboxDir.absolutePath)
1078 collectMode = _getCollectMode(local, mboxDir)
1079 compressMode = _getCompressMode(local, mboxDir)
1080 lastRevision = _loadLastRevision(config, mboxDir, fullBackup, collectMode)
1081 (excludePaths, excludePatterns) = _getExclusions(mboxDir)
1082 if fullBackup or (collectMode in ['daily', 'incr', ]) or (collectMode == 'weekly' and todayIsStart):
1083 logger.debug("Mbox directory meets criteria to be backed up today.")
1084 _backupMboxDir(config, mboxDir.absolutePath,
1085 fullBackup, collectMode, compressMode,
1086 lastRevision, newRevision,
1087 excludePaths, excludePatterns)
1088 else:
1089 logger.debug("Mbox directory will not be backed up, per collect mode.")
1090 if collectMode == 'incr':
1091 _writeNewRevision(config, mboxDir, newRevision)
1092 logger.info("Executed the mbox extended action successfully.")
1093
1095 """
1096 Gets the collect mode that should be used for an mbox file or directory.
1097 Use file- or directory-specific value if possible, otherwise take from mbox section.
1098 @param local: LocalConfig object.
1099 @param item: Mbox file or directory
1100 @return: Collect mode to use.
1101 """
1102 if item.collectMode is None:
1103 collectMode = local.mbox.collectMode
1104 else:
1105 collectMode = item.collectMode
1106 logger.debug("Collect mode is [%s]" % collectMode)
1107 return collectMode
1108
1110 """
1111 Gets the compress mode that should be used for an mbox file or directory.
1112 Use file- or directory-specific value if possible, otherwise take from mbox section.
1113 @param local: LocalConfig object.
1114 @param item: Mbox file or directory
1115 @return: Compress mode to use.
1116 """
1117 if item.compressMode is None:
1118 compressMode = local.mbox.compressMode
1119 else:
1120 compressMode = item.compressMode
1121 logger.debug("Compress mode is [%s]" % compressMode)
1122 return compressMode
1123
1125 """
1126 Gets the path to the revision file associated with a repository.
1127 @param config: Cedar Backup configuration.
1128 @param item: Mbox file or directory
1129 @return: Absolute path to the revision file associated with the repository.
1130 """
1131 normalized = buildNormalizedPath(item.absolutePath)
1132 filename = "%s.%s" % (normalized, REVISION_PATH_EXTENSION)
1133 revisionPath = os.path.join(config.options.workingDir, filename)
1134 logger.debug("Revision file path is [%s]" % revisionPath)
1135 return revisionPath
1136
1138 """
1139 Loads the last revision date for this item from disk and returns it.
1140
1141 If this is a full backup, or if the revision file cannot be loaded for some
1142 reason, then C{None} is returned. This indicates that there is no previous
1143 revision, so the entire mail file or directory should be backed up.
1144
1145 @note: We write the actual revision object to disk via pickle, so we don't
1146 deal with the datetime precision or format at all. Whatever's in the object
1147 is what we write.
1148
1149 @param config: Cedar Backup configuration.
1150 @param item: Mbox file or directory
1151 @param fullBackup: Indicates whether this is a full backup
1152 @param collectMode: Indicates the collect mode for this item
1153
1154 @return: Revision date as a datetime.datetime object or C{None}.
1155 """
1156 revisionPath = _getRevisionPath(config, item)
1157 if fullBackup:
1158 revisionDate = None
1159 logger.debug("Revision file ignored because this is a full backup.")
1160 elif collectMode in ['weekly', 'daily']:
1161 revisionDate = None
1162 logger.debug("No revision file based on collect mode [%s]." % collectMode)
1163 else:
1164 logger.debug("Revision file will be used for non-full incremental backup.")
1165 if not os.path.isfile(revisionPath):
1166 revisionDate = None
1167 logger.debug("Revision file [%s] does not exist on disk." % revisionPath)
1168 else:
1169 try:
1170 revisionDate = pickle.load(open(revisionPath, "r"))
1171 logger.debug("Loaded revision file [%s] from disk: [%s]" % (revisionPath, revisionDate))
1172 except:
1173 revisionDate = None
1174 logger.error("Failed loading revision file [%s] from disk." % revisionPath)
1175 return revisionDate
1176
1178 """
1179 Writes new revision information to disk.
1180
1181 If we can't write the revision file successfully for any reason, we'll log
1182 the condition but won't throw an exception.
1183
1184 @note: We write the actual revision object to disk via pickle, so we don't
1185 deal with the datetime precision or format at all. Whatever's in the object
1186 is what we write.
1187
1188 @param config: Cedar Backup configuration.
1189 @param item: Mbox file or directory
1190 @param newRevision: Revision date as a datetime.datetime object.
1191 """
1192 revisionPath = _getRevisionPath(config, item)
1193 try:
1194 pickle.dump(newRevision, open(revisionPath, "w"))
1195 changeOwnership(revisionPath, config.options.backupUser, config.options.backupGroup)
1196 logger.debug("Wrote new revision file [%s] to disk: [%s]" % (revisionPath, newRevision))
1197 except:
1198 logger.error("Failed to write revision file [%s] to disk." % revisionPath)
1199
1201 """
1202 Gets exclusions (file and patterns) associated with an mbox directory.
1203
1204 The returned files value is a list of absolute paths to be excluded from the
1205 backup for a given directory. It is derived from the mbox directory's
1206 relative exclude paths.
1207
1208 The returned patterns value is a list of patterns to be excluded from the
1209 backup for a given directory. It is derived from the mbox directory's list
1210 of patterns.
1211
1212 @param mboxDir: Mbox directory object.
1213
1214 @return: Tuple (files, patterns) indicating what to exclude.
1215 """
1216 paths = []
1217 if mboxDir.relativeExcludePaths is not None:
1218 for relativePath in mboxDir.relativeExcludePaths:
1219 paths.append(os.path.join(mboxDir.absolutePath, relativePath))
1220 patterns = []
1221 if mboxDir.excludePatterns is not None:
1222 patterns.extend(mboxDir.excludePatterns)
1223 logger.debug("Exclude paths: %s" % paths)
1224 logger.debug("Exclude patterns: %s" % patterns)
1225 return(paths, patterns)
1226
1227 -def _getBackupPath(config, mboxPath, compressMode, newRevision, targetDir=None):
1228 """
1229 Gets the backup file path (including correct extension) associated with an mbox path.
1230
1231 We assume that if the target directory is passed in, that we're backing up a
1232 directory. Under these circumstances, we'll just use the basename of the
1233 individual path as the output file.
1234
1235 @note: The backup path only contains the current date in YYYYMMDD format,
1236 but that's OK because the index information (stored elsewhere) is the actual
1237 date object.
1238
1239 @param config: Cedar Backup configuration.
1240 @param mboxPath: Path to the indicated mbox file or directory
1241 @param compressMode: Compress mode to use for this mbox path
1242 @param newRevision: Revision this backup path represents
1243 @param targetDir: Target directory in which the path should exist
1244
1245 @return: Absolute path to the backup file associated with the repository.
1246 """
1247 if targetDir is None:
1248 normalizedPath = buildNormalizedPath(mboxPath)
1249 revisionDate = newRevision.strftime("%Y%m%d")
1250 filename = "mbox-%s-%s" % (revisionDate, normalizedPath)
1251 else:
1252 filename = os.path.basename(mboxPath)
1253 if compressMode == 'gzip':
1254 filename = "%s.gz" % filename
1255 elif compressMode == 'bzip2':
1256 filename = "%s.bz2" % filename
1257 if targetDir is None:
1258 backupPath = os.path.join(config.collect.targetDir, filename)
1259 else:
1260 backupPath = os.path.join(targetDir, filename)
1261 logger.debug("Backup file path is [%s]" % backupPath)
1262 return backupPath
1263
1265 """
1266 Gets the tarfile backup file path (including correct extension) associated
1267 with an mbox path.
1268
1269 Along with the path, the tar archive mode is returned in a form that can
1270 be used with L{BackupFileList.generateTarfile}.
1271
1272 @note: The tarfile path only contains the current date in YYYYMMDD format,
1273 but that's OK because the index information (stored elsewhere) is the actual
1274 date object.
1275
1276 @param config: Cedar Backup configuration.
1277 @param mboxPath: Path to the indicated mbox file or directory
1278 @param compressMode: Compress mode to use for this mbox path
1279 @param newRevision: Revision this backup path represents
1280
1281 @return: Tuple of (absolute path to tarfile, tar archive mode)
1282 """
1283 normalizedPath = buildNormalizedPath(mboxPath)
1284 revisionDate = newRevision.strftime("%Y%m%d")
1285 filename = "mbox-%s-%s.tar" % (revisionDate, normalizedPath)
1286 if compressMode == 'gzip':
1287 filename = "%s.gz" % filename
1288 archiveMode = "targz"
1289 elif compressMode == 'bzip2':
1290 filename = "%s.bz2" % filename
1291 archiveMode = "tarbz2"
1292 else:
1293 archiveMode = "tar"
1294 tarfilePath = os.path.join(config.collect.targetDir, filename)
1295 logger.debug("Tarfile path is [%s]" % tarfilePath)
1296 return (tarfilePath, archiveMode)
1297
1299 """
1300 Opens the output file used for saving backup information.
1301
1302 If the compress mode is "gzip", we'll open a C{GzipFile}, and if the
1303 compress mode is "bzip2", we'll open a C{BZ2File}. Otherwise, we'll just
1304 return an object from the normal C{open()} method.
1305
1306 @param backupPath: Path to file to open.
1307 @param compressMode: Compress mode of file ("none", "gzip", "bzip").
1308
1309 @return: Output file object.
1310 """
1311 if compressMode == "gzip":
1312 return GzipFile(backupPath, "w")
1313 elif compressMode == "bzip2":
1314 return BZ2File(backupPath, "w")
1315 else:
1316 return open(backupPath, "w")
1317
1318 -def _backupMboxFile(config, absolutePath,
1319 fullBackup, collectMode, compressMode,
1320 lastRevision, newRevision, targetDir=None):
1321 """
1322 Backs up an individual mbox file.
1323
1324 @param config: Cedar Backup configuration.
1325 @param absolutePath: Path to mbox file to back up.
1326 @param fullBackup: Indicates whether this should be a full backup.
1327 @param collectMode: Indicates the collect mode for this item
1328 @param compressMode: Compress mode of file ("none", "gzip", "bzip")
1329 @param lastRevision: Date of last backup as datetime.datetime
1330 @param newRevision: Date of new (current) backup as datetime.datetime
1331 @param targetDir: Target directory to write the backed-up file into
1332
1333 @raise ValueError: If some value is missing or invalid.
1334 @raise IOError: If there is a problem backing up the mbox file.
1335 """
1336 backupPath = _getBackupPath(config, absolutePath, compressMode, newRevision, targetDir=targetDir)
1337 outputFile = _getOutputFile(backupPath, compressMode)
1338 if fullBackup or collectMode != "incr" or lastRevision is None:
1339 args = [ "-a", "-u", absolutePath, ]
1340 else:
1341 revisionDate = lastRevision.strftime("%Y-%m-%dT%H:%M:%S")
1342 args = [ "-a", "-u", "-d", "since %s" % revisionDate, absolutePath, ]
1343 command = resolveCommand(GREPMAIL_COMMAND)
1344 result = executeCommand(command, args, returnOutput=False, ignoreStderr=True, doNotLog=True, outputFile=outputFile)[0]
1345 if result != 0:
1346 raise IOError("Error [%d] executing grepmail on [%s]." % (result, absolutePath))
1347 logger.debug("Completed backing up mailbox [%s]." % absolutePath)
1348 return backupPath
1349
1350 -def _backupMboxDir(config, absolutePath,
1351 fullBackup, collectMode, compressMode,
1352 lastRevision, newRevision,
1353 excludePaths, excludePatterns):
1354 """
1355 Backs up a directory containing mbox files.
1356
1357 @param config: Cedar Backup configuration.
1358 @param absolutePath: Path to mbox directory to back up.
1359 @param fullBackup: Indicates whether this should be a full backup.
1360 @param collectMode: Indicates the collect mode for this item
1361 @param compressMode: Compress mode of file ("none", "gzip", "bzip")
1362 @param lastRevision: Date of last backup as datetime.datetime
1363 @param newRevision: Date of new (current) backup as datetime.datetime
1364 @param excludePaths: List of absolute paths to exclude.
1365 @param excludePatterns: List of patterns to exclude.
1366
1367 @raise ValueError: If some value is missing or invalid.
1368 @raise IOError: If there is a problem backing up the mbox file.
1369 """
1370 try:
1371 tmpdir = tempfile.mkdtemp(dir=config.options.workingDir)
1372 mboxList = FilesystemList()
1373 mboxList.excludeDirs = True
1374 mboxList.excludePaths = excludePaths
1375 mboxList.excludePatterns = excludePatterns
1376 mboxList.addDirContents(absolutePath, recursive=False)
1377 tarList = BackupFileList()
1378 for item in mboxList:
1379 backupPath = _backupMboxFile(config, item, fullBackup,
1380 collectMode, "none",
1381 lastRevision, newRevision,
1382 targetDir=tmpdir)
1383 tarList.addFile(backupPath)
1384 (tarfilePath, archiveMode) = _getTarfilePath(config, absolutePath, compressMode, newRevision)
1385 tarList.generateTarfile(tarfilePath, archiveMode, ignore=True, flat=True)
1386 changeOwnership(tarfilePath, config.options.backupUser, config.options.backupGroup)
1387 logger.debug("Completed backing up directory [%s]." % absolutePath)
1388 finally:
1389 try:
1390 for item in tarList:
1391 if os.path.exists(item):
1392 try:
1393 os.remove(item)
1394 except: pass
1395 except: pass
1396 try:
1397 os.rmdir(tmpdir)
1398 except: pass
1399