1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39 """
40 Provides an extension to back up Subversion repositories.
41
42 This is a Cedar Backup extension used to back up Subversion repositories via
43 the Cedar Backup command line. Each Subversion repository can be backed using
44 the same collect modes allowed for filesystems in the standard Cedar Backup
45 collect action: weekly, daily, incremental.
46
47 This extension requires a new configuration section <subversion> and is
48 intended to be run either immediately before or immediately after the standard
49 collect action. Aside from its own configuration, it requires the options and
50 collect configuration sections in the standard Cedar Backup configuration file.
51
52 There are two different kinds of Subversion repositories at this writing: BDB
53 (Berkeley Database) and FSFS (a "filesystem within a filesystem"). Although
54 the repository type can be specified in configuration, that information is just
55 kept around for reference. It doesn't affect the backup. Both kinds of
56 repositories are backed up in the same way, using C{svnadmin dump} in an
57 incremental mode.
58
59 It turns out that FSFS repositories can also be backed up just like any
60 other filesystem directory. If you would rather do that, then use the normal
61 collect action. This is probably simpler, although it carries its own
62 advantages and disadvantages (plus you will have to be careful to exclude
63 the working directories Subversion uses when building an update to commit).
64 Check the Subversion documentation for more information.
65
66 @author: Kenneth J. Pronovici <pronovic@ieee.org>
67 """
68
69
70
71
72
73
74 import os
75 import logging
76 import pickle
77 from bz2 import BZ2File
78 from gzip import GzipFile
79
80
81 from CedarBackup2.xmlutil import createInputDom, addContainerNode, addStringNode
82 from CedarBackup2.xmlutil import isElement, readChildren, readFirstChild, readString, readStringList
83 from CedarBackup2.config import VALID_COLLECT_MODES, VALID_COMPRESS_MODES
84 from CedarBackup2.filesystem import FilesystemList
85 from CedarBackup2.util import UnorderedList, RegexList
86 from CedarBackup2.util import isStartOfWeek, buildNormalizedPath
87 from CedarBackup2.util import resolveCommand, executeCommand
88 from CedarBackup2.util import ObjectTypeList, encodePath, changeOwnership
89
90
91
92
93
94
95 logger = logging.getLogger("CedarBackup2.log.extend.subversion")
96
97 SVNLOOK_COMMAND = [ "svnlook", ]
98 SVNADMIN_COMMAND = [ "svnadmin", ]
99
100 REVISION_PATH_EXTENSION = "svnlast"
108
109 """
110 Class representing Subversion repository directory.
111
112 A repository directory is a directory that contains one or more Subversion
113 repositories.
114
115 The following restrictions exist on data in this class:
116
117 - The directory path must be absolute.
118 - The collect mode must be one of the values in L{VALID_COLLECT_MODES}.
119 - The compress mode must be one of the values in L{VALID_COMPRESS_MODES}.
120
121 The repository type value is kept around just for reference. It doesn't
122 affect the behavior of the backup.
123
124 Relative exclusions are allowed here. However, there is no configured
125 ignore file, because repository dir backups are not recursive.
126
127 @sort: __init__, __repr__, __str__, __cmp__, directoryPath, collectMode, compressMode
128 """
129
130 - def __init__(self, repositoryType=None, directoryPath=None, collectMode=None, compressMode=None,
131 relativeExcludePaths=None, excludePatterns=None):
132 """
133 Constructor for the C{RepositoryDir} class.
134
135 @param repositoryType: Type of repository, for reference
136 @param directoryPath: Absolute path of the Subversion parent directory
137 @param collectMode: Overridden collect mode for this directory.
138 @param compressMode: Overridden compression mode for this directory.
139 @param relativeExcludePaths: List of relative paths to exclude.
140 @param excludePatterns: List of regular expression patterns to exclude
141 """
142 self._repositoryType = None
143 self._directoryPath = None
144 self._collectMode = None
145 self._compressMode = None
146 self._relativeExcludePaths = None
147 self._excludePatterns = None
148 self.repositoryType = repositoryType
149 self.directoryPath = directoryPath
150 self.collectMode = collectMode
151 self.compressMode = compressMode
152 self.relativeExcludePaths = relativeExcludePaths
153 self.excludePatterns = excludePatterns
154
156 """
157 Official string representation for class instance.
158 """
159 return "RepositoryDir(%s, %s, %s, %s, %s, %s)" % (self.repositoryType, self.directoryPath, self.collectMode,
160 self.compressMode, self.relativeExcludePaths, self.excludePatterns)
161
163 """
164 Informal string representation for class instance.
165 """
166 return self.__repr__()
167
207
209 """
210 Property target used to set the repository type.
211 There is no validation; this value is kept around just for reference.
212 """
213 self._repositoryType = value
214
216 """
217 Property target used to get the repository type.
218 """
219 return self._repositoryType
220
222 """
223 Property target used to set the directory path.
224 The value must be an absolute path if it is not C{None}.
225 It does not have to exist on disk at the time of assignment.
226 @raise ValueError: If the value is not an absolute path.
227 @raise ValueError: If the value cannot be encoded properly.
228 """
229 if value is not None:
230 if not os.path.isabs(value):
231 raise ValueError("Repository path must be an absolute path.")
232 self._directoryPath = encodePath(value)
233
235 """
236 Property target used to get the repository path.
237 """
238 return self._directoryPath
239
241 """
242 Property target used to set the collect mode.
243 If not C{None}, the mode must be one of the values in L{VALID_COLLECT_MODES}.
244 @raise ValueError: If the value is not valid.
245 """
246 if value is not None:
247 if value not in VALID_COLLECT_MODES:
248 raise ValueError("Collect mode must be one of %s." % VALID_COLLECT_MODES)
249 self._collectMode = value
250
252 """
253 Property target used to get the collect mode.
254 """
255 return self._collectMode
256
258 """
259 Property target used to set the compress mode.
260 If not C{None}, the mode must be one of the values in L{VALID_COMPRESS_MODES}.
261 @raise ValueError: If the value is not valid.
262 """
263 if value is not None:
264 if value not in VALID_COMPRESS_MODES:
265 raise ValueError("Compress mode must be one of %s." % VALID_COMPRESS_MODES)
266 self._compressMode = value
267
269 """
270 Property target used to get the compress mode.
271 """
272 return self._compressMode
273
275 """
276 Property target used to set the relative exclude paths list.
277 Elements do not have to exist on disk at the time of assignment.
278 """
279 if value is None:
280 self._relativeExcludePaths = None
281 else:
282 try:
283 saved = self._relativeExcludePaths
284 self._relativeExcludePaths = UnorderedList()
285 self._relativeExcludePaths.extend(value)
286 except Exception, e:
287 self._relativeExcludePaths = saved
288 raise e
289
291 """
292 Property target used to get the relative exclude paths list.
293 """
294 return self._relativeExcludePaths
295
297 """
298 Property target used to set the exclude patterns list.
299 """
300 if value is None:
301 self._excludePatterns = None
302 else:
303 try:
304 saved = self._excludePatterns
305 self._excludePatterns = RegexList()
306 self._excludePatterns.extend(value)
307 except Exception, e:
308 self._excludePatterns = saved
309 raise e
310
312 """
313 Property target used to get the exclude patterns list.
314 """
315 return self._excludePatterns
316
317 repositoryType = property(_getRepositoryType, _setRepositoryType, None, doc="Type of this repository, for reference.")
318 directoryPath = property(_getDirectoryPath, _setDirectoryPath, None, doc="Absolute path of the Subversion parent directory.")
319 collectMode = property(_getCollectMode, _setCollectMode, None, doc="Overridden collect mode for this repository.")
320 compressMode = property(_getCompressMode, _setCompressMode, None, doc="Overridden compress mode for this repository.")
321 relativeExcludePaths = property(_getRelativeExcludePaths, _setRelativeExcludePaths, None, "List of relative paths to exclude.")
322 excludePatterns = property(_getExcludePatterns, _setExcludePatterns, None, "List of regular expression patterns to exclude.")
323
330
331 """
332 Class representing generic Subversion repository configuration..
333
334 The following restrictions exist on data in this class:
335
336 - The respository path must be absolute.
337 - The collect mode must be one of the values in L{VALID_COLLECT_MODES}.
338 - The compress mode must be one of the values in L{VALID_COMPRESS_MODES}.
339
340 The repository type value is kept around just for reference. It doesn't
341 affect the behavior of the backup.
342
343 @sort: __init__, __repr__, __str__, __cmp__, repositoryPath, collectMode, compressMode
344 """
345
346 - def __init__(self, repositoryType=None, repositoryPath=None, collectMode=None, compressMode=None):
347 """
348 Constructor for the C{Repository} class.
349
350 @param repositoryType: Type of repository, for reference
351 @param repositoryPath: Absolute path to a Subversion repository on disk.
352 @param collectMode: Overridden collect mode for this directory.
353 @param compressMode: Overridden compression mode for this directory.
354 """
355 self._repositoryType = None
356 self._repositoryPath = None
357 self._collectMode = None
358 self._compressMode = None
359 self.repositoryType = repositoryType
360 self.repositoryPath = repositoryPath
361 self.collectMode = collectMode
362 self.compressMode = compressMode
363
369
371 """
372 Informal string representation for class instance.
373 """
374 return self.__repr__()
375
405
407 """
408 Property target used to set the repository type.
409 There is no validation; this value is kept around just for reference.
410 """
411 self._repositoryType = value
412
414 """
415 Property target used to get the repository type.
416 """
417 return self._repositoryType
418
420 """
421 Property target used to set the repository path.
422 The value must be an absolute path if it is not C{None}.
423 It does not have to exist on disk at the time of assignment.
424 @raise ValueError: If the value is not an absolute path.
425 @raise ValueError: If the value cannot be encoded properly.
426 """
427 if value is not None:
428 if not os.path.isabs(value):
429 raise ValueError("Repository path must be an absolute path.")
430 self._repositoryPath = encodePath(value)
431
433 """
434 Property target used to get the repository path.
435 """
436 return self._repositoryPath
437
439 """
440 Property target used to set the collect mode.
441 If not C{None}, the mode must be one of the values in L{VALID_COLLECT_MODES}.
442 @raise ValueError: If the value is not valid.
443 """
444 if value is not None:
445 if value not in VALID_COLLECT_MODES:
446 raise ValueError("Collect mode must be one of %s." % VALID_COLLECT_MODES)
447 self._collectMode = value
448
450 """
451 Property target used to get the collect mode.
452 """
453 return self._collectMode
454
456 """
457 Property target used to set the compress mode.
458 If not C{None}, the mode must be one of the values in L{VALID_COMPRESS_MODES}.
459 @raise ValueError: If the value is not valid.
460 """
461 if value is not None:
462 if value not in VALID_COMPRESS_MODES:
463 raise ValueError("Compress mode must be one of %s." % VALID_COMPRESS_MODES)
464 self._compressMode = value
465
467 """
468 Property target used to get the compress mode.
469 """
470 return self._compressMode
471
472 repositoryType = property(_getRepositoryType, _setRepositoryType, None, doc="Type of this repository, for reference.")
473 repositoryPath = property(_getRepositoryPath, _setRepositoryPath, None, doc="Path to the repository to collect.")
474 collectMode = property(_getCollectMode, _setCollectMode, None, doc="Overridden collect mode for this repository.")
475 compressMode = property(_getCompressMode, _setCompressMode, None, doc="Overridden compress mode for this repository.")
476
483
484 """
485 Class representing Subversion configuration.
486
487 Subversion configuration is used for backing up Subversion repositories.
488
489 The following restrictions exist on data in this class:
490
491 - The collect mode must be one of the values in L{VALID_COLLECT_MODES}.
492 - The compress mode must be one of the values in L{VALID_COMPRESS_MODES}.
493 - The repositories list must be a list of C{Repository} objects.
494 - The repositoryDirs list must be a list of C{RepositoryDir} objects.
495
496 For the two lists, validation is accomplished through the
497 L{util.ObjectTypeList} list implementation that overrides common list
498 methods and transparently ensures that each element has the correct type.
499
500 @note: Lists within this class are "unordered" for equality comparisons.
501
502 @sort: __init__, __repr__, __str__, __cmp__, collectMode, compressMode, repositories
503 """
504
505 - def __init__(self, collectMode=None, compressMode=None, repositories=None, repositoryDirs=None):
506 """
507 Constructor for the C{SubversionConfig} class.
508
509 @param collectMode: Default collect mode.
510 @param compressMode: Default compress mode.
511 @param repositories: List of Subversion repositories to back up.
512 @param repositoryDirs: List of Subversion parent directories to back up.
513
514 @raise ValueError: If one of the values is invalid.
515 """
516 self._collectMode = None
517 self._compressMode = None
518 self._repositories = None
519 self._repositoryDirs = None
520 self.collectMode = collectMode
521 self.compressMode = compressMode
522 self.repositories = repositories
523 self.repositoryDirs = repositoryDirs
524
530
532 """
533 Informal string representation for class instance.
534 """
535 return self.__repr__()
536
567
569 """
570 Property target used to set the collect mode.
571 If not C{None}, the mode must be one of the values in L{VALID_COLLECT_MODES}.
572 @raise ValueError: If the value is not valid.
573 """
574 if value is not None:
575 if value not in VALID_COLLECT_MODES:
576 raise ValueError("Collect mode must be one of %s." % VALID_COLLECT_MODES)
577 self._collectMode = value
578
580 """
581 Property target used to get the collect mode.
582 """
583 return self._collectMode
584
586 """
587 Property target used to set the compress mode.
588 If not C{None}, the mode must be one of the values in L{VALID_COMPRESS_MODES}.
589 @raise ValueError: If the value is not valid.
590 """
591 if value is not None:
592 if value not in VALID_COMPRESS_MODES:
593 raise ValueError("Compress mode must be one of %s." % VALID_COMPRESS_MODES)
594 self._compressMode = value
595
597 """
598 Property target used to get the compress mode.
599 """
600 return self._compressMode
601
603 """
604 Property target used to set the repositories list.
605 Either the value must be C{None} or each element must be a C{Repository}.
606 @raise ValueError: If the value is not a C{Repository}
607 """
608 if value is None:
609 self._repositories = None
610 else:
611 try:
612 saved = self._repositories
613 self._repositories = ObjectTypeList(Repository, "Repository")
614 self._repositories.extend(value)
615 except Exception, e:
616 self._repositories = saved
617 raise e
618
620 """
621 Property target used to get the repositories list.
622 """
623 return self._repositories
624
626 """
627 Property target used to set the repositoryDirs list.
628 Either the value must be C{None} or each element must be a C{Repository}.
629 @raise ValueError: If the value is not a C{Repository}
630 """
631 if value is None:
632 self._repositoryDirs = None
633 else:
634 try:
635 saved = self._repositoryDirs
636 self._repositoryDirs = ObjectTypeList(RepositoryDir, "RepositoryDir")
637 self._repositoryDirs.extend(value)
638 except Exception, e:
639 self._repositoryDirs = saved
640 raise e
641
643 """
644 Property target used to get the repositoryDirs list.
645 """
646 return self._repositoryDirs
647
648 collectMode = property(_getCollectMode, _setCollectMode, None, doc="Default collect mode.")
649 compressMode = property(_getCompressMode, _setCompressMode, None, doc="Default compress mode.")
650 repositories = property(_getRepositories, _setRepositories, None, doc="List of Subversion repositories to back up.")
651 repositoryDirs = property(_getRepositoryDirs, _setRepositoryDirs, None, doc="List of Subversion parent directories to back up.")
652
659
660 """
661 Class representing this extension's configuration document.
662
663 This is not a general-purpose configuration object like the main Cedar
664 Backup configuration object. Instead, it just knows how to parse and emit
665 Subversion-specific configuration values. Third parties who need to read
666 and write configuration related to this extension should access it through
667 the constructor, C{validate} and C{addConfig} methods.
668
669 @note: Lists within this class are "unordered" for equality comparisons.
670
671 @sort: __init__, __repr__, __str__, __cmp__, subversion, validate, addConfig
672 """
673
674 - def __init__(self, xmlData=None, xmlPath=None, validate=True):
675 """
676 Initializes a configuration object.
677
678 If you initialize the object without passing either C{xmlData} or
679 C{xmlPath} then configuration will be empty and will be invalid until it
680 is filled in properly.
681
682 No reference to the original XML data or original path is saved off by
683 this class. Once the data has been parsed (successfully or not) this
684 original information is discarded.
685
686 Unless the C{validate} argument is C{False}, the L{LocalConfig.validate}
687 method will be called (with its default arguments) against configuration
688 after successfully parsing any passed-in XML. Keep in mind that even if
689 C{validate} is C{False}, it might not be possible to parse the passed-in
690 XML document if lower-level validations fail.
691
692 @note: It is strongly suggested that the C{validate} option always be set
693 to C{True} (the default) unless there is a specific need to read in
694 invalid configuration from disk.
695
696 @param xmlData: XML data representing configuration.
697 @type xmlData: String data.
698
699 @param xmlPath: Path to an XML file on disk.
700 @type xmlPath: Absolute path to a file on disk.
701
702 @param validate: Validate the document after parsing it.
703 @type validate: Boolean true/false.
704
705 @raise ValueError: If both C{xmlData} and C{xmlPath} are passed-in.
706 @raise ValueError: If the XML data in C{xmlData} or C{xmlPath} cannot be parsed.
707 @raise ValueError: If the parsed configuration document is not valid.
708 """
709 self._subversion = None
710 self.subversion = None
711 if xmlData is not None and xmlPath is not None:
712 raise ValueError("Use either xmlData or xmlPath, but not both.")
713 if xmlData is not None:
714 self._parseXmlData(xmlData)
715 if validate:
716 self.validate()
717 elif xmlPath is not None:
718 xmlData = open(xmlPath).read()
719 self._parseXmlData(xmlData)
720 if validate:
721 self.validate()
722
724 """
725 Official string representation for class instance.
726 """
727 return "LocalConfig(%s)" % (self.subversion)
728
730 """
731 Informal string representation for class instance.
732 """
733 return self.__repr__()
734
736 """
737 Definition of equals operator for this class.
738 Lists within this class are "unordered" for equality comparisons.
739 @param other: Other object to compare to.
740 @return: -1/0/1 depending on whether self is C{<}, C{=} or C{>} other.
741 """
742 if other is None:
743 return 1
744 if self.subversion != other.subversion:
745 if self.subversion < other.subversion:
746 return -1
747 else:
748 return 1
749 return 0
750
752 """
753 Property target used to set the subversion configuration value.
754 If not C{None}, the value must be a C{SubversionConfig} object.
755 @raise ValueError: If the value is not a C{SubversionConfig}
756 """
757 if value is None:
758 self._subversion = None
759 else:
760 if not isinstance(value, SubversionConfig):
761 raise ValueError("Value must be a C{SubversionConfig} object.")
762 self._subversion = value
763
765 """
766 Property target used to get the subversion configuration value.
767 """
768 return self._subversion
769
770 subversion = property(_getSubversion, _setSubversion, None, "Subversion configuration in terms of a C{SubversionConfig} object.")
771
773 """
774 Validates configuration represented by the object.
775
776 Subversion configuration must be filled in. Within that, the collect
777 mode and compress mode are both optional, but the list of repositories
778 must contain at least one entry.
779
780 Each repository must contain a repository path, and then must be either
781 able to take collect mode and compress mode configuration from the parent
782 C{SubversionConfig} object, or must set each value on its own.
783
784 @raise ValueError: If one of the validations fails.
785 """
786 if self.subversion is None:
787 raise ValueError("Subversion section is required.")
788 if ((self.subversion.repositories is None or len(self.subversion.repositories) < 1) and
789 (self.subversion.repositoryDirs is None or len(self.subversion.repositoryDirs) <1)):
790 raise ValueError("At least one Subversion repository must be configured.")
791 if self.subversion.repositories is not None:
792 for repository in self.subversion.repositories:
793 if repository.repositoryPath is None:
794 raise ValueError("Each repository must set a repository path.")
795 if self.subversion.collectMode is None and repository.collectMode is None:
796 raise ValueError("Collect mode must either be set in parent section or individual repository.")
797 if self.subversion.compressMode is None and repository.compressMode is None:
798 raise ValueError("Compress mode must either be set in parent section or individual repository.")
799 if self.subversion.repositoryDirs is not None:
800 for repositoryDir in self.subversion.repositoryDirs:
801 if repositoryDir.directoryPath is None:
802 raise ValueError("Each repository directory must set a directory path.")
803 if self.subversion.collectMode is None and repositoryDir.collectMode is None:
804 raise ValueError("Collect mode must either be set in parent section or repository directory.")
805 if self.subversion.compressMode is None and repositoryDir.compressMode is None:
806 raise ValueError("Compress mode must either be set in parent section or repository directory.")
807
809 """
810 Adds a <subversion> configuration section as the next child of a parent.
811
812 Third parties should use this function to write configuration related to
813 this extension.
814
815 We add the following fields to the document::
816
817 collectMode //cb_config/subversion/collectMode
818 compressMode //cb_config/subversion/compressMode
819
820 We also add groups of the following items, one list element per
821 item::
822
823 repository //cb_config/subversion/repository
824 repository_dir //cb_config/subversion/repository_dir
825
826 @param xmlDom: DOM tree as from C{impl.createDocument()}.
827 @param parentNode: Parent that the section should be appended to.
828 """
829 if self.subversion is not None:
830 sectionNode = addContainerNode(xmlDom, parentNode, "subversion")
831 addStringNode(xmlDom, sectionNode, "collect_mode", self.subversion.collectMode)
832 addStringNode(xmlDom, sectionNode, "compress_mode", self.subversion.compressMode)
833 if self.subversion.repositories is not None:
834 for repository in self.subversion.repositories:
835 LocalConfig._addRepository(xmlDom, sectionNode, repository)
836 if self.subversion.repositoryDirs is not None:
837 for repositoryDir in self.subversion.repositoryDirs:
838 LocalConfig._addRepositoryDir(xmlDom, sectionNode, repositoryDir)
839
841 """
842 Internal method to parse an XML string into the object.
843
844 This method parses the XML document into a DOM tree (C{xmlDom}) and then
845 calls a static method to parse the subversion configuration section.
846
847 @param xmlData: XML data to be parsed
848 @type xmlData: String data
849
850 @raise ValueError: If the XML cannot be successfully parsed.
851 """
852 (xmlDom, parentNode) = createInputDom(xmlData)
853 self._subversion = LocalConfig._parseSubversion(parentNode)
854
855 @staticmethod
857 """
858 Parses a subversion configuration section.
859
860 We read the following individual fields::
861
862 collectMode //cb_config/subversion/collect_mode
863 compressMode //cb_config/subversion/compress_mode
864
865 We also read groups of the following item, one list element per
866 item::
867
868 repositories //cb_config/subversion/repository
869 repository_dirs //cb_config/subversion/repository_dir
870
871 The repositories are parsed by L{_parseRepositories}, and the repository
872 dirs are parsed by L{_parseRepositoryDirs}.
873
874 @param parent: Parent node to search beneath.
875
876 @return: C{SubversionConfig} object or C{None} if the section does not exist.
877 @raise ValueError: If some filled-in value is invalid.
878 """
879 subversion = None
880 section = readFirstChild(parent, "subversion")
881 if section is not None:
882 subversion = SubversionConfig()
883 subversion.collectMode = readString(section, "collect_mode")
884 subversion.compressMode = readString(section, "compress_mode")
885 subversion.repositories = LocalConfig._parseRepositories(section)
886 subversion.repositoryDirs = LocalConfig._parseRepositoryDirs(section)
887 return subversion
888
889 @staticmethod
891 """
892 Reads a list of C{Repository} objects from immediately beneath the parent.
893
894 We read the following individual fields::
895
896 repositoryType type
897 repositoryPath abs_path
898 collectMode collect_mode
899 compressMode compess_mode
900
901 The type field is optional, and its value is kept around only for
902 reference.
903
904 @param parent: Parent node to search beneath.
905
906 @return: List of C{Repository} objects or C{None} if none are found.
907 @raise ValueError: If some filled-in value is invalid.
908 """
909 lst = []
910 for entry in readChildren(parent, "repository"):
911 if isElement(entry):
912 repository = Repository()
913 repository.repositoryType = readString(entry, "type")
914 repository.repositoryPath = readString(entry, "abs_path")
915 repository.collectMode = readString(entry, "collect_mode")
916 repository.compressMode = readString(entry, "compress_mode")
917 lst.append(repository)
918 if lst == []:
919 lst = None
920 return lst
921
922 @staticmethod
924 """
925 Adds a repository container as the next child of a parent.
926
927 We add the following fields to the document::
928
929 repositoryType repository/type
930 repositoryPath repository/abs_path
931 collectMode repository/collect_mode
932 compressMode repository/compress_mode
933
934 The <repository> node itself is created as the next child of the parent
935 node. This method only adds one repository node. The parent must loop
936 for each repository in the C{SubversionConfig} object.
937
938 If C{repository} is C{None}, this method call will be a no-op.
939
940 @param xmlDom: DOM tree as from C{impl.createDocument()}.
941 @param parentNode: Parent that the section should be appended to.
942 @param repository: Repository to be added to the document.
943 """
944 if repository is not None:
945 sectionNode = addContainerNode(xmlDom, parentNode, "repository")
946 addStringNode(xmlDom, sectionNode, "type", repository.repositoryType)
947 addStringNode(xmlDom, sectionNode, "abs_path", repository.repositoryPath)
948 addStringNode(xmlDom, sectionNode, "collect_mode", repository.collectMode)
949 addStringNode(xmlDom, sectionNode, "compress_mode", repository.compressMode)
950
951 @staticmethod
953 """
954 Reads a list of C{RepositoryDir} objects from immediately beneath the parent.
955
956 We read the following individual fields::
957
958 repositoryType type
959 directoryPath abs_path
960 collectMode collect_mode
961 compressMode compess_mode
962
963 We also read groups of the following items, one list element per
964 item::
965
966 relativeExcludePaths exclude/rel_path
967 excludePatterns exclude/pattern
968
969 The exclusions are parsed by L{_parseExclusions}.
970
971 The type field is optional, and its value is kept around only for
972 reference.
973
974 @param parent: Parent node to search beneath.
975
976 @return: List of C{RepositoryDir} objects or C{None} if none are found.
977 @raise ValueError: If some filled-in value is invalid.
978 """
979 lst = []
980 for entry in readChildren(parent, "repository_dir"):
981 if isElement(entry):
982 repositoryDir = RepositoryDir()
983 repositoryDir.repositoryType = readString(entry, "type")
984 repositoryDir.directoryPath = readString(entry, "abs_path")
985 repositoryDir.collectMode = readString(entry, "collect_mode")
986 repositoryDir.compressMode = readString(entry, "compress_mode")
987 (repositoryDir.relativeExcludePaths, repositoryDir.excludePatterns) = LocalConfig._parseExclusions(entry)
988 lst.append(repositoryDir)
989 if lst == []:
990 lst = None
991 return lst
992
993 @staticmethod
995 """
996 Reads exclusions data from immediately beneath the parent.
997
998 We read groups of the following items, one list element per item::
999
1000 relative exclude/rel_path
1001 patterns exclude/pattern
1002
1003 If there are none of some pattern (i.e. no relative path items) then
1004 C{None} will be returned for that item in the tuple.
1005
1006 @param parentNode: Parent node to search beneath.
1007
1008 @return: Tuple of (relative, patterns) exclusions.
1009 """
1010 section = readFirstChild(parentNode, "exclude")
1011 if section is None:
1012 return (None, None)
1013 else:
1014 relative = readStringList(section, "rel_path")
1015 patterns = readStringList(section, "pattern")
1016 return (relative, patterns)
1017
1018 @staticmethod
1020 """
1021 Adds a repository dir container as the next child of a parent.
1022
1023 We add the following fields to the document::
1024
1025 repositoryType repository_dir/type
1026 directoryPath repository_dir/abs_path
1027 collectMode repository_dir/collect_mode
1028 compressMode repository_dir/compress_mode
1029
1030 We also add groups of the following items, one list element per item::
1031
1032 relativeExcludePaths dir/exclude/rel_path
1033 excludePatterns dir/exclude/pattern
1034
1035 The <repository_dir> node itself is created as the next child of the
1036 parent node. This method only adds one repository node. The parent must
1037 loop for each repository dir in the C{SubversionConfig} object.
1038
1039 If C{repositoryDir} is C{None}, this method call will be a no-op.
1040
1041 @param xmlDom: DOM tree as from C{impl.createDocument()}.
1042 @param parentNode: Parent that the section should be appended to.
1043 @param repositoryDir: Repository dir to be added to the document.
1044 """
1045 if repositoryDir is not None:
1046 sectionNode = addContainerNode(xmlDom, parentNode, "repository_dir")
1047 addStringNode(xmlDom, sectionNode, "type", repositoryDir.repositoryType)
1048 addStringNode(xmlDom, sectionNode, "abs_path", repositoryDir.directoryPath)
1049 addStringNode(xmlDom, sectionNode, "collect_mode", repositoryDir.collectMode)
1050 addStringNode(xmlDom, sectionNode, "compress_mode", repositoryDir.compressMode)
1051 if ((repositoryDir.relativeExcludePaths is not None and repositoryDir.relativeExcludePaths != []) or
1052 (repositoryDir.excludePatterns is not None and repositoryDir.excludePatterns != [])):
1053 excludeNode = addContainerNode(xmlDom, sectionNode, "exclude")
1054 if repositoryDir.relativeExcludePaths is not None:
1055 for relativePath in repositoryDir.relativeExcludePaths:
1056 addStringNode(xmlDom, excludeNode, "rel_path", relativePath)
1057 if repositoryDir.excludePatterns is not None:
1058 for pattern in repositoryDir.excludePatterns:
1059 addStringNode(xmlDom, excludeNode, "pattern", pattern)
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070 -def executeAction(configPath, options, config):
1071 """
1072 Executes the Subversion backup action.
1073
1074 @param configPath: Path to configuration file on disk.
1075 @type configPath: String representing a path on disk.
1076
1077 @param options: Program command-line options.
1078 @type options: Options object.
1079
1080 @param config: Program configuration.
1081 @type config: Config object.
1082
1083 @raise ValueError: Under many generic error conditions
1084 @raise IOError: If a backup could not be written for some reason.
1085 """
1086 logger.debug("Executing Subversion extended action.")
1087 if config.options is None or config.collect is None:
1088 raise ValueError("Cedar Backup configuration is not properly filled in.")
1089 local = LocalConfig(xmlPath=configPath)
1090 todayIsStart = isStartOfWeek(config.options.startingDay)
1091 fullBackup = options.full or todayIsStart
1092 logger.debug("Full backup flag is [%s]" % fullBackup)
1093 if local.subversion.repositories is not None:
1094 for repository in local.subversion.repositories:
1095 _backupRepository(config, local, todayIsStart, fullBackup, repository)
1096 if local.subversion.repositoryDirs is not None:
1097 for repositoryDir in local.subversion.repositoryDirs:
1098 logger.debug("Working with repository directory [%s]." % repositoryDir.directoryPath)
1099 for repositoryPath in _getRepositoryPaths(repositoryDir):
1100 repository = Repository(repositoryDir.repositoryType, repositoryPath,
1101 repositoryDir.collectMode, repositoryDir.compressMode)
1102 _backupRepository(config, local, todayIsStart, fullBackup, repository)
1103 logger.info("Completed backing up Subversion repository directory [%s]." % repositoryDir.directoryPath)
1104 logger.info("Executed the Subversion extended action successfully.")
1105
1119
1134
1136 """
1137 Gets the path to the revision file associated with a repository.
1138 @param config: Config object.
1139 @param repository: Repository object.
1140 @return: Absolute path to the revision file associated with the repository.
1141 """
1142 normalized = buildNormalizedPath(repository.repositoryPath)
1143 filename = "%s.%s" % (normalized, REVISION_PATH_EXTENSION)
1144 revisionPath = os.path.join(config.options.workingDir, filename)
1145 logger.debug("Revision file path is [%s]" % revisionPath)
1146 return revisionPath
1147
1148 -def _getBackupPath(config, repositoryPath, compressMode, startRevision, endRevision):
1149 """
1150 Gets the backup file path (including correct extension) associated with a repository.
1151 @param config: Config object.
1152 @param repositoryPath: Path to the indicated repository
1153 @param compressMode: Compress mode to use for this repository.
1154 @param startRevision: Starting repository revision.
1155 @param endRevision: Ending repository revision.
1156 @return: Absolute path to the backup file associated with the repository.
1157 """
1158 normalizedPath = buildNormalizedPath(repositoryPath)
1159 filename = "svndump-%d:%d-%s.txt" % (startRevision, endRevision, normalizedPath)
1160 if compressMode == 'gzip':
1161 filename = "%s.gz" % filename
1162 elif compressMode == 'bzip2':
1163 filename = "%s.bz2" % filename
1164 backupPath = os.path.join(config.collect.targetDir, filename)
1165 logger.debug("Backup file path is [%s]" % backupPath)
1166 return backupPath
1167
1181
1183 """
1184 Gets exclusions (file and patterns) associated with an repository directory.
1185
1186 The returned files value is a list of absolute paths to be excluded from the
1187 backup for a given directory. It is derived from the repository directory's
1188 relative exclude paths.
1189
1190 The returned patterns value is a list of patterns to be excluded from the
1191 backup for a given directory. It is derived from the repository directory's
1192 list of patterns.
1193
1194 @param repositoryDir: Repository directory object.
1195
1196 @return: Tuple (files, patterns) indicating what to exclude.
1197 """
1198 paths = []
1199 if repositoryDir.relativeExcludePaths is not None:
1200 for relativePath in repositoryDir.relativeExcludePaths:
1201 paths.append(os.path.join(repositoryDir.directoryPath, relativePath))
1202 patterns = []
1203 if repositoryDir.excludePatterns is not None:
1204 patterns.extend(repositoryDir.excludePatterns)
1205 logger.debug("Exclude paths: %s" % paths)
1206 logger.debug("Exclude patterns: %s" % patterns)
1207 return(paths, patterns)
1208
1210 """
1211 Backs up an individual Subversion repository.
1212
1213 This internal method wraps the public methods and adds some functionality
1214 to work better with the extended action itself.
1215
1216 @param config: Cedar Backup configuration.
1217 @param local: Local configuration
1218 @param todayIsStart: Indicates whether today is start of week
1219 @param fullBackup: Full backup flag
1220 @param repository: Repository to operate on
1221
1222 @raise ValueError: If some value is missing or invalid.
1223 @raise IOError: If there is a problem executing the Subversion dump.
1224 """
1225 logger.debug("Working with repository [%s]" % repository.repositoryPath)
1226 logger.debug("Repository type is [%s]" % repository.repositoryType)
1227 collectMode = _getCollectMode(local, repository)
1228 compressMode = _getCompressMode(local, repository)
1229 revisionPath = _getRevisionPath(config, repository)
1230 if not (fullBackup or (collectMode in ['daily', 'incr', ]) or (collectMode == 'weekly' and todayIsStart)):
1231 logger.debug("Repository will not be backed up, per collect mode.")
1232 return
1233 logger.debug("Repository meets criteria to be backed up today.")
1234 if collectMode != "incr" or fullBackup:
1235 startRevision = 0
1236 endRevision = getYoungestRevision(repository.repositoryPath)
1237 logger.debug("Using full backup, revision: (%d, %d)." % (startRevision, endRevision))
1238 else:
1239 if fullBackup:
1240 startRevision = 0
1241 endRevision = getYoungestRevision(repository.repositoryPath)
1242 else:
1243 startRevision = _loadLastRevision(revisionPath) + 1
1244 endRevision = getYoungestRevision(repository.repositoryPath)
1245 if startRevision > endRevision:
1246 logger.info("No need to back up repository [%s]; no new revisions." % repository.repositoryPath)
1247 return
1248 logger.debug("Using incremental backup, revision: (%d, %d)." % (startRevision, endRevision))
1249 backupPath = _getBackupPath(config, repository.repositoryPath, compressMode, startRevision, endRevision)
1250 outputFile = _getOutputFile(backupPath, compressMode)
1251 try:
1252 backupRepository(repository.repositoryPath, outputFile, startRevision, endRevision)
1253 finally:
1254 outputFile.close()
1255 if not os.path.exists(backupPath):
1256 raise IOError("Dump file [%s] does not seem to exist after backup completed." % backupPath)
1257 changeOwnership(backupPath, config.options.backupUser, config.options.backupGroup)
1258 if collectMode == "incr":
1259 _writeLastRevision(config, revisionPath, endRevision)
1260 logger.info("Completed backing up Subversion repository [%s]." % repository.repositoryPath)
1261
1263 """
1264 Opens the output file used for saving the Subversion dump.
1265
1266 If the compress mode is "gzip", we'll open a C{GzipFile}, and if the
1267 compress mode is "bzip2", we'll open a C{BZ2File}. Otherwise, we'll just
1268 return an object from the normal C{open()} method.
1269
1270 @param backupPath: Path to file to open.
1271 @param compressMode: Compress mode of file ("none", "gzip", "bzip").
1272
1273 @return: Output file object.
1274 """
1275 if compressMode == "gzip":
1276 return GzipFile(backupPath, "w")
1277 elif compressMode == "bzip2":
1278 return BZ2File(backupPath, "w")
1279 else:
1280 return open(backupPath, "w")
1281
1283 """
1284 Loads the indicated revision file from disk into an integer.
1285
1286 If we can't load the revision file successfully (either because it doesn't
1287 exist or for some other reason), then a revision of -1 will be returned -
1288 but the condition will be logged. This way, we err on the side of backing
1289 up too much, because anyone using this will presumably be adding 1 to the
1290 revision, so they don't duplicate any backups.
1291
1292 @param revisionPath: Path to the revision file on disk.
1293
1294 @return: Integer representing last backed-up revision, -1 on error or if none can be read.
1295 """
1296 if not os.path.isfile(revisionPath):
1297 startRevision = -1
1298 logger.debug("Revision file [%s] does not exist on disk." % revisionPath)
1299 else:
1300 try:
1301 startRevision = pickle.load(open(revisionPath, "r"))
1302 logger.debug("Loaded revision file [%s] from disk: %d." % (revisionPath, startRevision))
1303 except:
1304 startRevision = -1
1305 logger.error("Failed loading revision file [%s] from disk." % revisionPath)
1306 return startRevision
1307
1309 """
1310 Writes the end revision to the indicated revision file on disk.
1311
1312 If we can't write the revision file successfully for any reason, we'll log
1313 the condition but won't throw an exception.
1314
1315 @param config: Config object.
1316 @param revisionPath: Path to the revision file on disk.
1317 @param endRevision: Last revision backed up on this run.
1318 """
1319 try:
1320 pickle.dump(endRevision, open(revisionPath, "w"))
1321 changeOwnership(revisionPath, config.options.backupUser, config.options.backupGroup)
1322 logger.debug("Wrote new revision file [%s] to disk: %d." % (revisionPath, endRevision))
1323 except:
1324 logger.error("Failed to write revision file [%s] to disk." % revisionPath)
1325
1326
1327
1328
1329
1330
1331 -def backupRepository(repositoryPath, backupFile, startRevision=None, endRevision=None):
1332 """
1333 Backs up an individual Subversion repository.
1334
1335 The starting and ending revision values control an incremental backup. If
1336 the starting revision is not passed in, then revision zero (the start of the
1337 repository) is assumed. If the ending revision is not passed in, then the
1338 youngest revision in the database will be used as the endpoint.
1339
1340 The backup data will be written into the passed-in back file. Normally,
1341 this would be an object as returned from C{open}, but it is possible to use
1342 something like a C{GzipFile} to write compressed output. The caller is
1343 responsible for closing the passed-in backup file.
1344
1345 @note: This function should either be run as root or as the owner of the
1346 Subversion repository.
1347
1348 @note: It is apparently I{not} a good idea to interrupt this function.
1349 Sometimes, this leaves the repository in a "wedged" state, which requires
1350 recovery using C{svnadmin recover}.
1351
1352 @param repositoryPath: Path to Subversion repository to back up
1353 @type repositoryPath: String path representing Subversion repository on disk.
1354
1355 @param backupFile: Python file object to use for writing backup.
1356 @type backupFile: Python file object as from C{open()} or C{file()}.
1357
1358 @param startRevision: Starting repository revision to back up (for incremental backups)
1359 @type startRevision: Integer value >= 0.
1360
1361 @param endRevision: Ending repository revision to back up (for incremental backups)
1362 @type endRevision: Integer value >= 0.
1363
1364 @raise ValueError: If some value is missing or invalid.
1365 @raise IOError: If there is a problem executing the Subversion dump.
1366 """
1367 if startRevision is None:
1368 startRevision = 0
1369 if endRevision is None:
1370 endRevision = getYoungestRevision(repositoryPath)
1371 if int(startRevision) < 0:
1372 raise ValueError("Start revision must be >= 0.")
1373 if int(endRevision) < 0:
1374 raise ValueError("End revision must be >= 0.")
1375 if startRevision > endRevision:
1376 raise ValueError("Start revision must be <= end revision.")
1377 args = [ "dump", "--quiet", "-r%s:%s" % (startRevision, endRevision), "--incremental", repositoryPath, ]
1378 command = resolveCommand(SVNADMIN_COMMAND)
1379 result = executeCommand(command, args, returnOutput=False, ignoreStderr=True, doNotLog=True, outputFile=backupFile)[0]
1380 if result != 0:
1381 raise IOError("Error [%d] executing Subversion dump for repository [%s]." % (result, repositoryPath))
1382 logger.debug("Completed dumping subversion repository [%s]." % repositoryPath)
1383
1390 """
1391 Gets the youngest (newest) revision in a Subversion repository using C{svnlook}.
1392
1393 @note: This function should either be run as root or as the owner of the
1394 Subversion repository.
1395
1396 @param repositoryPath: Path to Subversion repository to look in.
1397 @type repositoryPath: String path representing Subversion repository on disk.
1398
1399 @return: Youngest revision as an integer.
1400
1401 @raise ValueError: If there is a problem parsing the C{svnlook} output.
1402 @raise IOError: If there is a problem executing the C{svnlook} command.
1403 """
1404 args = [ 'youngest', repositoryPath, ]
1405 command = resolveCommand(SVNLOOK_COMMAND)
1406 (result, output) = executeCommand(command, args, returnOutput=True, ignoreStderr=True)
1407 if result != 0:
1408 raise IOError("Error [%d] executing 'svnlook youngest' for repository [%s]." % (result, repositoryPath))
1409 if len(output) != 1:
1410 raise ValueError("Unable to parse 'svnlook youngest' output.")
1411 return int(output[0])
1412
1419
1420 """
1421 Class representing Subversion BDB (Berkeley Database) repository configuration.
1422 This object is deprecated. Use a simple L{Repository} instead.
1423 """
1424
1425 - def __init__(self, repositoryPath=None, collectMode=None, compressMode=None):
1430
1436
1439
1440 """
1441 Class representing Subversion FSFS repository configuration.
1442 This object is deprecated. Use a simple L{Repository} instead.
1443 """
1444
1445 - def __init__(self, repositoryPath=None, collectMode=None, compressMode=None):
1450
1456
1457
1458 -def backupBDBRepository(repositoryPath, backupFile, startRevision=None, endRevision=None):
1459 """
1460 Backs up an individual Subversion BDB repository.
1461 This function is deprecated. Use L{backupRepository} instead.
1462 """
1463 return backupRepository(repositoryPath, backupFile, startRevision, endRevision)
1464
1467 """
1468 Backs up an individual Subversion FSFS repository.
1469 This function is deprecated. Use L{backupRepository} instead.
1470 """
1471 return backupRepository(repositoryPath, backupFile, startRevision, endRevision)
1472