1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38 """
39 Provides an extension to split up large files in staging directories.
40
41 When this extension is executed, it will look through the configured Cedar
42 Backup staging directory for files exceeding a specified size limit, and split
43 them down into smaller files using the 'split' utility. Any directory which
44 has already been split (as indicated by the C{cback.split} file) will be
45 ignored.
46
47 This extension requires a new configuration section <split> and is intended
48 to be run immediately after the standard stage action or immediately before the
49 standard store action. Aside from its own configuration, it requires the
50 options and staging configuration sections in the standard Cedar Backup
51 configuration file.
52
53 @author: Kenneth J. Pronovici <pronovic@ieee.org>
54 """
55
56
57
58
59
60
61 import os
62 import re
63 import logging
64 from functools import total_ordering
65
66
67 from CedarBackup3.util import resolveCommand, executeCommand, changeOwnership
68 from CedarBackup3.xmlutil import createInputDom, addContainerNode
69 from CedarBackup3.xmlutil import readFirstChild
70 from CedarBackup3.actions.util import findDailyDirs, writeIndicatorFile, getBackupFiles
71 from CedarBackup3.config import ByteQuantity, readByteQuantity, addByteQuantityNode
72
73
74
75
76
77
78 logger = logging.getLogger("CedarBackup3.log.extend.split")
79
80 SPLIT_COMMAND = [ "split", ]
81 SPLIT_INDICATOR = "cback.split"
82
83
84
85
86
87
88 @total_ordering
89 -class SplitConfig(object):
90
91 """
92 Class representing split configuration.
93
94 Split configuration is used for splitting staging directories.
95
96 The following restrictions exist on data in this class:
97
98 - The size limit must be a ByteQuantity
99 - The split size must be a ByteQuantity
100
101 @sort: __init__, __repr__, __str__, __cmp__, __eq__, __lt__, __gt__,
102 sizeLimit, splitSize
103 """
104
105 - def __init__(self, sizeLimit=None, splitSize=None):
106 """
107 Constructor for the C{SplitCOnfig} class.
108
109 @param sizeLimit: Size limit of the files, in bytes
110 @param splitSize: Size that files exceeding the limit will be split into, in bytes
111
112 @raise ValueError: If one of the values is invalid.
113 """
114 self._sizeLimit = None
115 self._splitSize = None
116 self.sizeLimit = sizeLimit
117 self.splitSize = splitSize
118
120 """
121 Official string representation for class instance.
122 """
123 return "SplitConfig(%s, %s)" % (self.sizeLimit, self.splitSize)
124
126 """
127 Informal string representation for class instance.
128 """
129 return self.__repr__()
130
132 """Equals operator, iplemented in terms of original Python 2 compare operator."""
133 return self.__cmp__(other) == 0
134
136 """Less-than operator, iplemented in terms of original Python 2 compare operator."""
137 return self.__cmp__(other) < 0
138
140 """Greater-than operator, iplemented in terms of original Python 2 compare operator."""
141 return self.__cmp__(other) > 0
142
144 """
145 Original Python 2 comparison operator.
146 Lists within this class are "unordered" for equality comparisons.
147 @param other: Other object to compare to.
148 @return: -1/0/1 depending on whether self is C{<}, C{=} or C{>} other.
149 """
150 if other is None:
151 return 1
152 if self.sizeLimit != other.sizeLimit:
153 if (self.sizeLimit or ByteQuantity()) < (other.sizeLimit or ByteQuantity()):
154 return -1
155 else:
156 return 1
157 if self.splitSize != other.splitSize:
158 if (self.splitSize or ByteQuantity()) < (other.splitSize or ByteQuantity()):
159 return -1
160 else:
161 return 1
162 return 0
163
165 """
166 Property target used to set the size limit.
167 If not C{None}, the value must be a C{ByteQuantity} object.
168 @raise ValueError: If the value is not a C{ByteQuantity}
169 """
170 if value is None:
171 self._sizeLimit = None
172 else:
173 if not isinstance(value, ByteQuantity):
174 raise ValueError("Value must be a C{ByteQuantity} object.")
175 self._sizeLimit = value
176
178 """
179 Property target used to get the size limit.
180 """
181 return self._sizeLimit
182
184 """
185 Property target used to set the split size.
186 If not C{None}, the value must be a C{ByteQuantity} object.
187 @raise ValueError: If the value is not a C{ByteQuantity}
188 """
189 if value is None:
190 self._splitSize = None
191 else:
192 if not isinstance(value, ByteQuantity):
193 raise ValueError("Value must be a C{ByteQuantity} object.")
194 self._splitSize = value
195
197 """
198 Property target used to get the split size.
199 """
200 return self._splitSize
201
202 sizeLimit = property(_getSizeLimit, _setSizeLimit, None, doc="Size limit, as a ByteQuantity")
203 splitSize = property(_getSplitSize, _setSplitSize, None, doc="Split size, as a ByteQuantity")
204
205
206
207
208
209
210 @total_ordering
211 -class LocalConfig(object):
212
213 """
214 Class representing this extension's configuration document.
215
216 This is not a general-purpose configuration object like the main Cedar
217 Backup configuration object. Instead, it just knows how to parse and emit
218 split-specific configuration values. Third parties who need to read and
219 write configuration related to this extension should access it through the
220 constructor, C{validate} and C{addConfig} methods.
221
222 @note: Lists within this class are "unordered" for equality comparisons.
223
224 @sort: __init__, __repr__, __str__, __cmp__, __eq__, __lt__, __gt__, split,
225 validate, addConfig
226 """
227
228 - def __init__(self, xmlData=None, xmlPath=None, validate=True):
229 """
230 Initializes a configuration object.
231
232 If you initialize the object without passing either C{xmlData} or
233 C{xmlPath} then configuration will be empty and will be invalid until it
234 is filled in properly.
235
236 No reference to the original XML data or original path is saved off by
237 this class. Once the data has been parsed (successfully or not) this
238 original information is discarded.
239
240 Unless the C{validate} argument is C{False}, the L{LocalConfig.validate}
241 method will be called (with its default arguments) against configuration
242 after successfully parsing any passed-in XML. Keep in mind that even if
243 C{validate} is C{False}, it might not be possible to parse the passed-in
244 XML document if lower-level validations fail.
245
246 @note: It is strongly suggested that the C{validate} option always be set
247 to C{True} (the default) unless there is a specific need to read in
248 invalid configuration from disk.
249
250 @param xmlData: XML data representing configuration.
251 @type xmlData: String data.
252
253 @param xmlPath: Path to an XML file on disk.
254 @type xmlPath: Absolute path to a file on disk.
255
256 @param validate: Validate the document after parsing it.
257 @type validate: Boolean true/false.
258
259 @raise ValueError: If both C{xmlData} and C{xmlPath} are passed-in.
260 @raise ValueError: If the XML data in C{xmlData} or C{xmlPath} cannot be parsed.
261 @raise ValueError: If the parsed configuration document is not valid.
262 """
263 self._split = None
264 self.split = None
265 if xmlData is not None and xmlPath is not None:
266 raise ValueError("Use either xmlData or xmlPath, but not both.")
267 if xmlData is not None:
268 self._parseXmlData(xmlData)
269 if validate:
270 self.validate()
271 elif xmlPath is not None:
272 with open(xmlPath) as f:
273 xmlData = f.read()
274 self._parseXmlData(xmlData)
275 if validate:
276 self.validate()
277
279 """
280 Official string representation for class instance.
281 """
282 return "LocalConfig(%s)" % (self.split)
283
285 """
286 Informal string representation for class instance.
287 """
288 return self.__repr__()
289
291 """Equals operator, iplemented in terms of original Python 2 compare operator."""
292 return self.__cmp__(other) == 0
293
295 """Less-than operator, iplemented in terms of original Python 2 compare operator."""
296 return self.__cmp__(other) < 0
297
299 """Greater-than operator, iplemented in terms of original Python 2 compare operator."""
300 return self.__cmp__(other) > 0
301
303 """
304 Original Python 2 comparison operator.
305 Lists within this class are "unordered" for equality comparisons.
306 @param other: Other object to compare to.
307 @return: -1/0/1 depending on whether self is C{<}, C{=} or C{>} other.
308 """
309 if other is None:
310 return 1
311 if self.split != other.split:
312 if self.split < other.split:
313 return -1
314 else:
315 return 1
316 return 0
317
319 """
320 Property target used to set the split configuration value.
321 If not C{None}, the value must be a C{SplitConfig} object.
322 @raise ValueError: If the value is not a C{SplitConfig}
323 """
324 if value is None:
325 self._split = None
326 else:
327 if not isinstance(value, SplitConfig):
328 raise ValueError("Value must be a C{SplitConfig} object.")
329 self._split = value
330
332 """
333 Property target used to get the split configuration value.
334 """
335 return self._split
336
337 split = property(_getSplit, _setSplit, None, "Split configuration in terms of a C{SplitConfig} object.")
338
340 """
341 Validates configuration represented by the object.
342
343 Split configuration must be filled in. Within that, both the size limit
344 and split size must be filled in.
345
346 @raise ValueError: If one of the validations fails.
347 """
348 if self.split is None:
349 raise ValueError("Split section is required.")
350 if self.split.sizeLimit is None:
351 raise ValueError("Size limit must be set.")
352 if self.split.splitSize is None:
353 raise ValueError("Split size must be set.")
354
356 """
357 Adds a <split> configuration section as the next child of a parent.
358
359 Third parties should use this function to write configuration related to
360 this extension.
361
362 We add the following fields to the document::
363
364 sizeLimit //cb_config/split/size_limit
365 splitSize //cb_config/split/split_size
366
367 @param xmlDom: DOM tree as from C{impl.createDocument()}.
368 @param parentNode: Parent that the section should be appended to.
369 """
370 if self.split is not None:
371 sectionNode = addContainerNode(xmlDom, parentNode, "split")
372 addByteQuantityNode(xmlDom, sectionNode, "size_limit", self.split.sizeLimit)
373 addByteQuantityNode(xmlDom, sectionNode, "split_size", self.split.splitSize)
374
376 """
377 Internal method to parse an XML string into the object.
378
379 This method parses the XML document into a DOM tree (C{xmlDom}) and then
380 calls a static method to parse the split configuration section.
381
382 @param xmlData: XML data to be parsed
383 @type xmlData: String data
384
385 @raise ValueError: If the XML cannot be successfully parsed.
386 """
387 (xmlDom, parentNode) = createInputDom(xmlData)
388 self._split = LocalConfig._parseSplit(parentNode)
389
390 @staticmethod
392 """
393 Parses an split configuration section.
394
395 We read the following individual fields::
396
397 sizeLimit //cb_config/split/size_limit
398 splitSize //cb_config/split/split_size
399
400 @param parent: Parent node to search beneath.
401
402 @return: C{EncryptConfig} object or C{None} if the section does not exist.
403 @raise ValueError: If some filled-in value is invalid.
404 """
405 split = None
406 section = readFirstChild(parent, "split")
407 if section is not None:
408 split = SplitConfig()
409 split.sizeLimit = readByteQuantity(section, "size_limit")
410 split.splitSize = readByteQuantity(section, "split_size")
411 return split
412
413
414
415
416
417
418
419
420
421
422
423 -def executeAction(configPath, options, config):
449
450
451
452
453
454
455 -def _splitDailyDir(dailyDir, sizeLimit, splitSize, backupUser, backupGroup):
456 """
457 Splits large files in a daily staging directory.
458
459 Files that match INDICATOR_PATTERNS (i.e. C{"cback.store"},
460 C{"cback.stage"}, etc.) are assumed to be indicator files and are ignored.
461 All other files are split.
462
463 @param dailyDir: Daily directory to encrypt
464 @param sizeLimit: Size limit, in bytes
465 @param splitSize: Split size, in bytes
466 @param backupUser: User that target files should be owned by
467 @param backupGroup: Group that target files should be owned by
468
469 @raise ValueError: If the encrypt mode is not supported.
470 @raise ValueError: If the daily staging directory does not exist.
471 """
472 logger.debug("Begin splitting contents of [%s].", dailyDir)
473 fileList = getBackupFiles(dailyDir)
474 for path in fileList:
475 size = float(os.stat(path).st_size)
476 if size > sizeLimit:
477 _splitFile(path, splitSize, backupUser, backupGroup, removeSource=True)
478 logger.debug("Completed splitting contents of [%s].", dailyDir)
479
480
481
482
483
484
485 -def _splitFile(sourcePath, splitSize, backupUser, backupGroup, removeSource=False):
486 """
487 Splits the source file into chunks of the indicated size.
488
489 The split files will be owned by the indicated backup user and group. If
490 C{removeSource} is C{True}, then the source file will be removed after it is
491 successfully split.
492
493 @param sourcePath: Absolute path of the source file to split
494 @param splitSize: Encryption mode (only "gpg" is allowed)
495 @param backupUser: User that target files should be owned by
496 @param backupGroup: Group that target files should be owned by
497 @param removeSource: Indicates whether to remove the source file
498
499 @raise IOError: If there is a problem accessing, splitting or removing the source file.
500 """
501 cwd = os.getcwd()
502 try:
503 if not os.path.exists(sourcePath):
504 raise ValueError("Source path [%s] does not exist." % sourcePath)
505 dirname = os.path.dirname(sourcePath)
506 filename = os.path.basename(sourcePath)
507 prefix = "%s_" % filename
508 bytes = int(splitSize.bytes)
509 os.chdir(dirname)
510 command = resolveCommand(SPLIT_COMMAND)
511 args = [ "--verbose", "--numeric-suffixes", "--suffix-length=5", "--bytes=%d" % bytes, filename, prefix, ]
512 (result, output) = executeCommand(command, args, returnOutput=True, ignoreStderr=False)
513 if result != 0:
514 raise IOError("Error [%d] calling split for [%s]." % (result, sourcePath))
515 pattern = re.compile(r"(creating file [`'])(%s)(.*)(')" % prefix)
516 match = pattern.search(output[-1:][0])
517 if match is None:
518 raise IOError("Unable to parse output from split command.")
519 value = int(match.group(3).strip())
520 for index in range(0, value):
521 path = "%s%05d" % (prefix, index)
522 if not os.path.exists(path):
523 raise IOError("After call to split, expected file [%s] does not exist." % path)
524 changeOwnership(path, backupUser, backupGroup)
525 if removeSource:
526 if os.path.exists(sourcePath):
527 try:
528 os.remove(sourcePath)
529 logger.debug("Completed removing old file [%s].", sourcePath)
530 except:
531 raise IOError("Failed to remove file [%s] after splitting it." % (sourcePath))
532 finally:
533 os.chdir(cwd)
534