1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41 """
42 Provides an extension to back up PostgreSQL databases.
43
44 This is a Cedar Backup extension used to back up PostgreSQL databases via the
45 Cedar Backup command line. It requires a new configurations section
46 <postgresql> and is intended to be run either immediately before or immediately
47 after the standard collect action. Aside from its own configuration, it
48 requires the options and collect configuration sections in the standard Cedar
49 Backup configuration file.
50
51 The backup is done via the C{pg_dump} or C{pg_dumpall} commands included with
52 the PostgreSQL product. Output can be compressed using C{gzip} or C{bzip2}.
53 Administrators can configure the extension either to back up all databases or
54 to back up only specific databases. The extension assumes that the current
55 user has passwordless access to the database since there is no easy way to pass
56 a password to the C{pg_dump} client. This can be accomplished using appropriate
57 voodoo in the C{pg_hda.conf} file.
58
59 Note that this code always produces a full backup. There is currently no
60 facility for making incremental backups.
61
62 You should always make C{/etc/cback3.conf} unreadble to non-root users once you
63 place postgresql configuration into it, since postgresql configuration will
64 contain information about available PostgreSQL databases and usernames.
65
66 Use of this extension I{may} expose usernames in the process listing (via
67 C{ps}) when the backup is running if the username is specified in the
68 configuration.
69
70 @author: Kenneth J. Pronovici <pronovic@ieee.org>
71 @author: Antoine Beaupre <anarcat@koumbit.org>
72 """
73
74
75
76
77
78
79 import os
80 import logging
81 from gzip import GzipFile
82 from bz2 import BZ2File
83 from functools import total_ordering
84
85
86 from CedarBackup3.xmlutil import createInputDom, addContainerNode, addStringNode, addBooleanNode
87 from CedarBackup3.xmlutil import readFirstChild, readString, readStringList, readBoolean
88 from CedarBackup3.config import VALID_COMPRESS_MODES
89 from CedarBackup3.util import resolveCommand, executeCommand
90 from CedarBackup3.util import ObjectTypeList, changeOwnership
91
92
93
94
95
96
97 logger = logging.getLogger("CedarBackup3.log.extend.postgresql")
98 POSTGRESQLDUMP_COMMAND = [ "pg_dump", ]
99 POSTGRESQLDUMPALL_COMMAND = [ "pg_dumpall", ]
100
101
102
103
104
105
106 @total_ordering
107 -class PostgresqlConfig(object):
108
109 """
110 Class representing PostgreSQL configuration.
111
112 The PostgreSQL configuration information is used for backing up PostgreSQL databases.
113
114 The following restrictions exist on data in this class:
115
116 - The compress mode must be one of the values in L{VALID_COMPRESS_MODES}.
117 - The 'all' flag must be 'Y' if no databases are defined.
118 - The 'all' flag must be 'N' if any databases are defined.
119 - Any values in the databases list must be strings.
120
121 @sort: __init__, __repr__, __str__, __cmp__, __eq__, __lt__, __gt__, user,
122 all, databases
123 """
124
125 - def __init__(self, user=None, compressMode=None, all=None, databases=None):
126 """
127 Constructor for the C{PostgresqlConfig} class.
128
129 @param user: User to execute backup as.
130 @param compressMode: Compress mode for backed-up files.
131 @param all: Indicates whether to back up all databases.
132 @param databases: List of databases to back up.
133 """
134 self._user = None
135 self._compressMode = None
136 self._all = None
137 self._databases = None
138 self.user = user
139 self.compressMode = compressMode
140 self.all = all
141 self.databases = databases
142
143 - def __repr__(self):
144 """
145 Official string representation for class instance.
146 """
147 return "PostgresqlConfig(%s, %s, %s)" % (self.user, self.all, self.databases)
148
150 """
151 Informal string representation for class instance.
152 """
153 return self.__repr__()
154
155 - def __eq__(self, other):
156 """Equals operator, iplemented in terms of original Python 2 compare operator."""
157 return self.__cmp__(other) == 0
158
159 - def __lt__(self, other):
160 """Less-than operator, iplemented in terms of original Python 2 compare operator."""
161 return self.__cmp__(other) < 0
162
163 - def __gt__(self, other):
164 """Greater-than operator, iplemented in terms of original Python 2 compare operator."""
165 return self.__cmp__(other) > 0
166
167 - def __cmp__(self, other):
168 """
169 Original Python 2 comparison operator.
170 @param other: Other object to compare to.
171 @return: -1/0/1 depending on whether self is C{<}, C{=} or C{>} other.
172 """
173 if other is None:
174 return 1
175 if self.user != other.user:
176 if str(self.user or "") < str(other.user or ""):
177 return -1
178 else:
179 return 1
180 if self.compressMode != other.compressMode:
181 if str(self.compressMode or "") < str(other.compressMode or ""):
182 return -1
183 else:
184 return 1
185 if self.all != other.all:
186 if self.all < other.all:
187 return -1
188 else:
189 return 1
190 if self.databases != other.databases:
191 if self.databases < other.databases:
192 return -1
193 else:
194 return 1
195 return 0
196
197 - def _setUser(self, value):
198 """
199 Property target used to set the user value.
200 """
201 if value is not None:
202 if len(value) < 1:
203 raise ValueError("User must be non-empty string.")
204 self._user = value
205
206 - def _getUser(self):
207 """
208 Property target used to get the user value.
209 """
210 return self._user
211
212 - def _setCompressMode(self, value):
213 """
214 Property target used to set the compress mode.
215 If not C{None}, the mode must be one of the values in L{VALID_COMPRESS_MODES}.
216 @raise ValueError: If the value is not valid.
217 """
218 if value is not None:
219 if value not in VALID_COMPRESS_MODES:
220 raise ValueError("Compress mode must be one of %s." % VALID_COMPRESS_MODES)
221 self._compressMode = value
222
224 """
225 Property target used to get the compress mode.
226 """
227 return self._compressMode
228
229 - def _setAll(self, value):
230 """
231 Property target used to set the 'all' flag.
232 No validations, but we normalize the value to C{True} or C{False}.
233 """
234 if value:
235 self._all = True
236 else:
237 self._all = False
238
240 """
241 Property target used to get the 'all' flag.
242 """
243 return self._all
244
245 - def _setDatabases(self, value):
246 """
247 Property target used to set the databases list.
248 Either the value must be C{None} or each element must be a string.
249 @raise ValueError: If the value is not a string.
250 """
251 if value is None:
252 self._databases = None
253 else:
254 for database in value:
255 if len(database) < 1:
256 raise ValueError("Each database must be a non-empty string.")
257 try:
258 saved = self._databases
259 self._databases = ObjectTypeList(str, "string")
260 self._databases.extend(value)
261 except Exception as e:
262 self._databases = saved
263 raise e
264
265 - def _getDatabases(self):
266 """
267 Property target used to get the databases list.
268 """
269 return self._databases
270
271 user = property(_getUser, _setUser, None, "User to execute backup as.")
272 compressMode = property(_getCompressMode, _setCompressMode, None, "Compress mode to be used for backed-up files.")
273 all = property(_getAll, _setAll, None, "Indicates whether to back up all databases.")
274 databases = property(_getDatabases, _setDatabases, None, "List of databases to back up.")
275
276
277
278
279
280
281 @total_ordering
282 -class LocalConfig(object):
283
284 """
285 Class representing this extension's configuration document.
286
287 This is not a general-purpose configuration object like the main Cedar
288 Backup configuration object. Instead, it just knows how to parse and emit
289 PostgreSQL-specific configuration values. Third parties who need to read and
290 write configuration related to this extension should access it through the
291 constructor, C{validate} and C{addConfig} methods.
292
293 @note: Lists within this class are "unordered" for equality comparisons.
294
295 @sort: __init__, __repr__, __str__, __cmp__, __eq__, __lt__, __gt__,
296 postgresql, validate, addConfig
297 """
298
299 - def __init__(self, xmlData=None, xmlPath=None, validate=True):
300 """
301 Initializes a configuration object.
302
303 If you initialize the object without passing either C{xmlData} or
304 C{xmlPath} then configuration will be empty and will be invalid until it
305 is filled in properly.
306
307 No reference to the original XML data or original path is saved off by
308 this class. Once the data has been parsed (successfully or not) this
309 original information is discarded.
310
311 Unless the C{validate} argument is C{False}, the L{LocalConfig.validate}
312 method will be called (with its default arguments) against configuration
313 after successfully parsing any passed-in XML. Keep in mind that even if
314 C{validate} is C{False}, it might not be possible to parse the passed-in
315 XML document if lower-level validations fail.
316
317 @note: It is strongly suggested that the C{validate} option always be set
318 to C{True} (the default) unless there is a specific need to read in
319 invalid configuration from disk.
320
321 @param xmlData: XML data representing configuration.
322 @type xmlData: String data.
323
324 @param xmlPath: Path to an XML file on disk.
325 @type xmlPath: Absolute path to a file on disk.
326
327 @param validate: Validate the document after parsing it.
328 @type validate: Boolean true/false.
329
330 @raise ValueError: If both C{xmlData} and C{xmlPath} are passed-in.
331 @raise ValueError: If the XML data in C{xmlData} or C{xmlPath} cannot be parsed.
332 @raise ValueError: If the parsed configuration document is not valid.
333 """
334 self._postgresql = None
335 self.postgresql = None
336 if xmlData is not None and xmlPath is not None:
337 raise ValueError("Use either xmlData or xmlPath, but not both.")
338 if xmlData is not None:
339 self._parseXmlData(xmlData)
340 if validate:
341 self.validate()
342 elif xmlPath is not None:
343 with open(xmlPath) as f:
344 xmlData = f.read()
345 self._parseXmlData(xmlData)
346 if validate:
347 self.validate()
348
350 """
351 Official string representation for class instance.
352 """
353 return "LocalConfig(%s)" % (self.postgresql)
354
356 """
357 Informal string representation for class instance.
358 """
359 return self.__repr__()
360
362 """Equals operator, iplemented in terms of original Python 2 compare operator."""
363 return self.__cmp__(other) == 0
364
366 """Less-than operator, iplemented in terms of original Python 2 compare operator."""
367 return self.__cmp__(other) < 0
368
370 """Greater-than operator, iplemented in terms of original Python 2 compare operator."""
371 return self.__cmp__(other) > 0
372
374 """
375 Original Python 2 comparison operator.
376 Lists within this class are "unordered" for equality comparisons.
377 @param other: Other object to compare to.
378 @return: -1/0/1 depending on whether self is C{<}, C{=} or C{>} other.
379 """
380 if other is None:
381 return 1
382 if self.postgresql != other.postgresql:
383 if self.postgresql < other.postgresql:
384 return -1
385 else:
386 return 1
387 return 0
388
389 - def _setPostgresql(self, value):
390 """
391 Property target used to set the postgresql configuration value.
392 If not C{None}, the value must be a C{PostgresqlConfig} object.
393 @raise ValueError: If the value is not a C{PostgresqlConfig}
394 """
395 if value is None:
396 self._postgresql = None
397 else:
398 if not isinstance(value, PostgresqlConfig):
399 raise ValueError("Value must be a C{PostgresqlConfig} object.")
400 self._postgresql = value
401
402 - def _getPostgresql(self):
403 """
404 Property target used to get the postgresql configuration value.
405 """
406 return self._postgresql
407
408 postgresql = property(_getPostgresql, _setPostgresql, None, "Postgresql configuration in terms of a C{PostgresqlConfig} object.")
409
411 """
412 Validates configuration represented by the object.
413
414 The compress mode must be filled in. Then, if the 'all' flag
415 I{is} set, no databases are allowed, and if the 'all' flag is
416 I{not} set, at least one database is required.
417
418 @raise ValueError: If one of the validations fails.
419 """
420 if self.postgresql is None:
421 raise ValueError("PostgreSQL section is required.")
422 if self.postgresql.compressMode is None:
423 raise ValueError("Compress mode value is required.")
424 if self.postgresql.all:
425 if self.postgresql.databases is not None and self.postgresql.databases != []:
426 raise ValueError("Databases cannot be specified if 'all' flag is set.")
427 else:
428 if self.postgresql.databases is None or len(self.postgresql.databases) < 1:
429 raise ValueError("At least one PostgreSQL database must be indicated if 'all' flag is not set.")
430
432 """
433 Adds a <postgresql> configuration section as the next child of a parent.
434
435 Third parties should use this function to write configuration related to
436 this extension.
437
438 We add the following fields to the document::
439
440 user //cb_config/postgresql/user
441 compressMode //cb_config/postgresql/compress_mode
442 all //cb_config/postgresql/all
443
444 We also add groups of the following items, one list element per
445 item::
446
447 database //cb_config/postgresql/database
448
449 @param xmlDom: DOM tree as from C{impl.createDocument()}.
450 @param parentNode: Parent that the section should be appended to.
451 """
452 if self.postgresql is not None:
453 sectionNode = addContainerNode(xmlDom, parentNode, "postgresql")
454 addStringNode(xmlDom, sectionNode, "user", self.postgresql.user)
455 addStringNode(xmlDom, sectionNode, "compress_mode", self.postgresql.compressMode)
456 addBooleanNode(xmlDom, sectionNode, "all", self.postgresql.all)
457 if self.postgresql.databases is not None:
458 for database in self.postgresql.databases:
459 addStringNode(xmlDom, sectionNode, "database", database)
460
462 """
463 Internal method to parse an XML string into the object.
464
465 This method parses the XML document into a DOM tree (C{xmlDom}) and then
466 calls a static method to parse the postgresql configuration section.
467
468 @param xmlData: XML data to be parsed
469 @type xmlData: String data
470
471 @raise ValueError: If the XML cannot be successfully parsed.
472 """
473 (xmlDom, parentNode) = createInputDom(xmlData)
474 self._postgresql = LocalConfig._parsePostgresql(parentNode)
475
476 @staticmethod
477 - def _parsePostgresql(parent):
478 """
479 Parses a postgresql configuration section.
480
481 We read the following fields::
482
483 user //cb_config/postgresql/user
484 compressMode //cb_config/postgresql/compress_mode
485 all //cb_config/postgresql/all
486
487 We also read groups of the following item, one list element per
488 item::
489
490 databases //cb_config/postgresql/database
491
492 @param parent: Parent node to search beneath.
493
494 @return: C{PostgresqlConfig} object or C{None} if the section does not exist.
495 @raise ValueError: If some filled-in value is invalid.
496 """
497 postgresql = None
498 section = readFirstChild(parent, "postgresql")
499 if section is not None:
500 postgresql = PostgresqlConfig()
501 postgresql.user = readString(section, "user")
502 postgresql.compressMode = readString(section, "compress_mode")
503 postgresql.all = readBoolean(section, "all")
504 postgresql.databases = readStringList(section, "database")
505 return postgresql
506
507
508
509
510
511
512
513
514
515
516 -def executeAction(configPath, options, config):
517 """
518 Executes the PostgreSQL backup action.
519
520 @param configPath: Path to configuration file on disk.
521 @type configPath: String representing a path on disk.
522
523 @param options: Program command-line options.
524 @type options: Options object.
525
526 @param config: Program configuration.
527 @type config: Config object.
528
529 @raise ValueError: Under many generic error conditions
530 @raise IOError: If a backup could not be written for some reason.
531 """
532 logger.debug("Executing PostgreSQL extended action.")
533 if config.options is None or config.collect is None:
534 raise ValueError("Cedar Backup configuration is not properly filled in.")
535 local = LocalConfig(xmlPath=configPath)
536 if local.postgresql.all:
537 logger.info("Backing up all databases.")
538 _backupDatabase(config.collect.targetDir, local.postgresql.compressMode, local.postgresql.user,
539 config.options.backupUser, config.options.backupGroup, None)
540 if local.postgresql.databases is not None and local.postgresql.databases != []:
541 logger.debug("Backing up %d individual databases.", len(local.postgresql.databases))
542 for database in local.postgresql.databases:
543 logger.info("Backing up database [%s].", database)
544 _backupDatabase(config.collect.targetDir, local.postgresql.compressMode, local.postgresql.user,
545 config.options.backupUser, config.options.backupGroup, database)
546 logger.info("Executed the PostgreSQL extended action successfully.")
547
548 -def _backupDatabase(targetDir, compressMode, user, backupUser, backupGroup, database=None):
549 """
550 Backs up an individual PostgreSQL database, or all databases.
551
552 This internal method wraps the public method and adds some functionality,
553 like figuring out a filename, etc.
554
555 @param targetDir: Directory into which backups should be written.
556 @param compressMode: Compress mode to be used for backed-up files.
557 @param user: User to use for connecting to the database.
558 @param backupUser: User to own resulting file.
559 @param backupGroup: Group to own resulting file.
560 @param database: Name of database, or C{None} for all databases.
561
562 @return: Name of the generated backup file.
563
564 @raise ValueError: If some value is missing or invalid.
565 @raise IOError: If there is a problem executing the PostgreSQL dump.
566 """
567 (outputFile, filename) = _getOutputFile(targetDir, database, compressMode)
568 with outputFile:
569 backupDatabase(user, outputFile, database)
570 if not os.path.exists(filename):
571 raise IOError("Dump file [%s] does not seem to exist after backup completed." % filename)
572 changeOwnership(filename, backupUser, backupGroup)
573
576 """
577 Opens the output file used for saving the PostgreSQL dump.
578
579 The filename is either C{"postgresqldump.txt"} or
580 C{"postgresqldump-<database>.txt"}. The C{".gz"} or C{".bz2"} extension is
581 added if C{compress} is C{True}.
582
583 @param targetDir: Target directory to write file in.
584 @param database: Name of the database (if any)
585 @param compressMode: Compress mode to be used for backed-up files.
586
587 @return: Tuple of (Output file object, filename), file opened in binary mode for use with executeCommand()
588 """
589 if database is None:
590 filename = os.path.join(targetDir, "postgresqldump.txt")
591 else:
592 filename = os.path.join(targetDir, "postgresqldump-%s.txt" % database)
593 if compressMode == "gzip":
594 filename = "%s.gz" % filename
595 outputFile = GzipFile(filename, "wb")
596 elif compressMode == "bzip2":
597 filename = "%s.bz2" % filename
598 outputFile = BZ2File(filename, "wb")
599 else:
600 outputFile = open(filename, "wb")
601 logger.debug("PostgreSQL dump file will be [%s].", filename)
602 return (outputFile, filename)
603
604
605
606
607
608
609 -def backupDatabase(user, backupFile, database=None):
610 """
611 Backs up an individual PostgreSQL database, or all databases.
612
613 This function backs up either a named local PostgreSQL database or all local
614 PostgreSQL databases, using the passed in user for connectivity.
615 This is I{always} a full backup. There is no facility for incremental
616 backups.
617
618 The backup data will be written into the passed-in back file. Normally,
619 this would be an object as returned from C{open()}, but it is possible to
620 use something like a C{GzipFile} to write compressed output. The caller is
621 responsible for closing the passed-in backup file.
622
623 @note: Typically, you would use the C{root} user to back up all databases.
624
625 @param user: User to use for connecting to the database.
626 @type user: String representing PostgreSQL username.
627
628 @param backupFile: File use for writing backup.
629 @type backupFile: Python file object as from C{open()} or C{file()}.
630
631 @param database: Name of the database to be backed up.
632 @type database: String representing database name, or C{None} for all databases.
633
634 @raise ValueError: If some value is missing or invalid.
635 @raise IOError: If there is a problem executing the PostgreSQL dump.
636 """
637 args = []
638 if user is not None:
639 args.append('-U')
640 args.append(user)
641
642 if database is None:
643 command = resolveCommand(POSTGRESQLDUMPALL_COMMAND)
644 else:
645 command = resolveCommand(POSTGRESQLDUMP_COMMAND)
646 args.append(database)
647
648 result = executeCommand(command, args, returnOutput=False, ignoreStderr=True, doNotLog=True, outputFile=backupFile)[0]
649 if result != 0:
650 if database is None:
651 raise IOError("Error [%d] executing PostgreSQL database dump for all databases." % result)
652 else:
653 raise IOError("Error [%d] executing PostgreSQL database dump for database [%s]." % (result, database))
654