Package csb :: Module build
[frames] | no frames]

Source Code for Module csb.build

  1  """ 
  2  CSB build related tools and programs. 
  3   
  4  When executed as a program, this module will run the CSB Build Console and 
  5  build the source tree it belongs to. The source tree is added at the 
  6  B{beginning} of sys.path to make sure that all subsequent imports from the 
  7  Test and Doc consoles will import the right thing (think of multiple CSB 
  8  packages installed on the same server). 
  9   
 10  Here is how to build, test and package the whole project:: 
 11   
 12      $ hg clone https://hg.codeplex.com/csb CSB 
 13      $ CSB/csb/build.py -o <output directory> 
 14   
 15  The Console can also be imported and instantiated as a regular Python class. 
 16  In this case the Console again builds the source tree it is part of, but 
 17  sys.path will remain intact. Therefore, the Console will assume that all 
 18  modules currently in memory, as well as those that can be subsequently imported 
 19  by the Console itself, belong to the same CSB package. 
 20   
 21  @note: The CSB build services no longer support the option to build external 
 22         source trees. 
 23  @see: [CSB 0000038] 
 24  """ 
 25  from __future__ import print_function 
 26   
 27  import os 
 28  import sys 
 29  import getopt 
 30  import traceback 
 31  import compileall 
 32           
 33  if os.path.basename(__file__) == '__init__.py': 
 34      PARENT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) 
 35  else: 
 36      PARENT = os.path.abspath(os.path.dirname(__file__)) 
 37   
 38  ROOT = 'csb' 
 39  SOURCETREE = os.path.abspath(os.path.join(PARENT, "..")) 
 40   
 41  if __name__ == '__main__': 
 42   
 43      # make sure "import io" imports the built in module, not csb.io 
 44      # io is required by tarfile     
 45      for path in sys.path: 
 46          if path.startswith(SOURCETREE): 
 47              sys.path.remove(path) 
 48               
 49      import io 
 50      assert hasattr(io, 'BufferedIOBase')    
 51               
 52      sys.path = [SOURCETREE] + sys.path 
 53   
 54   
 55  """ 
 56  It is now safe to import any modules   
 57  """ 
 58  import imp 
 59  import shutil 
 60  import tarfile 
 61   
 62  import csb 
 63   
 64  from abc import ABCMeta, abstractmethod 
 65  from csb.io import Shell 
66 67 68 -class BuildTypes(object):
69 """ 70 Enumeration of build types. 71 """ 72 73 SOURCE = 'source' 74 BINARY = 'binary' 75 76 _du = { SOURCE: 'sdist', BINARY: 'bdist' } 77 78 @staticmethod
79 - def get(key):
80 try: 81 return BuildTypes._du[key] 82 except KeyError: 83 raise ValueError('Unhandled build type: {0}'.format(key))
84
85 86 -class Console(object):
87 """ 88 CSB Build Bot. Run with -h for usage. 89 90 @param output: build output directory 91 @type output: str 92 @param verbosity: verbosity level 93 @type verbosity: int 94 95 @note: The build console automatically detects and builds the csb package 96 it belongs to. You cannot build a different source tree with it. 97 See the module documentation for more info. 98 """ 99 100 PROGRAM = __file__ 101 102 USAGE = r""" 103 CSB Build Console: build, test and package the entire csb project. 104 105 Usage: 106 python {program} -o output [-v verbosity] [-t type] [-h] 107 108 Options: 109 -o output Build output directory 110 -v verbosity Verbosity level, default is 1 111 -t type Build type: 112 source - build source code distribution (default) 113 binary - build executable 114 -h, --help Display this help 115 """ 116
117 - def __init__(self, output='.', verbosity=1, buildtype=BuildTypes.SOURCE):
118 119 self._input = None 120 self._output = None 121 self._temp = None 122 self._docs = None 123 self._apidocs = None 124 self._root = None 125 self._verbosity = None 126 self._type = buildtype 127 self._dist = BuildTypes.get(buildtype) 128 129 if os.path.join(SOURCETREE, ROOT) != PARENT: 130 raise IOError('{0} must be a sub-package or sub-module of {1}'.format(__file__, ROOT)) 131 self._input = SOURCETREE 132 133 self._success = True 134 135 self.output = output 136 self.verbosity = verbosity
137 138 @property
139 - def input(self):
140 return self._input
141 142 @property
143 - def output(self):
144 return self._output
145 @output.setter
146 - def output(self, value):
147 #value = os.path.dirname(value) 148 self._output = os.path.abspath(value) 149 self._temp = os.path.join(self._output, 'build') 150 self._docs = os.path.join(self._temp, 'docs') 151 self._apidocs = os.path.join(self._docs, 'api') 152 self._root = os.path.join(self._temp, ROOT)
153 154 @property
155 - def verbosity(self):
156 return self._verbosity
157 @verbosity.setter
158 - def verbosity(self, value):
159 self._verbosity = int(value)
160
161 - def build(self):
162 """ 163 Run the console. 164 """ 165 self.log('\n# Building package {0} from {1}\n'.format(ROOT, SOURCETREE)) 166 167 self._init() 168 v = self._revision() 169 self._doc(v) 170 self._test() 171 172 self._compile() 173 vn = self._package() 174 175 if self._success: 176 self.log('\n# Done ({0}).\n'.format(vn.full)) 177 else: 178 self.log('\n# Build failed.\n')
179 180
181 - def log(self, message, level=1, ending='\n'):
182 183 if self._verbosity >= level: 184 sys.stdout.write(message) 185 sys.stdout.write(ending) 186 sys.stdout.flush()
187
188 - def _init(self):
189 """ 190 Collect all required stuff in the output folder. 191 """ 192 self.log('# Preparing the file system...') 193 194 if not os.path.exists(self._output): 195 self.log('Creating output directory {0}'.format(self._output), level=2) 196 os.mkdir(self._output) 197 198 if os.path.exists(self._temp): 199 self.log('Deleting existing temp directory {0}'.format(self._temp), level=2) 200 shutil.rmtree(self._temp) 201 202 self.log('Copying the source tree to temp directory {0}'.format(self._temp), level=2) 203 shutil.copytree(self._input, self._temp) 204 205 if os.path.exists(self._apidocs): 206 self.log('Deleting existing API docs directory {0}'.format(self._apidocs), level=2) 207 shutil.rmtree(self._apidocs) 208 if not os.path.isdir(self._docs): 209 self.log('Creating docs directory {0}'.format(self._docs), level=2) 210 os.mkdir(self._docs) 211 self.log('Creating API docs directory {0}'.format(self._apidocs), level=2) 212 os.mkdir(self._apidocs)
213
214 - def _revision(self):
215 """ 216 Write the actual revision number to L{ROOT}.__version__ 217 """ 218 self.log('\n# Setting the most recent Revision Number...') 219 root = os.path.join(self._root, '__init__.py') 220 221 self.log('Retrieving revision number from {0}'.format(root), level=2) 222 rh = MercurialHandler(root) 223 revision = rh.read().revision 224 225 self.log('Writing back revision number {0}'.format(revision), level=2) 226 version = rh.write(revision, root) 227 228 self.log(' This is {0}.__version__ {1}'.format(ROOT, version), level=1) 229 csb.__version__ = version 230 231 return version
232
233 - def _test(self):
234 """ 235 Run tests. Also make sure the current environment loads all modules from 236 the input folder. 237 """ 238 import csb.test 239 assert csb.test.__file__.startswith(self._input), 'csb.test not loaded from the input!' #@UndefinedVariable 240 241 from csb.test import unittest 242 243 newdata = os.path.join(self._temp, ROOT, 'test', 'data') 244 csb.test.Config.setDefaultDataRoot(newdata) 245 246 self.log('\n# Updating all test pickles in {0} if necessary...'.format(newdata), level=2) 247 csb.test.Config().ensureDataConsistency() 248 249 self.log('\n# Running the Test Console...') 250 251 builder = csb.test.AnyTestBuilder() 252 suite = builder.loadTests(ROOT + '.test.cases.*') 253 254 runner = unittest.TextTestRunner(stream=sys.stderr, verbosity=self.verbosity) 255 result = runner.run(suite) 256 if result.wasSuccessful(): 257 self.log('\n Passed all unit tests') 258 else: 259 self.log('\n DID NOT PASS: The build might be broken')
260
261 - def _doc(self, version):
262 """ 263 Build documentation in the output folder. 264 """ 265 self.log('\n# Generating API documentation...') 266 try: 267 import epydoc.cli 268 except ImportError: 269 self.log('\n Skipped: epydoc is missing') 270 return 271 272 self.log('\n# Emulating ARGV for the Doc Builder...', level=2) 273 argv = sys.argv 274 sys.argv = ['epydoc', '--html', '-o', self._apidocs, 275 '--name', '{0} v{1}'.format(ROOT.upper(), version), 276 '--no-private', '--introspect-only', '--exclude', 'csb.test.cases', 277 '--css', os.path.join(self._temp, 'epydoc.css'), 278 '--fail-on-error', '--fail-on-warning', '--fail-on-docstring-warning', 279 self._root] 280 281 if self._verbosity > 0: 282 sys.argv.append('-v') 283 284 try: 285 epydoc.cli.cli() 286 sys.exit(0) 287 except SystemExit as ex: 288 if ex.code is 0: 289 self.log('\n Passed all doc tests') 290 else: 291 if ex.code == 2: 292 self.log('\n DID NOT PASS: The docs might be broken') 293 else: 294 self.log('\n FAIL: Epydoc returned "#{0.code}: {0}"'.format(ex)) 295 self._success = False 296 297 self.log('\n# Restoring the previous ARGV...', level=2) 298 sys.argv = argv
299
300 - def _compile(self):
301 """ 302 Byte-compile all modules and packages. 303 """ 304 self.log('\n# Byte-compiling all *.py files...') 305 306 quiet = self.verbosity <= 1 307 valid = compileall.compile_dir(self._root, quiet=quiet, force=True) 308 309 if not valid: 310 self.log('\n FAIL: Compilation error(s)\n') 311 self._success = False
312
313 - def _package(self):
314 """ 315 Make package. 316 """ 317 self.log('\n# Configuring CWD and ARGV for the Setup...', level=2) 318 cwd = os.curdir 319 os.chdir(self._temp) 320 321 if self._verbosity > 1: 322 verbosity = '-v' 323 else: 324 verbosity = '-q' 325 argv = sys.argv 326 sys.argv = ['setup.py', verbosity, self._dist, '-d', self._output] 327 328 self.log('\n# Building {0} distribution...'.format(self._type)) 329 try: 330 setup = imp.load_source('setupcsb', 'setup.py') 331 d = setup.build() 332 version = setup.VERSION 333 package = d.dist_files[0][2] 334 335 if self._type == BuildTypes.BINARY: 336 self._strip_source(package) 337 338 except SystemExit as ex: 339 if ex.code is not 0: 340 self.log('\n FAIL: Setup returned: \n\n{0}\n'.format(ex)) 341 self._success = False 342 package = 'FAIL' 343 344 self.log('\n# Restoring the previous CWD and ARGV...', level=2) 345 os.chdir(cwd) 346 sys.argv = argv 347 348 self.log(' Packaged ' + package) 349 return version
350
351 - def _strip_source(self, package, source='*.py'):
352 """ 353 Delete plain text source code files from the package. 354 """ 355 cwd = os.getcwd() 356 357 try: 358 tmp = os.path.join(self.output, 'tmp') 359 os.mkdir(tmp) 360 361 self.log('\n# Entering {1} in order to delete .py files from {0}...'.format(package, tmp), level=2) 362 os.chdir(tmp) 363 364 oldtar = tarfile.open(package, mode='r:gz') 365 oldtar.extractall(tmp) 366 oldtar.close() 367 368 newtar = tarfile.open(package, mode='w:gz') 369 370 try: 371 for i in os.walk('.'): 372 for fn in i[2]: 373 if fn.endswith('.py'): 374 module = os.path.join(i[0], fn); 375 if not os.path.isfile(module.replace('.py', '.pyc')): 376 raise ValueError('Missing bytecode for module {0}'.format(module)) 377 else: 378 os.remove(os.path.join(i[0], fn)) 379 380 for i in os.listdir('.'): 381 newtar.add(i) 382 finally: 383 newtar.close() 384 385 finally: 386 self.log('\n# Restoring the previous CWD...', level=2) 387 os.chdir(cwd) 388 if os.path.exists(tmp): 389 shutil.rmtree(tmp)
390 391 @staticmethod
392 - def exit(message=None, code=0, usage=True):
393 394 if message: 395 print(message) 396 if usage: 397 print(Console.USAGE.format(program=Console.PROGRAM)) 398 399 sys.exit(code)
400 401 @staticmethod
402 - def run(argv=None):
403 404 if argv is None: 405 argv = sys.argv[1:] 406 407 output = None 408 verb = 1 409 buildtype = BuildTypes.SOURCE 410 411 try: 412 options, dummy = getopt.getopt(argv, 'o:v:t:h', ['output=', 'verbosity=', 'type=', 'help']) 413 414 for option, value in options: 415 if option in('-h', '--help'): 416 Console.exit(message=None, code=0) 417 if option in('-o', '--output'): 418 if not os.path.isdir(value): 419 Console.exit(message='E: Output directory not found "{0}".'.format(value), code=3) 420 output = value 421 if option in('-v', '--verbosity'): 422 try: 423 verb = int(value) 424 except ValueError: 425 Console.exit(message='E: Verbosity must be an integer.', code=4) 426 if option in('-t', '--type'): 427 if value not in [BuildTypes.SOURCE, BuildTypes.BINARY]: 428 Console.exit(message='E: Invalid build type "{0}".'.format(value), code=5) 429 buildtype = value 430 except getopt.GetoptError as oe: 431 Console.exit(message='E: ' + str(oe), code=1) 432 433 if not output: 434 Console.exit(code=1, usage=True) 435 else: 436 try: 437 Console(output, verbosity=verb, buildtype=buildtype).build() 438 except Exception as ex: 439 msg = 'Unexpected Error: {0}\n\n{1}'.format(ex, traceback.format_exc()) 440 Console.exit(message=msg, code=99, usage=False)
441
442 443 -class RevisionError(RuntimeError):
444
445 - def __init__(self, msg, code, cmd):
446 447 super(RevisionError, self).__init__(msg) 448 self.code = code 449 self.cmd = cmd
450
451 -class RevisionHandler(object):
452 """ 453 Determines the current repository revision number of a working copy. 454 455 @param path: a local checkout path to be examined 456 @type path: str 457 @param sc: name of the source control program 458 @type sc: str 459 """ 460
461 - def __init__(self, path, sc):
462 463 self._path = None 464 self._sc = None 465 466 if os.path.exists(path): 467 self._path = path 468 else: 469 raise IOError('Path not found: {0}'.format(path)) 470 if Shell.run([sc, 'help']).code is 0: 471 self._sc = sc 472 else: 473 raise RevisionError('Source control binary probe failed', None, None)
474 475 @property
476 - def path(self):
477 return self._path
478 479 @property
480 - def sc(self):
481 return self._sc
482 483 @abstractmethod
484 - def read(self):
485 """ 486 Return the current revision information. 487 @rtype: L{RevisionInfo} 488 """ 489 pass
490
491 - def write(self, revision, sourcefile):
492 """ 493 Finalize the __version__ = major.minor.micro.{revision} tag. 494 Overwrite C{sourcefile} in place by substituting the {revision} macro. 495 496 @param revision: revision number to write to the source file. 497 @type revision: int 498 @param sourcefile: python source file with a __version__ tag, typically 499 "csb/__init__.py" 500 @type sourcefile: str 501 502 @return: sourcefile.__version__ 503 """ 504 content = open(sourcefile).readlines() 505 506 with open(sourcefile, 'w') as src: 507 for line in content: 508 if line.startswith('__version__'): 509 src.write(line.format(revision=revision)) 510 else: 511 src.write(line) 512 513 self._delcache(sourcefile) 514 return imp.load_source('____source', sourcefile).__version__
515
516 - def _run(self, cmd):
517 518 si = Shell.run(cmd) 519 if si.code > 0: 520 raise RevisionError('SC failed ({0.code}): {0.stderr}'.format(si), si.code, si.cmd) 521 522 return si.stdout.splitlines()
523
524 - def _delcache(self, sourcefile):
525 526 compiled = os.path.splitext(sourcefile)[0] + '.pyc' 527 if os.path.isfile(compiled): 528 os.remove(compiled) 529 530 pycache = os.path.join(os.path.dirname(compiled), '__pycache__') 531 if os.path.isdir(pycache): 532 shutil.rmtree(pycache)
533
534 -class SubversionHandler(RevisionHandler):
535
536 - def __init__(self, path, sc='svn'):
537 super(SubversionHandler, self).__init__(path, sc)
538
539 - def read(self):
540 541 cmd = '{0.sc} info {0.path} -R'.format(self) 542 maxrevision = None 543 544 for line in self._run(cmd): 545 if line.startswith('Revision:'): 546 rev = int(line[9:] .strip()) 547 if rev > maxrevision: 548 maxrevision = rev 549 550 if maxrevision is None: 551 raise RevisionError('No revision number found', code=0, cmd=cmd) 552 553 return RevisionInfo(self.path, maxrevision)
554
555 -class MercurialHandler(RevisionHandler):
556
557 - def __init__(self, path, sc='hg'):
558 559 if os.path.isfile(path): 560 path = os.path.dirname(path) 561 562 super(MercurialHandler, self).__init__(path, sc)
563
564 - def read(self):
565 566 wd = os.getcwd() 567 os.chdir(self.path) 568 569 try: 570 cmd = '{0.sc} log -r tip'.format(self) 571 572 revision = None 573 changeset = '' 574 575 for line in self._run(cmd): 576 if line.startswith('changeset:'): 577 items = line[10:].split(':') 578 revision = int(items[0]) 579 changeset = items[1].strip() 580 break 581 582 if revision is None: 583 raise RevisionError('No revision number found', code=0, cmd=cmd) 584 585 return RevisionInfo(self.path, revision, changeset) 586 587 finally: 588 os.chdir(wd)
589
590 -class RevisionInfo(object):
591
592 - def __init__(self, item, revision, id=None):
593 594 self.item = item 595 self.revision = revision 596 self.id = id
597
598 599 -def main():
600 Console.run()
601 602 603 if __name__ == '__main__': 604 605 main() 606