1import sys 2if not hasattr(sys, 'version_info'): 3 print '*** Python version 1 is not supported. Please get the latest version from www.python.org ***' 4 sys.exit(4) 5 6import cPickle 7 8try: 9 import subprocess 10 USE_SUBPROCESS = 1 11except ImportError: 12 USE_SUBPROCESS = 0 13 14# Some features related to detecting login failures cannot be easily 15# implemented with the 'subprocess' module. Disable it for now ... 16USE_SUBPROCESS = 0 17# In Python 2.6 and above, the 'popen2' module is deprecated 18if sys.version_info[:2] >= (2, 6) and not USE_SUBPROCESS: 19 import warnings 20 warnings.filterwarnings('ignore', category=DeprecationWarning, module=__name__) 21 22import nargs 23useThreads = nargs.Arg.findArgument('useThreads', sys.argv[1:]) 24if useThreads is None: 25 useThreads = 0 # worarround issue with parallel configure 26else: 27 useThreads = int(useThreads) 28 29useSelect = nargs.Arg.findArgument('useSelect', sys.argv[1:]) 30if useSelect is None: 31 useSelect = 1 32else: 33 useSelect = int(useSelect) 34 35import logger 36 37class Script(logger.Logger): 38 def __init__(self, clArgs = None, argDB = None, log = None): 39 self.checkPython() 40 logger.Logger.__init__(self, clArgs, argDB, log) 41 self.shell = '/bin/sh' 42 self.showHelp = 1 43 return 44 45 def hasHelpFlag(self): 46 '''Decide whether to display the help message and exit''' 47 import nargs 48 49 if not self.showHelp: 50 return 0 51 if nargs.Arg.findArgument('help', self.clArgs) is None and nargs.Arg.findArgument('h', self.clArgs) is None: 52 return 0 53 return 1 54 55 def hasListFlag(self): 56 '''Decide whether to display the list of download files and exit''' 57 import nargs 58 59 if not self.showHelp: 60 return 0 61 if nargs.Arg.findArgument('with-packages-dir', self.clArgs) is None: 62 return 0 63 return 1 64 65 def setupArguments(self, argDB): 66 '''This method now also creates the help and action logs''' 67 import help 68 69 argDB = logger.Logger.setupArguments(self, argDB) 70 71 self.help = help.Help(argDB) 72 self.help.title = 'Script Help' 73 74 self.actions = help.Info(argDB) 75 self.actions.title = 'Script Actions' 76 77 self.setupHelp(self.help) 78 return argDB 79 80 def setupHelp(self, help): 81 '''This method should be overidden to provide help for arguments''' 82 import nargs 83 84 help.addArgument('Script', '-help', nargs.ArgBool(None, 0, 'Print this help message', isTemporary = 1), ignoreDuplicates = 1) 85 help.addArgument('Script', '-with-packages-dir', nargs.ArgDir(None,None, 'Directory to store downloaded external package tarballs', isTemporary = 1), ignoreDuplicates = 1) 86 help.addArgument('Script', '-h', nargs.ArgBool(None, 0, 'Print this help message', isTemporary = 1), ignoreDuplicates = 1) 87 return help 88 89 def setup(self): 90 ''' This method checks to see whether help was requested''' 91 if hasattr(self, '_setup'): 92 return 93 logger.Logger.setup(self) 94 self._setup = 1 95 if self.hasHelpFlag(): 96 self.argDB.readonly = True 97 if self.argDB.target == ['default']: 98 sections = None 99 else: 100 sections = self.argDB.target 101 self.help.output(sections = sections) 102 sys.exit() 103 if self.hasListFlag(): 104 self.argDB.readonly = True 105 self.help.outputDownload() 106 return 107 108 def cleanup(self): 109 '''This method outputs the action log''' 110 self.actions.output(self.log) 111 return 112 113 def checkPython(self): 114 if not hasattr(sys, 'version_info') or float(sys.version_info[0]) != 2 or float(sys.version_info[1]) < 6: 115 raise RuntimeError('BuildSystem requires Python2 version 2.6 or higher. Get Python at http://www.python.org') 116 return 117 118 def getModule(root, name): 119 '''Retrieve a specific module from the directory root, bypassing the usual paths''' 120 import imp 121 122 (fp, pathname, description) = imp.find_module(name, [root]) 123 try: 124 return imp.load_module(name, fp, pathname, description) 125 finally: 126 if fp: fp.close() 127 return 128 getModule = staticmethod(getModule) 129 130 def importModule(moduleName): 131 '''Import the named module, and return the module object 132 - Works properly for fully qualified names''' 133 module = __import__(moduleName) 134 components = moduleName.split('.') 135 for comp in components[1:]: 136 module = getattr(module, comp) 137 return module 138 importModule = staticmethod(importModule) 139 140 if USE_SUBPROCESS: 141 142 def runShellCommand(command, log=None, cwd=None): 143 Popen = subprocess.Popen 144 PIPE = subprocess.PIPE 145 if log: log.write('Executing: %s\n' % (command,)) 146 pipe = Popen(command, cwd=cwd, stdin=None, stdout=PIPE, stderr=PIPE, 147 bufsize=-1, shell=True, universal_newlines=True) 148 (out, err) = pipe.communicate() 149 ret = pipe.returncode 150 return (out, err, ret) 151 152 else: 153 154 def openPipe(command): 155 '''We need to use the asynchronous version here since we want to avoid blocking reads''' 156 import popen2 157 158 pipe = None 159 if hasattr(popen2, 'Popen3'): 160 pipe = popen2.Popen3(command, 1) 161 input = pipe.tochild 162 output = pipe.fromchild 163 err = pipe.childerr 164 else: 165 import os 166 (input, output, err) = os.popen3(command) 167 return (input, output, err, pipe) 168 openPipe = staticmethod(openPipe) 169 170 def runShellCommand(command, log = None, cwd = None): 171 import select, os 172 173 ret = None 174 out = '' 175 err = '' 176 loginError = 0 177 if cwd is not None: 178 oldpath = os.getcwd() 179 os.chdir(cwd) 180 if log: log.write('Executing: %s\n' % (command,)) 181 (input, output, error, pipe) = Script.openPipe(command) 182 if cwd is not None: 183 os.chdir(oldpath) 184 input.close() 185 if useSelect: 186 outputClosed = 0 187 errorClosed = 0 188 lst = [output, error] 189 while 1: 190 try: 191 ready = select.select(lst, [], []) 192 except Exception, e: 193 if log: log.write('** Error calling select() : '+str(e)+'\n') 194 continue 195 if len(ready[0]): 196 if error in ready[0]: 197 msg = error.readline() 198 if msg: 199 err += msg 200 else: 201 errorClosed = 1 202 lst.remove(error) 203 if output in ready[0]: 204 msg = output.readline() 205 if msg: 206 out += msg 207 else: 208 outputClosed = 1 209 lst.remove(output) 210 if msg and msg.find('password:') >= 0: 211 loginError = 1 212 break 213 if outputClosed and errorClosed: 214 break 215 else: 216 out = output.read() 217 err = error.read() 218 output.close() 219 error.close() 220 if pipe: 221 # We would like the NOHANG argument here 222 ret = pipe.wait() 223 if loginError: 224 raise RuntimeError('Could not login to site') 225 return (out, err, ret) 226 227 runShellCommand = staticmethod(runShellCommand) 228 229 def defaultCheckCommand(command, status, output, error): 230 '''Raise an error if the exit status is nonzero''' 231 if status: raise RuntimeError('Could not execute "%s":\n%s' % (command,output+error)) 232 defaultCheckCommand = staticmethod(defaultCheckCommand) 233 234 def executeShellCommand(command, checkCommand = None, timeout = 600.0, log = None, lineLimit = 0, cwd=None): 235 '''Execute a shell command returning the output, and optionally provide a custom error checker 236 - This returns a tuple of the (output, error, statuscode)''' 237 if not checkCommand: 238 checkCommand = Script.defaultCheckCommand 239 if log is None: 240 log = logger.Logger.defaultLog 241 def logOutput(log, output): 242 import re 243 # get rid of multiple blank lines 244 output = re.sub('\n+','\n', output).strip() 245 if output: 246 if lineLimit: 247 output = '\n'.join(output.split('\n')[:lineLimit]) 248 if '\n' in output: # multi-line output 249 log.write('stdout:\n'+output+'\n') 250 else: 251 log.write('stdout: '+output+'\n') 252 return output 253 def runInShell(command, log, cwd): 254 if useThreads: 255 import threading 256 class InShell(threading.Thread): 257 def __init__(self): 258 threading.Thread.__init__(self) 259 self.name = 'Shell Command' 260 self.setDaemon(1) 261 def run(self): 262 (self.output, self.error, self.status) = ('', '', -1) # So these fields exist even if command fails with no output 263 (self.output, self.error, self.status) = Script.runShellCommand(command, log, cwd) 264 thread = InShell() 265 thread.start() 266 thread.join(timeout) 267 if thread.isAlive(): 268 error = 'Runaway process exceeded time limit of '+str(timeout)+'s\n' 269 log.write(error) 270 return ('', error, -1) 271 else: 272 return (thread.output, thread.error, thread.status) 273 else: 274 return Script.runShellCommand(command, log, cwd) 275 276 (output, error, status) = runInShell(command, log, cwd) 277 output = logOutput(log, output) 278 checkCommand(command, status, output, error) 279 return (output, error, status) 280 executeShellCommand = staticmethod(executeShellCommand) 281 282 def loadConfigure(self, argDB = None): 283 if argDB is None: 284 argDB = self.argDB 285 if not 'configureCache' in argDB: 286 self.logPrint('No cached configure in RDict at '+str(argDB.saveFilename)) 287 return None 288 try: 289 cache = argDB['configureCache'] 290 framework = cPickle.loads(cache) 291 framework.framework = framework 292 framework.argDB = argDB 293 self.logPrint('Loaded configure to cache: size '+str(len(cache))) 294 except cPickle.UnpicklingError, e: 295 framework = None 296 self.logPrint('Invalid cached configure: '+str(e)) 297 return framework 298 299import args 300 301class LanguageProcessor(args.ArgumentProcessor): 302 def __init__(self, clArgs = None, argDB = None, framework = None, versionControl = None): 303 self.languageModule = {} 304 self.preprocessorObject = {} 305 self.compilerObject = {} 306 self.linkerObject = {} 307 self.sharedLinkerObject = {} 308 self.dynamicLinkerObject = {} 309 self.framework = framework 310 self.versionControl = versionControl 311 args.ArgumentProcessor.__init__(self, clArgs, argDB) 312 self.outputFiles = {} 313 self.modulePath = 'config.compile' 314 return 315 316 def getCompilers(self): 317 if self.framework is None: 318 return 319 return self.framework.require('config.compilers', None) 320 compilers = property(getCompilers, doc = 'The config.compilers configure object') 321 def getLibraries(self): 322 if self.framework is None: 323 return 324 return self.framework.require('config.libraries', None) 325 libraries = property(getLibraries, doc = 'The config.libraries configure object') 326 327 def __getstate__(self, d = None): 328 '''We only want to pickle the language module names and output files. The other objects are set by configure.''' 329 if d is None: 330 d = args.ArgumentProcessor.__getstate__(self) 331 if 'languageModule' in d: 332 d['languageModule'] = dict([(lang,mod._loadName) for lang,mod in d['languageModule'].items()]) 333 for member in ['preprocessorObject', 'compilerObject', 'linkerObject', 'sharedLinkerObject', 'dynamicLinkerObject', 'framework']: 334 if member in d: 335 del d[member] 336 return d 337 338 def __setstate__(self, d): 339 '''We must create the language modules''' 340 args.ArgumentProcessor.__setstate__(self, d) 341 self.__dict__.update(d) 342 [self.getLanguageModule(language, moduleName) for language,moduleName in self.languageModule.items()] 343 self.preprocessorObject = {} 344 self.compilerObject = {} 345 self.linkerObject = {} 346 self.sharedLinkerObject = {} 347 self.dynamicLinkerObject = {} 348 return 349 350 def setArgDB(self, argDB): 351 args.ArgumentProcessor.setArgDB(self, argDB) 352 for obj in self.preprocessorObject.values(): 353 if not hasattr(obj, 'argDB') or not obj.argDB == argDB: 354 obj.argDB = argDB 355 for obj in self.compilerObject.values(): 356 if not hasattr(obj, 'argDB') or not obj.argDB == argDB: 357 obj.argDB = argDB 358 for obj in self.linkerObject.values(): 359 if not hasattr(obj, 'argDB') or not obj.argDB == argDB: 360 obj.argDB = argDB 361 for obj in self.sharedLinkerObject.values(): 362 if not hasattr(obj, 'argDB') or not obj.argDB == argDB: 363 obj.argDB = argDB 364 for obj in self.dynamicLinkerObject.values(): 365 if not hasattr(obj, 'argDB') or not obj.argDB == argDB: 366 obj.argDB = argDB 367 if not self.compilers is None: 368 self.compilers.argDB = argDB 369 for obj in self.preprocessorObject.values(): 370 if hasattr(obj, 'configCompilers'): 371 obj.configCompilers.argDB = argDB 372 for obj in self.compilerObject.values(): 373 if hasattr(obj, 'configCompilers'): 374 obj.configCompilers.argDB = argDB 375 for obj in self.linkerObject.values(): 376 if hasattr(obj, 'configCompilers'): 377 obj.configCompilers.argDB = argDB 378 for obj in self.sharedLinkerObject.values(): 379 if hasattr(obj, 'configCompilers'): 380 obj.configCompilers.argDB = argDB 381 for obj in self.dynamicLinkerObject.values(): 382 if hasattr(obj, 'configCompilers'): 383 obj.configCompilers.argDB = argDB 384 if not self.libraries is None: 385 self.libraries.argDB = argDB 386 for obj in self.linkerObject.values(): 387 if hasattr(obj, 'configLibraries'): 388 obj.configLibraries.argDB = argDB 389 for obj in self.sharedLinkerObject.values(): 390 if hasattr(obj, 'configLibraries'): 391 obj.configLibraries.argDB = argDB 392 for obj in self.dynamicLinkerObject.values(): 393 if hasattr(obj, 'configLibraries'): 394 obj.configLibraries.argDB = argDB 395 return 396 argDB = property(args.ArgumentProcessor.getArgDB, setArgDB, doc = 'The RDict argument database') 397 398 def getLanguageModule(self, language, moduleName = None): 399 '''Return the module associated with operations for a given language 400 - Giving a moduleName explicitly forces a reimport''' 401 if not language in self.languageModule or not moduleName is None: 402 try: 403 if moduleName is None: 404 moduleName = self.modulePath+'.'+language 405 module = __import__(moduleName) 406 except ImportError, e: 407 if not moduleName is None: 408 self.logPrint('Failure to find language module: '+str(e)) 409 try: 410 moduleName = self.modulePath+'.'+language 411 module = __import__(moduleName) 412 except ImportError, e: 413 self.logPrint('Failure to find language module: '+str(e)) 414 moduleName = 'config.compile.'+language 415 module = __import__(moduleName) 416 components = moduleName.split('.') 417 for component in components[1:]: 418 module = getattr(module, component) 419 module._loadName = moduleName 420 self.languageModule[language] = module 421 return self.languageModule[language] 422 423 def getPreprocessorObject(self, language): 424 if not language in self.preprocessorObject: 425 self.preprocessorObject[language] = self.getLanguageModule(language).Preprocessor(self.argDB) 426 self.preprocessorObject[language].setup() 427 if not self.compilers is None: 428 self.preprocessorObject[language].configCompilers = self.compilers 429 if not self.versionControl is None: 430 self.preprocessorObject[language].versionControl = self.versionControl 431 return self.preprocessorObject[language] 432 433 def setPreprocessorObject(self, language, preprocessor): 434 self.preprocessorObject[language] = preprocessor 435 return self.getPreprocessorObject(language) 436 437 def getCompilerObject(self, language): 438 if not language in self.compilerObject: 439 self.compilerObject[language] = self.getLanguageModule(language).Compiler(self.argDB) 440 self.compilerObject[language].setup() 441 if not self.compilers is None: 442 self.compilerObject[language].configCompilers = self.compilers 443 if not self.versionControl is None: 444 self.compilerObject[language].versionControl = self.versionControl 445 return self.compilerObject[language] 446 447 def setCompilerObject(self, language, compiler): 448 self.compilerObject[language] = compiler 449 return self.getCompilerObject(language) 450 451 def getLinkerObject(self, language): 452 if not language in self.linkerObject: 453 self.linkerObject[language] = self.getLanguageModule(language).Linker(self.argDB) 454 self.linkerObject[language].setup() 455 if not self.compilers is None: 456 self.linkerObject[language].configCompilers = self.compilers 457 if not self.libraries is None: 458 self.linkerObject[language].configLibraries = self.libraries 459 if not self.versionControl is None: 460 self.linkerObject[language].versionControl = self.versionControl 461 return self.linkerObject[language] 462 463 def setLinkerObject(self, language, linker): 464 self.linkerObject[language] = linker 465 return self.getLinkerObject(language) 466 467 def getSharedLinkerObject(self, language): 468 if not language in self.sharedLinkerObject: 469 self.sharedLinkerObject[language] = self.getLanguageModule(language).SharedLinker(self.argDB) 470 self.sharedLinkerObject[language].setup() 471 if not self.compilers is None: 472 self.sharedLinkerObject[language].configCompilers = self.compilers 473 if not self.libraries is None: 474 self.sharedLinkerObject[language].configLibraries = self.libraries 475 if not self.versionControl is None: 476 self.sharedLinkerObject[language].versionControl = self.versionControl 477 return self.sharedLinkerObject[language] 478 479 def setSharedLinkerObject(self, language, linker): 480 self.sharedLinkerObject[language] = linker 481 return self.getSharedLinkerObject(language) 482 483 def getDynamicLinkerObject(self, language): 484 if not language in self.dynamicLinkerObject: 485 self.dynamicLinkerObject[language] = self.getLanguageModule(language).DynamicLinker(self.argDB) 486 self.dynamicLinkerObject[language].setup() 487 if not self.compilers is None: 488 self.dynamicLinkerObject[language].configCompilers = self.compilers 489 if not self.libraries is None: 490 self.dynamicLinkerObject[language].configLibraries = self.libraries 491 if not self.versionControl is None: 492 self.dynamicLinkerObject[language].versionControl = self.versionControl 493 return self.dynamicLinkerObject[language] 494 495 def setDynamicLinkerObject(self, language, linker): 496 self.dynamicLinkerObject[language] = linker 497 return self.getDynamicLinkerObject(language) 498