1import sys 2if not hasattr(sys, 'version_info'): 3 print '*** Python version 1 is not supported. Please get the latest version from www.python.org ***' 4 sys.exit(4) 5 6import cPickle 7 8try: 9 import subprocess 10 USE_SUBPROCESS = 1 11except ImportError: 12 USE_SUBPROCESS = 0 13 14# Some features related to detecting login failures cannot be easily 15# implemented with the 'subprocess' module. Disable it for now ... 16USE_SUBPROCESS = 0 17# In Python 2.6 and above, the 'popen2' module is deprecated 18if sys.version_info[:2] >= (2, 6) and not USE_SUBPROCESS: 19 import warnings 20 warnings.filterwarnings('ignore', category=DeprecationWarning, module=__name__) 21 22import nargs 23useThreads = nargs.Arg.findArgument('useThreads', sys.argv[1:]) 24if useThreads is None: 25 useThreads = 0 # worarround issue with parallel configure 26else: 27 useThreads = int(useThreads) 28 29useSelect = nargs.Arg.findArgument('useSelect', sys.argv[1:]) 30if useSelect is None: 31 useSelect = 1 32else: 33 useSelect = int(useSelect) 34 35import logger 36 37class Script(logger.Logger): 38 def __init__(self, clArgs = None, argDB = None, log = None): 39 self.checkPython() 40 logger.Logger.__init__(self, clArgs, argDB, log) 41 self.shell = '/bin/sh' 42 self.showHelp = 1 43 return 44 45 def hasHelpFlag(self): 46 '''Decide whether to display the help message and exit''' 47 import nargs 48 49 if not self.showHelp: 50 return 0 51 if nargs.Arg.findArgument('help', self.clArgs) is None and nargs.Arg.findArgument('h', self.clArgs) is None: 52 return 0 53 return 1 54 55 def setupArguments(self, argDB): 56 '''This method now also creates the help and action logs''' 57 import help 58 59 argDB = logger.Logger.setupArguments(self, argDB) 60 61 self.help = help.Help(argDB) 62 self.help.title = 'Script Help' 63 64 self.actions = help.Info(argDB) 65 self.actions.title = 'Script Actions' 66 67 self.setupHelp(self.help) 68 return argDB 69 70 def setupHelp(self, help): 71 '''This method should be overidden to provide help for arguments''' 72 import nargs 73 74 help.addArgument('Script', '-help', nargs.ArgBool(None, 0, 'Print this help message', isTemporary = 1), ignoreDuplicates = 1) 75 help.addArgument('Script', '-h', nargs.ArgBool(None, 0, 'Print this help message', isTemporary = 1), ignoreDuplicates = 1) 76 return help 77 78 def setup(self): 79 ''' This method checks to see whether help was requested''' 80 if hasattr(self, '_setup'): 81 return 82 logger.Logger.setup(self) 83 self._setup = 1 84 if self.hasHelpFlag(): 85 self.argDB.readonly = True 86 if self.argDB.target == ['default']: 87 sections = None 88 else: 89 sections = self.argDB.target 90 self.help.output(sections = sections) 91 sys.exit() 92 return 93 94 def cleanup(self): 95 '''This method outputs the action log''' 96 self.actions.output(self.log) 97 return 98 99 def checkPython(self): 100 if not hasattr(sys, 'version_info') or float(sys.version_info[0]) != 2 or float(sys.version_info[1]) < 6: 101 raise RuntimeError('BuildSystem requires Python2 version 2.6 or higher. Get Python at http://www.python.org') 102 return 103 104 def getModule(root, name): 105 '''Retrieve a specific module from the directory root, bypassing the usual paths''' 106 import imp 107 108 (fp, pathname, description) = imp.find_module(name, [root]) 109 try: 110 return imp.load_module(name, fp, pathname, description) 111 finally: 112 if fp: fp.close() 113 return 114 getModule = staticmethod(getModule) 115 116 def importModule(moduleName): 117 '''Import the named module, and return the module object 118 - Works properly for fully qualified names''' 119 module = __import__(moduleName) 120 components = moduleName.split('.') 121 for comp in components[1:]: 122 module = getattr(module, comp) 123 return module 124 importModule = staticmethod(importModule) 125 126 if USE_SUBPROCESS: 127 128 def runShellCommand(command, log=None, cwd=None): 129 Popen = subprocess.Popen 130 PIPE = subprocess.PIPE 131 if log: log.write('Executing: %s\n' % (command,)) 132 pipe = Popen(command, cwd=cwd, stdin=None, stdout=PIPE, stderr=PIPE, 133 bufsize=-1, shell=True, universal_newlines=True) 134 (out, err) = pipe.communicate() 135 ret = pipe.returncode 136 return (out, err, ret) 137 138 else: 139 140 def openPipe(command): 141 '''We need to use the asynchronous version here since we want to avoid blocking reads''' 142 import popen2 143 144 pipe = None 145 if hasattr(popen2, 'Popen3'): 146 pipe = popen2.Popen3(command, 1) 147 input = pipe.tochild 148 output = pipe.fromchild 149 err = pipe.childerr 150 else: 151 import os 152 (input, output, err) = os.popen3(command) 153 return (input, output, err, pipe) 154 openPipe = staticmethod(openPipe) 155 156 def runShellCommand(command, log = None, cwd = None): 157 import select, os 158 159 ret = None 160 out = '' 161 err = '' 162 loginError = 0 163 if cwd is not None: 164 oldpath = os.getcwd() 165 os.chdir(cwd) 166 if log: log.write('Executing: %s\n' % (command,)) 167 (input, output, error, pipe) = Script.openPipe(command) 168 if cwd is not None: 169 os.chdir(oldpath) 170 input.close() 171 if useSelect: 172 outputClosed = 0 173 errorClosed = 0 174 lst = [output, error] 175 while 1: 176 try: 177 ready = select.select(lst, [], []) 178 except Exception, e: 179 if log: log.write('** Error calling select() : '+str(e)+'\n') 180 continue 181 if len(ready[0]): 182 if error in ready[0]: 183 msg = error.readline() 184 if msg: 185 err += msg 186 else: 187 errorClosed = 1 188 lst.remove(error) 189 if output in ready[0]: 190 msg = output.readline() 191 if msg: 192 out += msg 193 else: 194 outputClosed = 1 195 lst.remove(output) 196 if out.find('password:') >= 0 or err.find('password:') >= 0: 197 loginError = 1 198 break 199 if outputClosed and errorClosed: 200 break 201 else: 202 out = output.read() 203 err = error.read() 204 output.close() 205 error.close() 206 if pipe: 207 # We would like the NOHANG argument here 208 ret = pipe.wait() 209 if loginError: 210 raise RuntimeError('Could not login to site') 211 return (out, err, ret) 212 213 runShellCommand = staticmethod(runShellCommand) 214 215 def defaultCheckCommand(command, status, output, error): 216 '''Raise an error if the exit status is nonzero''' 217 if status: raise RuntimeError('Could not execute "%s":\n%s' % (command,output+error)) 218 defaultCheckCommand = staticmethod(defaultCheckCommand) 219 220 def executeShellCommand(command, checkCommand = None, timeout = 600.0, log = None, lineLimit = 0, cwd=None): 221 '''Execute a shell command returning the output, and optionally provide a custom error checker 222 - This returns a tuple of the (output, error, statuscode)''' 223 if not checkCommand: 224 checkCommand = Script.defaultCheckCommand 225 if log is None: 226 log = logger.Logger.defaultLog 227 def logOutput(log, output): 228 import re 229 # get rid of multiple blank lines 230 output = re.sub('\n+','\n', output).strip() 231 if output: 232 if lineLimit: 233 output = '\n'.join(output.split('\n')[:lineLimit]) 234 if '\n' in output: # multi-line output 235 log.write('stdout:\n'+output+'\n') 236 else: 237 log.write('stdout: '+output+'\n') 238 return output 239 def runInShell(command, log, cwd): 240 if useThreads: 241 import threading 242 class InShell(threading.Thread): 243 def __init__(self): 244 threading.Thread.__init__(self) 245 self.name = 'Shell Command' 246 self.setDaemon(1) 247 def run(self): 248 (self.output, self.error, self.status) = ('', '', -1) # So these fields exist even if command fails with no output 249 (self.output, self.error, self.status) = Script.runShellCommand(command, log, cwd) 250 thread = InShell() 251 thread.start() 252 thread.join(timeout) 253 if thread.isAlive(): 254 error = 'Runaway process exceeded time limit of '+str(timeout)+'s\n' 255 log.write(error) 256 return ('', error, -1) 257 else: 258 return (thread.output, thread.error, thread.status) 259 else: 260 return Script.runShellCommand(command, log, cwd) 261 262 (output, error, status) = runInShell(command, log, cwd) 263 output = logOutput(log, output) 264 checkCommand(command, status, output, error) 265 return (output, error, status) 266 executeShellCommand = staticmethod(executeShellCommand) 267 268 def loadConfigure(self, argDB = None): 269 if argDB is None: 270 argDB = self.argDB 271 if not 'configureCache' in argDB: 272 self.logPrint('No cached configure in RDict at '+str(argDB.saveFilename)) 273 return None 274 try: 275 cache = argDB['configureCache'] 276 framework = cPickle.loads(cache) 277 framework.framework = framework 278 framework.argDB = argDB 279 self.logPrint('Loaded configure to cache: size '+str(len(cache))) 280 except cPickle.UnpicklingError, e: 281 framework = None 282 self.logPrint('Invalid cached configure: '+str(e)) 283 return framework 284 285import args 286 287class LanguageProcessor(args.ArgumentProcessor): 288 def __init__(self, clArgs = None, argDB = None, framework = None, versionControl = None): 289 self.languageModule = {} 290 self.preprocessorObject = {} 291 self.compilerObject = {} 292 self.linkerObject = {} 293 self.sharedLinkerObject = {} 294 self.dynamicLinkerObject = {} 295 self.framework = framework 296 self.versionControl = versionControl 297 args.ArgumentProcessor.__init__(self, clArgs, argDB) 298 self.outputFiles = {} 299 self.modulePath = 'config.compile' 300 return 301 302 def getCompilers(self): 303 if self.framework is None: 304 return 305 return self.framework.require('config.compilers', None) 306 compilers = property(getCompilers, doc = 'The config.compilers configure object') 307 def getLibraries(self): 308 if self.framework is None: 309 return 310 return self.framework.require('config.libraries', None) 311 libraries = property(getLibraries, doc = 'The config.libraries configure object') 312 313 def __getstate__(self, d = None): 314 '''We only want to pickle the language module names and output files. The other objects are set by configure.''' 315 if d is None: 316 d = args.ArgumentProcessor.__getstate__(self) 317 if 'languageModule' in d: 318 d['languageModule'] = dict([(lang,mod._loadName) for lang,mod in d['languageModule'].items()]) 319 for member in ['preprocessorObject', 'compilerObject', 'linkerObject', 'sharedLinkerObject', 'dynamicLinkerObject', 'framework']: 320 if member in d: 321 del d[member] 322 return d 323 324 def __setstate__(self, d): 325 '''We must create the language modules''' 326 args.ArgumentProcessor.__setstate__(self, d) 327 self.__dict__.update(d) 328 [self.getLanguageModule(language, moduleName) for language,moduleName in self.languageModule.items()] 329 self.preprocessorObject = {} 330 self.compilerObject = {} 331 self.linkerObject = {} 332 self.sharedLinkerObject = {} 333 self.dynamicLinkerObject = {} 334 return 335 336 def setArgDB(self, argDB): 337 args.ArgumentProcessor.setArgDB(self, argDB) 338 for obj in self.preprocessorObject.values(): 339 if not hasattr(obj, 'argDB') or not obj.argDB == argDB: 340 obj.argDB = argDB 341 for obj in self.compilerObject.values(): 342 if not hasattr(obj, 'argDB') or not obj.argDB == argDB: 343 obj.argDB = argDB 344 for obj in self.linkerObject.values(): 345 if not hasattr(obj, 'argDB') or not obj.argDB == argDB: 346 obj.argDB = argDB 347 for obj in self.sharedLinkerObject.values(): 348 if not hasattr(obj, 'argDB') or not obj.argDB == argDB: 349 obj.argDB = argDB 350 for obj in self.dynamicLinkerObject.values(): 351 if not hasattr(obj, 'argDB') or not obj.argDB == argDB: 352 obj.argDB = argDB 353 if not self.compilers is None: 354 self.compilers.argDB = argDB 355 for obj in self.preprocessorObject.values(): 356 if hasattr(obj, 'configCompilers'): 357 obj.configCompilers.argDB = argDB 358 for obj in self.compilerObject.values(): 359 if hasattr(obj, 'configCompilers'): 360 obj.configCompilers.argDB = argDB 361 for obj in self.linkerObject.values(): 362 if hasattr(obj, 'configCompilers'): 363 obj.configCompilers.argDB = argDB 364 for obj in self.sharedLinkerObject.values(): 365 if hasattr(obj, 'configCompilers'): 366 obj.configCompilers.argDB = argDB 367 for obj in self.dynamicLinkerObject.values(): 368 if hasattr(obj, 'configCompilers'): 369 obj.configCompilers.argDB = argDB 370 if not self.libraries is None: 371 self.libraries.argDB = argDB 372 for obj in self.linkerObject.values(): 373 if hasattr(obj, 'configLibraries'): 374 obj.configLibraries.argDB = argDB 375 for obj in self.sharedLinkerObject.values(): 376 if hasattr(obj, 'configLibraries'): 377 obj.configLibraries.argDB = argDB 378 for obj in self.dynamicLinkerObject.values(): 379 if hasattr(obj, 'configLibraries'): 380 obj.configLibraries.argDB = argDB 381 return 382 argDB = property(args.ArgumentProcessor.getArgDB, setArgDB, doc = 'The RDict argument database') 383 384 def getLanguageModule(self, language, moduleName = None): 385 '''Return the module associated with operations for a given language 386 - Giving a moduleName explicitly forces a reimport''' 387 if not language in self.languageModule or not moduleName is None: 388 try: 389 if moduleName is None: 390 moduleName = self.modulePath+'.'+language 391 module = __import__(moduleName) 392 except ImportError, e: 393 if not moduleName is None: 394 self.logPrint('Failure to find language module: '+str(e)) 395 try: 396 moduleName = self.modulePath+'.'+language 397 module = __import__(moduleName) 398 except ImportError, e: 399 self.logPrint('Failure to find language module: '+str(e)) 400 moduleName = 'config.compile.'+language 401 module = __import__(moduleName) 402 components = moduleName.split('.') 403 for component in components[1:]: 404 module = getattr(module, component) 405 module._loadName = moduleName 406 self.languageModule[language] = module 407 return self.languageModule[language] 408 409 def getPreprocessorObject(self, language): 410 if not language in self.preprocessorObject: 411 self.preprocessorObject[language] = self.getLanguageModule(language).Preprocessor(self.argDB) 412 self.preprocessorObject[language].setup() 413 if not self.compilers is None: 414 self.preprocessorObject[language].configCompilers = self.compilers 415 if not self.versionControl is None: 416 self.preprocessorObject[language].versionControl = self.versionControl 417 return self.preprocessorObject[language] 418 419 def setPreprocessorObject(self, language, preprocessor): 420 self.preprocessorObject[language] = preprocessor 421 return self.getPreprocessorObject(language) 422 423 def getCompilerObject(self, language): 424 if not language in self.compilerObject: 425 self.compilerObject[language] = self.getLanguageModule(language).Compiler(self.argDB) 426 self.compilerObject[language].setup() 427 if not self.compilers is None: 428 self.compilerObject[language].configCompilers = self.compilers 429 if not self.versionControl is None: 430 self.compilerObject[language].versionControl = self.versionControl 431 return self.compilerObject[language] 432 433 def setCompilerObject(self, language, compiler): 434 self.compilerObject[language] = compiler 435 return self.getCompilerObject(language) 436 437 def getLinkerObject(self, language): 438 if not language in self.linkerObject: 439 self.linkerObject[language] = self.getLanguageModule(language).Linker(self.argDB) 440 self.linkerObject[language].setup() 441 if not self.compilers is None: 442 self.linkerObject[language].configCompilers = self.compilers 443 if not self.libraries is None: 444 self.linkerObject[language].configLibraries = self.libraries 445 if not self.versionControl is None: 446 self.linkerObject[language].versionControl = self.versionControl 447 return self.linkerObject[language] 448 449 def setLinkerObject(self, language, linker): 450 self.linkerObject[language] = linker 451 return self.getLinkerObject(language) 452 453 def getSharedLinkerObject(self, language): 454 if not language in self.sharedLinkerObject: 455 self.sharedLinkerObject[language] = self.getLanguageModule(language).SharedLinker(self.argDB) 456 self.sharedLinkerObject[language].setup() 457 if not self.compilers is None: 458 self.sharedLinkerObject[language].configCompilers = self.compilers 459 if not self.libraries is None: 460 self.sharedLinkerObject[language].configLibraries = self.libraries 461 if not self.versionControl is None: 462 self.sharedLinkerObject[language].versionControl = self.versionControl 463 return self.sharedLinkerObject[language] 464 465 def setSharedLinkerObject(self, language, linker): 466 self.sharedLinkerObject[language] = linker 467 return self.getSharedLinkerObject(language) 468 469 def getDynamicLinkerObject(self, language): 470 if not language in self.dynamicLinkerObject: 471 self.dynamicLinkerObject[language] = self.getLanguageModule(language).DynamicLinker(self.argDB) 472 self.dynamicLinkerObject[language].setup() 473 if not self.compilers is None: 474 self.dynamicLinkerObject[language].configCompilers = self.compilers 475 if not self.libraries is None: 476 self.dynamicLinkerObject[language].configLibraries = self.libraries 477 if not self.versionControl is None: 478 self.dynamicLinkerObject[language].versionControl = self.versionControl 479 return self.dynamicLinkerObject[language] 480 481 def setDynamicLinkerObject(self, language, linker): 482 self.dynamicLinkerObject[language] = linker 483 return self.getDynamicLinkerObject(language) 484