1import sys 2if not hasattr(sys, 'version_info'): 3 print '*** Python version 1 is not supported. Please get the latest version from www.python.org ***' 4 sys.exit(4) 5 6import cPickle 7 8try: 9 import subprocess 10 USE_SUBPROCESS = 1 11except ImportError: 12 USE_SUBPROCESS = 0 13 14# Some features related to detecting login failures cannot be easily 15# implemented with the 'subprocess' module. Disable it for now ... 16USE_SUBPROCESS = 0 17# In Python 2.6 and above, the 'popen2' module is deprecated 18if sys.version_info[:2] >= (2, 6) and not USE_SUBPROCESS: 19 import warnings 20 warnings.filterwarnings('ignore', category=DeprecationWarning, module=__name__) 21 22import nargs 23useThreads = nargs.Arg.findArgument('useThreads', sys.argv[1:]) 24if useThreads is None: 25 useThreads = 0 # worarround issue with parallel configure 26else: 27 useThreads = int(useThreads) 28 29useSelect = nargs.Arg.findArgument('useSelect', sys.argv[1:]) 30if useSelect is None: 31 useSelect = 1 32else: 33 useSelect = int(useSelect) 34 35import logger 36 37class Script(logger.Logger): 38 def __init__(self, clArgs = None, argDB = None, log = None): 39 self.checkPython() 40 logger.Logger.__init__(self, clArgs, argDB, log) 41 self.shell = '/bin/sh' 42 self.showHelp = 1 43 return 44 45 def hasHelpFlag(self): 46 '''Decide whether to display the help message and exit''' 47 import nargs 48 49 if not self.showHelp: 50 return 0 51 if nargs.Arg.findArgument('help', self.clArgs) is None and nargs.Arg.findArgument('h', self.clArgs) is None: 52 return 0 53 return 1 54 55 def setupArguments(self, argDB): 56 '''This method now also creates the help and action logs''' 57 import help 58 59 argDB = logger.Logger.setupArguments(self, argDB) 60 61 self.help = help.Help(argDB) 62 self.help.title = 'Script Help' 63 64 self.actions = help.Info(argDB) 65 self.actions.title = 'Script Actions' 66 67 self.setupHelp(self.help) 68 return argDB 69 70 def setupHelp(self, help): 71 '''This method should be overidden to provide help for arguments''' 72 import nargs 73 74 help.addArgument('Script', '-help', nargs.ArgBool(None, 0, 'Print this help message', isTemporary = 1), ignoreDuplicates = 1) 75 help.addArgument('Script', '-h', nargs.ArgBool(None, 0, 'Print this help message', isTemporary = 1), ignoreDuplicates = 1) 76 return help 77 78 def setup(self): 79 ''' This method checks to see whether help was requested''' 80 if hasattr(self, '_setup'): 81 return 82 logger.Logger.setup(self) 83 self._setup = 1 84 if self.hasHelpFlag(): 85 if self.argDB.target == ['default']: 86 sections = None 87 else: 88 sections = self.argDB.target 89 self.help.output(sections = sections) 90 sys.exit() 91 return 92 93 def cleanup(self): 94 '''This method outputs the action log''' 95 self.actions.output(self.log) 96 return 97 98 def checkPython(self): 99 if not hasattr(sys, 'version_info') or float(sys.version_info[0]) != 2 or float(sys.version_info[1]) < 4: 100 raise RuntimeError('BuildSystem requires Python2 version 2.4 or higher. Get Python at http://www.python.org') 101 return 102 103 def getModule(root, name): 104 '''Retrieve a specific module from the directory root, bypassing the usual paths''' 105 import imp 106 107 (fp, pathname, description) = imp.find_module(name, [root]) 108 try: 109 return imp.load_module(name, fp, pathname, description) 110 finally: 111 if fp: fp.close() 112 return 113 getModule = staticmethod(getModule) 114 115 def importModule(moduleName): 116 '''Import the named module, and return the module object 117 - Works properly for fully qualified names''' 118 module = __import__(moduleName) 119 components = moduleName.split('.') 120 for comp in components[1:]: 121 module = getattr(module, comp) 122 return module 123 importModule = staticmethod(importModule) 124 125 if USE_SUBPROCESS: 126 127 def runShellCommand(command, log=None, cwd=None): 128 Popen = subprocess.Popen 129 PIPE = subprocess.PIPE 130 if log: log.write('Executing: %s\n' % (command,)) 131 pipe = Popen(command, cwd=cwd, stdin=None, stdout=PIPE, stderr=PIPE, 132 bufsize=-1, shell=True, universal_newlines=True) 133 (out, err) = pipe.communicate() 134 ret = pipe.returncode 135 return (out, err, ret) 136 137 else: 138 139 def openPipe(command): 140 '''We need to use the asynchronous version here since we want to avoid blocking reads''' 141 import popen2 142 143 pipe = None 144 if hasattr(popen2, 'Popen3'): 145 pipe = popen2.Popen3(command, 1) 146 input = pipe.tochild 147 output = pipe.fromchild 148 err = pipe.childerr 149 else: 150 import os 151 (input, output, err) = os.popen3(command) 152 return (input, output, err, pipe) 153 openPipe = staticmethod(openPipe) 154 155 def runShellCommand(command, log = None, cwd = None): 156 import select, os 157 158 ret = None 159 out = '' 160 err = '' 161 loginError = 0 162 if cwd is not None: 163 oldpath = os.getcwd() 164 os.chdir(cwd) 165 if log: log.write('Executing: %s\n' % (command,)) 166 (input, output, error, pipe) = Script.openPipe(command) 167 if cwd is not None: 168 os.chdir(oldpath) 169 input.close() 170 if useSelect: 171 outputClosed = 0 172 errorClosed = 0 173 lst = [output, error] 174 while 1: 175 try: 176 ready = select.select(lst, [], []) 177 except Exception, e: 178 if log: log.write('** Error calling select() : '+str(e)+'\n') 179 continue 180 if len(ready[0]): 181 if error in ready[0]: 182 msg = error.readline() 183 if msg: 184 err += msg 185 else: 186 errorClosed = 1 187 lst.remove(error) 188 if output in ready[0]: 189 msg = output.readline() 190 if msg: 191 out += msg 192 else: 193 outputClosed = 1 194 lst.remove(output) 195 if out.find('password:') >= 0 or err.find('password:') >= 0: 196 loginError = 1 197 break 198 if outputClosed and errorClosed: 199 break 200 else: 201 out = output.read() 202 err = error.read() 203 output.close() 204 error.close() 205 if pipe: 206 # We would like the NOHANG argument here 207 ret = pipe.wait() 208 if loginError: 209 raise RuntimeError('Could not login to site') 210 return (out, err, ret) 211 212 runShellCommand = staticmethod(runShellCommand) 213 214 def defaultCheckCommand(command, status, output, error): 215 '''Raise an error if the exit status is nonzero''' 216 if status: raise RuntimeError('Could not execute "%s":\n%s' % (command,output+error)) 217 defaultCheckCommand = staticmethod(defaultCheckCommand) 218 219 def executeShellCommand(command, checkCommand = None, timeout = 600.0, log = None, lineLimit = 0, cwd=None): 220 '''Execute a shell command returning the output, and optionally provide a custom error checker 221 - This returns a tuple of the (output, error, statuscode)''' 222 if not checkCommand: 223 checkCommand = Script.defaultCheckCommand 224 if log is None: 225 log = logger.Logger.defaultLog 226 def logOutput(log, output): 227 import re 228 # get rid of multiple blank lines 229 output = re.sub('\n+','\n', output).strip() 230 if output: 231 if lineLimit: 232 output = '\n'.join(output.split('\n')[:lineLimit]) 233 if '\n' in output: # multi-line output 234 log.write('stdout:\n'+output+'\n') 235 else: 236 log.write('stdout: '+output+'\n') 237 return output 238 def runInShell(command, log, cwd): 239 if useThreads: 240 import threading 241 class InShell(threading.Thread): 242 def __init__(self): 243 threading.Thread.__init__(self) 244 self.name = 'Shell Command' 245 self.setDaemon(1) 246 def run(self): 247 (self.output, self.error, self.status) = ('', '', -1) # So these fields exist even if command fails with no output 248 (self.output, self.error, self.status) = Script.runShellCommand(command, log, cwd) 249 thread = InShell() 250 thread.start() 251 thread.join(timeout) 252 if thread.isAlive(): 253 error = 'Runaway process exceeded time limit of '+str(timeout)+'s\n' 254 log.write(error) 255 return ('', error, -1) 256 else: 257 return (thread.output, thread.error, thread.status) 258 else: 259 return Script.runShellCommand(command, log, cwd) 260 261 (output, error, status) = runInShell(command, log, cwd) 262 output = logOutput(log, output) 263 checkCommand(command, status, output, error) 264 return (output, error, status) 265 executeShellCommand = staticmethod(executeShellCommand) 266 267 def loadConfigure(self, argDB = None): 268 if argDB is None: 269 argDB = self.argDB 270 if not 'configureCache' in argDB: 271 self.logPrint('No cached configure in RDict at '+str(argDB.saveFilename)) 272 return None 273 try: 274 cache = argDB['configureCache'] 275 framework = cPickle.loads(cache) 276 framework.framework = framework 277 framework.argDB = argDB 278 self.logPrint('Loaded configure to cache: size '+str(len(cache))) 279 except cPickle.UnpicklingError, e: 280 framework = None 281 self.logPrint('Invalid cached configure: '+str(e)) 282 return framework 283 284import args 285 286class LanguageProcessor(args.ArgumentProcessor): 287 def __init__(self, clArgs = None, argDB = None, framework = None, versionControl = None): 288 self.languageModule = {} 289 self.preprocessorObject = {} 290 self.compilerObject = {} 291 self.linkerObject = {} 292 self.sharedLinkerObject = {} 293 self.dynamicLinkerObject = {} 294 self.framework = framework 295 self.versionControl = versionControl 296 args.ArgumentProcessor.__init__(self, clArgs, argDB) 297 self.outputFiles = {} 298 self.modulePath = 'config.compile' 299 return 300 301 def getCompilers(self): 302 if self.framework is None: 303 return 304 return self.framework.require('config.compilers', None) 305 compilers = property(getCompilers, doc = 'The config.compilers configure object') 306 def getLibraries(self): 307 if self.framework is None: 308 return 309 return self.framework.require('config.libraries', None) 310 libraries = property(getLibraries, doc = 'The config.libraries configure object') 311 312 def __getstate__(self, d = None): 313 '''We only want to pickle the language module names and output files. The other objects are set by configure.''' 314 if d is None: 315 d = args.ArgumentProcessor.__getstate__(self) 316 if 'languageModule' in d: 317 d['languageModule'] = dict([(lang,mod._loadName) for lang,mod in d['languageModule'].items()]) 318 for member in ['preprocessorObject', 'compilerObject', 'linkerObject', 'sharedLinkerObject', 'dynamicLinkerObject', 'framework']: 319 if member in d: 320 del d[member] 321 return d 322 323 def __setstate__(self, d): 324 '''We must create the language modules''' 325 args.ArgumentProcessor.__setstate__(self, d) 326 self.__dict__.update(d) 327 [self.getLanguageModule(language, moduleName) for language,moduleName in self.languageModule.items()] 328 self.preprocessorObject = {} 329 self.compilerObject = {} 330 self.linkerObject = {} 331 self.sharedLinkerObject = {} 332 self.dynamicLinkerObject = {} 333 return 334 335 def setArgDB(self, argDB): 336 args.ArgumentProcessor.setArgDB(self, argDB) 337 for obj in self.preprocessorObject.values(): 338 if not hasattr(obj, 'argDB') or not obj.argDB == argDB: 339 obj.argDB = argDB 340 for obj in self.compilerObject.values(): 341 if not hasattr(obj, 'argDB') or not obj.argDB == argDB: 342 obj.argDB = argDB 343 for obj in self.linkerObject.values(): 344 if not hasattr(obj, 'argDB') or not obj.argDB == argDB: 345 obj.argDB = argDB 346 for obj in self.sharedLinkerObject.values(): 347 if not hasattr(obj, 'argDB') or not obj.argDB == argDB: 348 obj.argDB = argDB 349 for obj in self.dynamicLinkerObject.values(): 350 if not hasattr(obj, 'argDB') or not obj.argDB == argDB: 351 obj.argDB = argDB 352 if not self.compilers is None: 353 self.compilers.argDB = argDB 354 for obj in self.preprocessorObject.values(): 355 if hasattr(obj, 'configCompilers'): 356 obj.configCompilers.argDB = argDB 357 for obj in self.compilerObject.values(): 358 if hasattr(obj, 'configCompilers'): 359 obj.configCompilers.argDB = argDB 360 for obj in self.linkerObject.values(): 361 if hasattr(obj, 'configCompilers'): 362 obj.configCompilers.argDB = argDB 363 for obj in self.sharedLinkerObject.values(): 364 if hasattr(obj, 'configCompilers'): 365 obj.configCompilers.argDB = argDB 366 for obj in self.dynamicLinkerObject.values(): 367 if hasattr(obj, 'configCompilers'): 368 obj.configCompilers.argDB = argDB 369 if not self.libraries is None: 370 self.libraries.argDB = argDB 371 for obj in self.linkerObject.values(): 372 if hasattr(obj, 'configLibraries'): 373 obj.configLibraries.argDB = argDB 374 for obj in self.sharedLinkerObject.values(): 375 if hasattr(obj, 'configLibraries'): 376 obj.configLibraries.argDB = argDB 377 for obj in self.dynamicLinkerObject.values(): 378 if hasattr(obj, 'configLibraries'): 379 obj.configLibraries.argDB = argDB 380 return 381 argDB = property(args.ArgumentProcessor.getArgDB, setArgDB, doc = 'The RDict argument database') 382 383 def getLanguageModule(self, language, moduleName = None): 384 '''Return the module associated with operations for a given language 385 - Giving a moduleName explicitly forces a reimport''' 386 if not language in self.languageModule or not moduleName is None: 387 try: 388 if moduleName is None: 389 moduleName = self.modulePath+'.'+language 390 module = __import__(moduleName) 391 except ImportError, e: 392 if not moduleName is None: 393 self.logPrint('Failure to find language module: '+str(e)) 394 try: 395 moduleName = self.modulePath+'.'+language 396 module = __import__(moduleName) 397 except ImportError, e: 398 self.logPrint('Failure to find language module: '+str(e)) 399 moduleName = 'config.compile.'+language 400 module = __import__(moduleName) 401 components = moduleName.split('.') 402 for component in components[1:]: 403 module = getattr(module, component) 404 module._loadName = moduleName 405 self.languageModule[language] = module 406 return self.languageModule[language] 407 408 def getPreprocessorObject(self, language): 409 if not language in self.preprocessorObject: 410 self.preprocessorObject[language] = self.getLanguageModule(language).Preprocessor(self.argDB) 411 self.preprocessorObject[language].setup() 412 if not self.compilers is None: 413 self.preprocessorObject[language].configCompilers = self.compilers 414 if not self.versionControl is None: 415 self.preprocessorObject[language].versionControl = self.versionControl 416 return self.preprocessorObject[language] 417 418 def setPreprocessorObject(self, language, preprocessor): 419 self.preprocessorObject[language] = preprocessor 420 return self.getPreprocessorObject(language) 421 422 def getCompilerObject(self, language): 423 if not language in self.compilerObject: 424 self.compilerObject[language] = self.getLanguageModule(language).Compiler(self.argDB) 425 self.compilerObject[language].setup() 426 if not self.compilers is None: 427 self.compilerObject[language].configCompilers = self.compilers 428 if not self.versionControl is None: 429 self.compilerObject[language].versionControl = self.versionControl 430 return self.compilerObject[language] 431 432 def setCompilerObject(self, language, compiler): 433 self.compilerObject[language] = compiler 434 return self.getCompilerObject(language) 435 436 def getLinkerObject(self, language): 437 if not language in self.linkerObject: 438 self.linkerObject[language] = self.getLanguageModule(language).Linker(self.argDB) 439 self.linkerObject[language].setup() 440 if not self.compilers is None: 441 self.linkerObject[language].configCompilers = self.compilers 442 if not self.libraries is None: 443 self.linkerObject[language].configLibraries = self.libraries 444 if not self.versionControl is None: 445 self.linkerObject[language].versionControl = self.versionControl 446 return self.linkerObject[language] 447 448 def setLinkerObject(self, language, linker): 449 self.linkerObject[language] = linker 450 return self.getLinkerObject(language) 451 452 def getSharedLinkerObject(self, language): 453 if not language in self.sharedLinkerObject: 454 self.sharedLinkerObject[language] = self.getLanguageModule(language).SharedLinker(self.argDB) 455 self.sharedLinkerObject[language].setup() 456 if not self.compilers is None: 457 self.sharedLinkerObject[language].configCompilers = self.compilers 458 if not self.libraries is None: 459 self.sharedLinkerObject[language].configLibraries = self.libraries 460 if not self.versionControl is None: 461 self.sharedLinkerObject[language].versionControl = self.versionControl 462 return self.sharedLinkerObject[language] 463 464 def setSharedLinkerObject(self, language, linker): 465 self.sharedLinkerObject[language] = linker 466 return self.getSharedLinkerObject(language) 467 468 def getDynamicLinkerObject(self, language): 469 if not language in self.dynamicLinkerObject: 470 self.dynamicLinkerObject[language] = self.getLanguageModule(language).DynamicLinker(self.argDB) 471 self.dynamicLinkerObject[language].setup() 472 if not self.compilers is None: 473 self.dynamicLinkerObject[language].configCompilers = self.compilers 474 if not self.libraries is None: 475 self.dynamicLinkerObject[language].configLibraries = self.libraries 476 if not self.versionControl is None: 477 self.dynamicLinkerObject[language].versionControl = self.versionControl 478 return self.dynamicLinkerObject[language] 479 480 def setDynamicLinkerObject(self, language, linker): 481 self.dynamicLinkerObject[language] = linker 482 return self.getDynamicLinkerObject(language) 483