1import sys 2if not hasattr(sys, 'version_info'): 3 print '*** Python version 1 is not supported. Please get the latest version from www.python.org ***' 4 sys.exit(4) 5 6import cPickle 7 8try: 9 import subprocess 10 USE_SUBPROCESS = 1 11except ImportError: 12 USE_SUBPROCESS = 0 13 14# Some features related to detecting login failures cannot be easily 15# implemented with the 'subprocess' module. Disable it for now ... 16USE_SUBPROCESS = 0 17# In Python 2.6 and above, the 'popen2' module is deprecated 18if sys.version_info[:2] >= (2, 6) and not USE_SUBPROCESS: 19 import warnings 20 warnings.filterwarnings('ignore', category=DeprecationWarning, module=__name__) 21 22import nargs 23useThreads = nargs.Arg.findArgument('useThreads', sys.argv[1:]) 24if useThreads is None: 25 useThreads = 0 # workaround issue with parallel configure 26elif useThreads == 'no' or useThreads == '0': 27 useThreads = 0 28elif useThreads == 'yes' or useThreads == '1': 29 useThreads = 1 30else: 31 raise RuntimeError('Unknown option value for --useThreads ',useThreads) 32 33useSelect = nargs.Arg.findArgument('useSelect', sys.argv[1:]) 34if useSelect is None: 35 useSelect = 1 36elif useSelect == 'no' or useSelect == '0': 37 useSelect = 0 38elif useSelect == 'yes' or useSelect == '1': 39 useSelect = 1 40else: 41 raise RuntimeError('Unknown option value for --useSelect ',useSelect) 42 43import logger 44 45class Script(logger.Logger): 46 def __init__(self, clArgs = None, argDB = None, log = None): 47 self.checkPython() 48 logger.Logger.__init__(self, clArgs, argDB, log) 49 self.shell = '/bin/sh' 50 self.showHelp = 1 51 return 52 53 def hasHelpFlag(self): 54 '''Decide whether to display the help message and exit''' 55 import nargs 56 57 if not self.showHelp: 58 return 0 59 if nargs.Arg.findArgument('help', self.clArgs) is None and nargs.Arg.findArgument('h', self.clArgs) is None: 60 return 0 61 return 1 62 63 def hasListFlag(self): 64 '''Decide whether to display the list of download files and exit''' 65 import nargs 66 67 if not self.showHelp: 68 return 0 69 if nargs.Arg.findArgument('with-packages-dir', self.clArgs) is None: 70 return 0 71 return 1 72 73 def setupArguments(self, argDB): 74 '''This method now also creates the help and action logs''' 75 import help 76 77 argDB = logger.Logger.setupArguments(self, argDB) 78 79 self.help = help.Help(argDB) 80 self.help.title = 'Script Help' 81 82 self.actions = help.Info(argDB) 83 self.actions.title = 'Script Actions' 84 85 self.setupHelp(self.help) 86 return argDB 87 88 def setupHelp(self, help): 89 '''This method should be overidden to provide help for arguments''' 90 import nargs 91 92 help.addArgument('Script', '-help', nargs.ArgBool(None, 0, 'Print this help message', isTemporary = 1), ignoreDuplicates = 1) 93 help.addArgument('Script', '-with-packages-dir', nargs.ArgDir(None,None, 'Directory to store downloaded external package tarballs', isTemporary = 1), ignoreDuplicates = 1) 94 help.addArgument('Script', '-h', nargs.ArgBool(None, 0, 'Print this help message', isTemporary = 1), ignoreDuplicates = 1) 95 return help 96 97 def setup(self): 98 ''' This method checks to see whether help was requested''' 99 if hasattr(self, '_setup'): 100 return 101 logger.Logger.setup(self) 102 self._setup = 1 103 if self.hasHelpFlag(): 104 self.argDB.readonly = True 105 if self.argDB.target == ['default']: 106 sections = None 107 else: 108 sections = self.argDB.target 109 self.help.output(sections = sections) 110 sys.exit() 111 if self.hasListFlag(): 112 self.argDB.readonly = True 113 self.help.outputDownload() 114 return 115 116 def cleanup(self): 117 '''This method outputs the action log''' 118 self.actions.output(self.log) 119 return 120 121 def checkPython(self): 122 if not hasattr(sys, 'version_info') or float(sys.version_info[0]) != 2 or float(sys.version_info[1]) < 6: 123 raise RuntimeError('BuildSystem requires Python2 version 2.6 or higher. Get Python at http://www.python.org') 124 return 125 126 def getModule(root, name): 127 '''Retrieve a specific module from the directory root, bypassing the usual paths''' 128 import imp 129 130 (fp, pathname, description) = imp.find_module(name, [root]) 131 try: 132 return imp.load_module(name, fp, pathname, description) 133 finally: 134 if fp: fp.close() 135 return 136 getModule = staticmethod(getModule) 137 138 def importModule(moduleName): 139 '''Import the named module, and return the module object 140 - Works properly for fully qualified names''' 141 module = __import__(moduleName) 142 components = moduleName.split('.') 143 for comp in components[1:]: 144 module = getattr(module, comp) 145 return module 146 importModule = staticmethod(importModule) 147 148 if USE_SUBPROCESS: 149 150 def runShellCommand(command, log=None, cwd=None): 151 Popen = subprocess.Popen 152 PIPE = subprocess.PIPE 153 if log: log.write('Executing: %s\n' % (command,)) 154 pipe = Popen(command, cwd=cwd, stdin=None, stdout=PIPE, stderr=PIPE, 155 bufsize=-1, shell=True, universal_newlines=True) 156 (out, err) = pipe.communicate() 157 ret = pipe.returncode 158 return (out, err, ret) 159 160 else: 161 162 def openPipe(command): 163 '''We need to use the asynchronous version here since we want to avoid blocking reads''' 164 import popen2 165 166 pipe = None 167 if hasattr(popen2, 'Popen3'): 168 pipe = popen2.Popen3(command, 1) 169 input = pipe.tochild 170 output = pipe.fromchild 171 err = pipe.childerr 172 else: 173 import os 174 (input, output, err) = os.popen3(command) 175 return (input, output, err, pipe) 176 openPipe = staticmethod(openPipe) 177 178 def runShellCommand(command, log = None, cwd = None): 179 import select, os 180 181 ret = None 182 out = '' 183 err = '' 184 loginError = 0 185 if cwd is not None: 186 oldpath = os.getcwd() 187 os.chdir(cwd) 188 if log: log.write('Executing: %s\n' % (command,)) 189 (input, output, error, pipe) = Script.openPipe(command) 190 if cwd is not None: 191 os.chdir(oldpath) 192 input.close() 193 if useSelect: 194 outputClosed = 0 195 errorClosed = 0 196 lst = [output, error] 197 while 1: 198 try: 199 ready = select.select(lst, [], []) 200 except Exception, e: 201 if log: log.write('** Error calling select() : '+str(e)+'\n') 202 continue 203 if len(ready[0]): 204 if error in ready[0]: 205 msg = error.readline() 206 if msg: 207 err += msg 208 else: 209 errorClosed = 1 210 lst.remove(error) 211 if output in ready[0]: 212 msg = output.readline() 213 if msg: 214 out += msg 215 else: 216 outputClosed = 1 217 lst.remove(output) 218 if msg and msg.find('password:') >= 0: 219 loginError = 1 220 break 221 if outputClosed and errorClosed: 222 break 223 else: 224 out = output.read() 225 err = error.read() 226 output.close() 227 error.close() 228 if pipe: 229 # We would like the NOHANG argument here 230 ret = pipe.wait() 231 if loginError: 232 raise RuntimeError('Could not login to site') 233 return (out, err, ret) 234 235 runShellCommand = staticmethod(runShellCommand) 236 237 def defaultCheckCommand(command, status, output, error): 238 '''Raise an error if the exit status is nonzero''' 239 if status: raise RuntimeError('Could not execute "%s":\n%s' % (command,output+error)) 240 defaultCheckCommand = staticmethod(defaultCheckCommand) 241 242 def executeShellCommand(command, checkCommand = None, timeout = 600.0, log = None, lineLimit = 0, cwd=None): 243 '''Execute a shell command returning the output, and optionally provide a custom error checker 244 - This returns a tuple of the (output, error, statuscode)''' 245 if not checkCommand: 246 checkCommand = Script.defaultCheckCommand 247 if log is None: 248 log = logger.Logger.defaultLog 249 def logOutput(log, output): 250 import re 251 # get rid of multiple blank lines 252 output = re.sub('\n+','\n', output).strip() 253 if output: 254 if lineLimit: 255 output = '\n'.join(output.split('\n')[:lineLimit]) 256 if '\n' in output: # multi-line output 257 log.write('stdout:\n'+output+'\n') 258 else: 259 log.write('stdout: '+output+'\n') 260 return output 261 def runInShell(command, log, cwd): 262 if useThreads: 263 import threading 264 class InShell(threading.Thread): 265 def __init__(self): 266 threading.Thread.__init__(self) 267 self.name = 'Shell Command' 268 self.setDaemon(1) 269 def run(self): 270 (self.output, self.error, self.status) = ('', '', -1) # So these fields exist even if command fails with no output 271 (self.output, self.error, self.status) = Script.runShellCommand(command, log, cwd) 272 thread = InShell() 273 thread.start() 274 thread.join(timeout) 275 if thread.isAlive(): 276 error = 'Runaway process exceeded time limit of '+str(timeout)+'s\n' 277 log.write(error) 278 return ('', error, -1) 279 else: 280 return (thread.output, thread.error, thread.status) 281 else: 282 return Script.runShellCommand(command, log, cwd) 283 284 (output, error, status) = runInShell(command, log, cwd) 285 output = logOutput(log, output) 286 checkCommand(command, status, output, error) 287 return (output, error, status) 288 executeShellCommand = staticmethod(executeShellCommand) 289 290 def loadConfigure(self, argDB = None): 291 if argDB is None: 292 argDB = self.argDB 293 if not 'configureCache' in argDB: 294 self.logPrint('No cached configure in RDict at '+str(argDB.saveFilename)) 295 return None 296 try: 297 cache = argDB['configureCache'] 298 framework = cPickle.loads(cache) 299 framework.framework = framework 300 framework.argDB = argDB 301 self.logPrint('Loaded configure to cache: size '+str(len(cache))) 302 except cPickle.UnpicklingError, e: 303 framework = None 304 self.logPrint('Invalid cached configure: '+str(e)) 305 return framework 306 307import args 308 309class LanguageProcessor(args.ArgumentProcessor): 310 def __init__(self, clArgs = None, argDB = None, framework = None, versionControl = None): 311 self.languageModule = {} 312 self.preprocessorObject = {} 313 self.compilerObject = {} 314 self.linkerObject = {} 315 self.sharedLinkerObject = {} 316 self.dynamicLinkerObject = {} 317 self.framework = framework 318 self.versionControl = versionControl 319 args.ArgumentProcessor.__init__(self, clArgs, argDB) 320 self.outputFiles = {} 321 self.modulePath = 'config.compile' 322 return 323 324 def getCompilers(self): 325 if self.framework is None: 326 return 327 return self.framework.require('config.compilers', None) 328 compilers = property(getCompilers, doc = 'The config.compilers configure object') 329 def getLibraries(self): 330 if self.framework is None: 331 return 332 return self.framework.require('config.libraries', None) 333 libraries = property(getLibraries, doc = 'The config.libraries configure object') 334 335 def __getstate__(self, d = None): 336 '''We only want to pickle the language module names and output files. The other objects are set by configure.''' 337 if d is None: 338 d = args.ArgumentProcessor.__getstate__(self) 339 if 'languageModule' in d: 340 d['languageModule'] = dict([(lang,mod._loadName) for lang,mod in d['languageModule'].items()]) 341 for member in ['preprocessorObject', 'compilerObject', 'linkerObject', 'sharedLinkerObject', 'dynamicLinkerObject', 'framework']: 342 if member in d: 343 del d[member] 344 return d 345 346 def __setstate__(self, d): 347 '''We must create the language modules''' 348 args.ArgumentProcessor.__setstate__(self, d) 349 self.__dict__.update(d) 350 [self.getLanguageModule(language, moduleName) for language,moduleName in self.languageModule.items()] 351 self.preprocessorObject = {} 352 self.compilerObject = {} 353 self.linkerObject = {} 354 self.sharedLinkerObject = {} 355 self.dynamicLinkerObject = {} 356 return 357 358 def setArgDB(self, argDB): 359 args.ArgumentProcessor.setArgDB(self, argDB) 360 for obj in self.preprocessorObject.values(): 361 if not hasattr(obj, 'argDB') or not obj.argDB == argDB: 362 obj.argDB = argDB 363 for obj in self.compilerObject.values(): 364 if not hasattr(obj, 'argDB') or not obj.argDB == argDB: 365 obj.argDB = argDB 366 for obj in self.linkerObject.values(): 367 if not hasattr(obj, 'argDB') or not obj.argDB == argDB: 368 obj.argDB = argDB 369 for obj in self.sharedLinkerObject.values(): 370 if not hasattr(obj, 'argDB') or not obj.argDB == argDB: 371 obj.argDB = argDB 372 for obj in self.dynamicLinkerObject.values(): 373 if not hasattr(obj, 'argDB') or not obj.argDB == argDB: 374 obj.argDB = argDB 375 if not self.compilers is None: 376 self.compilers.argDB = argDB 377 for obj in self.preprocessorObject.values(): 378 if hasattr(obj, 'configCompilers'): 379 obj.configCompilers.argDB = argDB 380 for obj in self.compilerObject.values(): 381 if hasattr(obj, 'configCompilers'): 382 obj.configCompilers.argDB = argDB 383 for obj in self.linkerObject.values(): 384 if hasattr(obj, 'configCompilers'): 385 obj.configCompilers.argDB = argDB 386 for obj in self.sharedLinkerObject.values(): 387 if hasattr(obj, 'configCompilers'): 388 obj.configCompilers.argDB = argDB 389 for obj in self.dynamicLinkerObject.values(): 390 if hasattr(obj, 'configCompilers'): 391 obj.configCompilers.argDB = argDB 392 if not self.libraries is None: 393 self.libraries.argDB = argDB 394 for obj in self.linkerObject.values(): 395 if hasattr(obj, 'configLibraries'): 396 obj.configLibraries.argDB = argDB 397 for obj in self.sharedLinkerObject.values(): 398 if hasattr(obj, 'configLibraries'): 399 obj.configLibraries.argDB = argDB 400 for obj in self.dynamicLinkerObject.values(): 401 if hasattr(obj, 'configLibraries'): 402 obj.configLibraries.argDB = argDB 403 return 404 argDB = property(args.ArgumentProcessor.getArgDB, setArgDB, doc = 'The RDict argument database') 405 406 def getLanguageModule(self, language, moduleName = None): 407 '''Return the module associated with operations for a given language 408 - Giving a moduleName explicitly forces a reimport''' 409 if not language in self.languageModule or not moduleName is None: 410 try: 411 if moduleName is None: 412 moduleName = self.modulePath+'.'+language 413 module = __import__(moduleName) 414 except ImportError, e: 415 if not moduleName is None: 416 self.logPrint('Failure to find language module: '+str(e)) 417 try: 418 moduleName = self.modulePath+'.'+language 419 module = __import__(moduleName) 420 except ImportError, e: 421 self.logPrint('Failure to find language module: '+str(e)) 422 moduleName = 'config.compile.'+language 423 module = __import__(moduleName) 424 components = moduleName.split('.') 425 for component in components[1:]: 426 module = getattr(module, component) 427 module._loadName = moduleName 428 self.languageModule[language] = module 429 return self.languageModule[language] 430 431 def getPreprocessorObject(self, language): 432 if not language in self.preprocessorObject: 433 self.preprocessorObject[language] = self.getLanguageModule(language).Preprocessor(self.argDB) 434 self.preprocessorObject[language].setup() 435 if not self.compilers is None: 436 self.preprocessorObject[language].configCompilers = self.compilers 437 if not self.versionControl is None: 438 self.preprocessorObject[language].versionControl = self.versionControl 439 return self.preprocessorObject[language] 440 441 def setPreprocessorObject(self, language, preprocessor): 442 self.preprocessorObject[language] = preprocessor 443 return self.getPreprocessorObject(language) 444 445 def getCompilerObject(self, language): 446 if not language in self.compilerObject: 447 self.compilerObject[language] = self.getLanguageModule(language).Compiler(self.argDB) 448 self.compilerObject[language].setup() 449 if not self.compilers is None: 450 self.compilerObject[language].configCompilers = self.compilers 451 if not self.versionControl is None: 452 self.compilerObject[language].versionControl = self.versionControl 453 return self.compilerObject[language] 454 455 def setCompilerObject(self, language, compiler): 456 self.compilerObject[language] = compiler 457 return self.getCompilerObject(language) 458 459 def getLinkerObject(self, language): 460 if not language in self.linkerObject: 461 self.linkerObject[language] = self.getLanguageModule(language).Linker(self.argDB) 462 self.linkerObject[language].setup() 463 if not self.compilers is None: 464 self.linkerObject[language].configCompilers = self.compilers 465 if not self.libraries is None: 466 self.linkerObject[language].configLibraries = self.libraries 467 if not self.versionControl is None: 468 self.linkerObject[language].versionControl = self.versionControl 469 return self.linkerObject[language] 470 471 def setLinkerObject(self, language, linker): 472 self.linkerObject[language] = linker 473 return self.getLinkerObject(language) 474 475 def getSharedLinkerObject(self, language): 476 if not language in self.sharedLinkerObject: 477 self.sharedLinkerObject[language] = self.getLanguageModule(language).SharedLinker(self.argDB) 478 self.sharedLinkerObject[language].setup() 479 if not self.compilers is None: 480 self.sharedLinkerObject[language].configCompilers = self.compilers 481 if not self.libraries is None: 482 self.sharedLinkerObject[language].configLibraries = self.libraries 483 if not self.versionControl is None: 484 self.sharedLinkerObject[language].versionControl = self.versionControl 485 return self.sharedLinkerObject[language] 486 487 def setSharedLinkerObject(self, language, linker): 488 self.sharedLinkerObject[language] = linker 489 return self.getSharedLinkerObject(language) 490 491 def getDynamicLinkerObject(self, language): 492 if not language in self.dynamicLinkerObject: 493 self.dynamicLinkerObject[language] = self.getLanguageModule(language).DynamicLinker(self.argDB) 494 self.dynamicLinkerObject[language].setup() 495 if not self.compilers is None: 496 self.dynamicLinkerObject[language].configCompilers = self.compilers 497 if not self.libraries is None: 498 self.dynamicLinkerObject[language].configLibraries = self.libraries 499 if not self.versionControl is None: 500 self.dynamicLinkerObject[language].versionControl = self.versionControl 501 return self.dynamicLinkerObject[language] 502 503 def setDynamicLinkerObject(self, language, linker): 504 self.dynamicLinkerObject[language] = linker 505 return self.getDynamicLinkerObject(language) 506