1from __future__ import print_function 2from __future__ import absolute_import 3import sys 4if not hasattr(sys, 'version_info'): 5 print('*** Python version 1 is not supported. Please get the latest version from www.python.org ***') 6 sys.exit(4) 7 8import cPickle 9 10try: 11 import subprocess 12 USE_SUBPROCESS = 1 13except ImportError: 14 USE_SUBPROCESS = 0 15 16# Some features related to detecting login failures cannot be easily 17# implemented with the 'subprocess' module. Disable it for now ... 18USE_SUBPROCESS = 0 19# In Python 2.6 and above, the 'popen2' module is deprecated 20if sys.version_info[:2] >= (2, 6) and not USE_SUBPROCESS: 21 import warnings 22 warnings.filterwarnings('ignore', category=DeprecationWarning, module=__name__) 23 24import nargs 25useThreads = nargs.Arg.findArgument('useThreads', sys.argv[1:]) 26if useThreads is None: 27 useThreads = 0 # workaround issue with parallel configure 28elif useThreads == 'no' or useThreads == '0': 29 useThreads = 0 30elif useThreads == 'yes' or useThreads == '1': 31 useThreads = 1 32else: 33 raise RuntimeError('Unknown option value for --useThreads ',useThreads) 34 35useSelect = nargs.Arg.findArgument('useSelect', sys.argv[1:]) 36if useSelect is None: 37 useSelect = 1 38elif useSelect == 'no' or useSelect == '0': 39 useSelect = 0 40elif useSelect == 'yes' or useSelect == '1': 41 useSelect = 1 42else: 43 raise RuntimeError('Unknown option value for --useSelect ',useSelect) 44 45import logger 46 47class Script(logger.Logger): 48 def __init__(self, clArgs = None, argDB = None, log = None): 49 self.checkPython() 50 logger.Logger.__init__(self, clArgs, argDB, log) 51 self.shell = '/bin/sh' 52 self.showHelp = 1 53 return 54 55 def hasHelpFlag(self): 56 '''Decide whether to display the help message and exit''' 57 import nargs 58 59 if not self.showHelp: 60 return 0 61 if nargs.Arg.findArgument('help', self.clArgs) is None and nargs.Arg.findArgument('h', self.clArgs) is None: 62 return 0 63 return 1 64 65 def hasListFlag(self): 66 '''Decide whether to display the list of download files and exit''' 67 import nargs 68 69 if not self.showHelp: 70 return 0 71 if nargs.Arg.findArgument('with-packages-download-dir', self.clArgs) is None: 72 return 0 73 return 1 74 75 def setupArguments(self, argDB): 76 '''This method now also creates the help and action logs''' 77 import help 78 79 argDB = logger.Logger.setupArguments(self, argDB) 80 81 self.help = help.Help(argDB) 82 self.help.title = 'Script Help' 83 84 self.actions = help.Info(argDB) 85 self.actions.title = 'Script Actions' 86 87 self.setupHelp(self.help) 88 return argDB 89 90 def setupHelp(self, help): 91 '''This method should be overidden to provide help for arguments''' 92 import nargs 93 94 help.addArgument('Script', '-h', nargs.ArgBool(None, 0, 'Print this help message', isTemporary = 1), ignoreDuplicates = 1) 95 help.addArgument('Script', '-help', nargs.ArgBool(None, 0, 'Print this help message', isTemporary = 1), ignoreDuplicates = 1) 96 help.addArgument('Script', '-with-packages-download-dir=<dir>', nargs.ArgDir(None,None, 'Skip network download of package tarballs and locate them in specified dir. If not found in dir, print package URL - so it can be obtained manually.', isTemporary = 1), ignoreDuplicates = 1) 97 return help 98 99 def setup(self): 100 ''' This method checks to see whether help was requested''' 101 if hasattr(self, '_setup'): 102 return 103 logger.Logger.setup(self) 104 self._setup = 1 105 if self.hasHelpFlag(): 106 self.argDB.readonly = True 107 if self.argDB.target == ['default']: 108 sections = None 109 else: 110 sections = self.argDB.target 111 self.help.output(sections = sections) 112 sys.exit() 113 if self.hasListFlag(): 114 self.help.outputDownload() 115 return 116 117 def cleanup(self): 118 '''This method outputs the action log''' 119 self.actions.output(self.log) 120 return 121 122 def checkPython(self): 123 if not hasattr(sys, 'version_info') or float(sys.version_info[0]) != 2 or float(sys.version_info[1]) < 6: 124 raise RuntimeError('BuildSystem requires Python2 version 2.6 or higher. Get Python at http://www.python.org') 125 return 126 127 def getModule(root, name): 128 '''Retrieve a specific module from the directory root, bypassing the usual paths''' 129 import imp 130 131 (fp, pathname, description) = imp.find_module(name, [root]) 132 try: 133 return imp.load_module(name, fp, pathname, description) 134 finally: 135 if fp: fp.close() 136 return 137 getModule = staticmethod(getModule) 138 139 def importModule(moduleName): 140 '''Import the named module, and return the module object 141 - Works properly for fully qualified names''' 142 module = __import__(moduleName) 143 components = moduleName.split('.') 144 for comp in components[1:]: 145 module = getattr(module, comp) 146 return module 147 importModule = staticmethod(importModule) 148 149 if USE_SUBPROCESS: 150 151 def runShellCommand(command, log=None, cwd=None): 152 Popen = subprocess.Popen 153 PIPE = subprocess.PIPE 154 if log: log.write('Executing: %s\n' % (command,)) 155 pipe = Popen(command, cwd=cwd, stdin=None, stdout=PIPE, stderr=PIPE, 156 bufsize=-1, shell=True, universal_newlines=True) 157 (out, err) = pipe.communicate() 158 ret = pipe.returncode 159 return (out, err, ret) 160 161 else: 162 163 def openPipe(command): 164 '''We need to use the asynchronous version here since we want to avoid blocking reads''' 165 import popen2 166 167 pipe = None 168 if hasattr(popen2, 'Popen3'): 169 pipe = popen2.Popen3(command, 1) 170 input = pipe.tochild 171 output = pipe.fromchild 172 err = pipe.childerr 173 else: 174 import os 175 (input, output, err) = os.popen3(command) 176 return (input, output, err, pipe) 177 openPipe = staticmethod(openPipe) 178 179 def runShellCommand(command, log = None, cwd = None): 180 import select, os 181 182 ret = None 183 out = '' 184 err = '' 185 loginError = 0 186 if cwd is not None: 187 oldpath = os.getcwd() 188 os.chdir(cwd) 189 if log: log.write('Executing: %s\n' % (command,)) 190 (input, output, error, pipe) = Script.openPipe(command) 191 if cwd is not None: 192 os.chdir(oldpath) 193 input.close() 194 if useSelect: 195 outputClosed = 0 196 errorClosed = 0 197 lst = [output, error] 198 while 1: 199 try: 200 ready = select.select(lst, [], []) 201 except Exception as e: 202 if log: log.write('** Error calling select() : '+str(e)+'\n') 203 continue 204 if len(ready[0]): 205 if error in ready[0]: 206 msg = error.readline() 207 if msg: 208 err += msg 209 else: 210 errorClosed = 1 211 lst.remove(error) 212 if output in ready[0]: 213 msg = output.readline() 214 if msg: 215 out += msg 216 else: 217 outputClosed = 1 218 lst.remove(output) 219 if msg and msg.find('password:') >= 0: 220 loginError = 1 221 break 222 if outputClosed and errorClosed: 223 break 224 else: 225 out = output.read() 226 err = error.read() 227 output.close() 228 error.close() 229 if pipe: 230 # We would like the NOHANG argument here 231 ret = pipe.wait() 232 if loginError: 233 raise RuntimeError('Could not login to site') 234 return (out, err, ret) 235 236 runShellCommand = staticmethod(runShellCommand) 237 238 def defaultCheckCommand(command, status, output, error): 239 '''Raise an error if the exit status is nonzero''' 240 if status: raise RuntimeError('Could not execute "%s":\n%s' % (command,output+error)) 241 defaultCheckCommand = staticmethod(defaultCheckCommand) 242 243 def executeShellCommand(command, checkCommand = None, timeout = 600.0, log = None, lineLimit = 0, cwd=None): 244 '''Execute a shell command returning the output, and optionally provide a custom error checker 245 - This returns a tuple of the (output, error, statuscode)''' 246 if not checkCommand: 247 checkCommand = Script.defaultCheckCommand 248 if log is None: 249 log = logger.Logger.defaultLog 250 def logOutput(log, output): 251 import re 252 # get rid of multiple blank lines 253 output = re.sub('\n+','\n', output).strip() 254 if output: 255 if lineLimit: 256 output = '\n'.join(output.split('\n')[:lineLimit]) 257 if '\n' in output: # multi-line output 258 log.write('stdout:\n'+output+'\n') 259 else: 260 log.write('stdout: '+output+'\n') 261 return output 262 def runInShell(command, log, cwd): 263 if useThreads: 264 import threading 265 class InShell(threading.Thread): 266 def __init__(self): 267 threading.Thread.__init__(self) 268 self.name = 'Shell Command' 269 self.setDaemon(1) 270 def run(self): 271 (self.output, self.error, self.status) = ('', '', -1) # So these fields exist even if command fails with no output 272 (self.output, self.error, self.status) = Script.runShellCommand(command, log, cwd) 273 thread = InShell() 274 thread.start() 275 thread.join(timeout) 276 if thread.isAlive(): 277 error = 'Runaway process exceeded time limit of '+str(timeout)+'s\n' 278 log.write(error) 279 return ('', error, -1) 280 else: 281 return (thread.output, thread.error, thread.status) 282 else: 283 return Script.runShellCommand(command, log, cwd) 284 285 (output, error, status) = runInShell(command, log, cwd) 286 output = logOutput(log, output) 287 checkCommand(command, status, output, error) 288 return (output, error, status) 289 executeShellCommand = staticmethod(executeShellCommand) 290 291 def loadConfigure(self, argDB = None): 292 if argDB is None: 293 argDB = self.argDB 294 if not 'configureCache' in argDB: 295 self.logPrint('No cached configure in RDict at '+str(argDB.saveFilename)) 296 return None 297 try: 298 cache = argDB['configureCache'] 299 framework = cPickle.loads(cache) 300 framework.framework = framework 301 framework.argDB = argDB 302 self.logPrint('Loaded configure to cache: size '+str(len(cache))) 303 except cPickle.UnpicklingError as e: 304 framework = None 305 self.logPrint('Invalid cached configure: '+str(e)) 306 return framework 307 308import args 309 310class LanguageProcessor(args.ArgumentProcessor): 311 def __init__(self, clArgs = None, argDB = None, framework = None, versionControl = None): 312 self.languageModule = {} 313 self.preprocessorObject = {} 314 self.compilerObject = {} 315 self.linkerObject = {} 316 self.sharedLinkerObject = {} 317 self.dynamicLinkerObject = {} 318 self.framework = framework 319 self.versionControl = versionControl 320 args.ArgumentProcessor.__init__(self, clArgs, argDB) 321 self.outputFiles = {} 322 self.modulePath = 'config.compile' 323 return 324 325 def getCompilers(self): 326 if self.framework is None: 327 return 328 return self.framework.require('config.compilers', None) 329 compilers = property(getCompilers, doc = 'The config.compilers configure object') 330 def getLibraries(self): 331 if self.framework is None: 332 return 333 return self.framework.require('config.libraries', None) 334 libraries = property(getLibraries, doc = 'The config.libraries configure object') 335 336 def __getstate__(self, d = None): 337 '''We only want to pickle the language module names and output files. The other objects are set by configure.''' 338 if d is None: 339 d = args.ArgumentProcessor.__getstate__(self) 340 if 'languageModule' in d: 341 d['languageModule'] = dict([(lang,mod._loadName) for lang,mod in d['languageModule'].items()]) 342 for member in ['preprocessorObject', 'compilerObject', 'linkerObject', 'sharedLinkerObject', 'dynamicLinkerObject', 'framework']: 343 if member in d: 344 del d[member] 345 return d 346 347 def __setstate__(self, d): 348 '''We must create the language modules''' 349 args.ArgumentProcessor.__setstate__(self, d) 350 self.__dict__.update(d) 351 [self.getLanguageModule(language, moduleName) for language,moduleName in self.languageModule.items()] 352 self.preprocessorObject = {} 353 self.compilerObject = {} 354 self.linkerObject = {} 355 self.sharedLinkerObject = {} 356 self.dynamicLinkerObject = {} 357 return 358 359 def setArgDB(self, argDB): 360 args.ArgumentProcessor.setArgDB(self, argDB) 361 for obj in self.preprocessorObject.values(): 362 if not hasattr(obj, 'argDB') or not obj.argDB == argDB: 363 obj.argDB = argDB 364 for obj in self.compilerObject.values(): 365 if not hasattr(obj, 'argDB') or not obj.argDB == argDB: 366 obj.argDB = argDB 367 for obj in self.linkerObject.values(): 368 if not hasattr(obj, 'argDB') or not obj.argDB == argDB: 369 obj.argDB = argDB 370 for obj in self.sharedLinkerObject.values(): 371 if not hasattr(obj, 'argDB') or not obj.argDB == argDB: 372 obj.argDB = argDB 373 for obj in self.dynamicLinkerObject.values(): 374 if not hasattr(obj, 'argDB') or not obj.argDB == argDB: 375 obj.argDB = argDB 376 if not self.compilers is None: 377 self.compilers.argDB = argDB 378 for obj in self.preprocessorObject.values(): 379 if hasattr(obj, 'configCompilers'): 380 obj.configCompilers.argDB = argDB 381 for obj in self.compilerObject.values(): 382 if hasattr(obj, 'configCompilers'): 383 obj.configCompilers.argDB = argDB 384 for obj in self.linkerObject.values(): 385 if hasattr(obj, 'configCompilers'): 386 obj.configCompilers.argDB = argDB 387 for obj in self.sharedLinkerObject.values(): 388 if hasattr(obj, 'configCompilers'): 389 obj.configCompilers.argDB = argDB 390 for obj in self.dynamicLinkerObject.values(): 391 if hasattr(obj, 'configCompilers'): 392 obj.configCompilers.argDB = argDB 393 if not self.libraries is None: 394 self.libraries.argDB = argDB 395 for obj in self.linkerObject.values(): 396 if hasattr(obj, 'configLibraries'): 397 obj.configLibraries.argDB = argDB 398 for obj in self.sharedLinkerObject.values(): 399 if hasattr(obj, 'configLibraries'): 400 obj.configLibraries.argDB = argDB 401 for obj in self.dynamicLinkerObject.values(): 402 if hasattr(obj, 'configLibraries'): 403 obj.configLibraries.argDB = argDB 404 return 405 argDB = property(args.ArgumentProcessor.getArgDB, setArgDB, doc = 'The RDict argument database') 406 407 def getLanguageModule(self, language, moduleName = None): 408 '''Return the module associated with operations for a given language 409 - Giving a moduleName explicitly forces a reimport''' 410 if not language in self.languageModule or not moduleName is None: 411 try: 412 if moduleName is None: 413 moduleName = self.modulePath+'.'+language 414 module = __import__(moduleName) 415 except ImportError as e: 416 if not moduleName is None: 417 self.logPrint('Failure to find language module: '+str(e)) 418 try: 419 moduleName = self.modulePath+'.'+language 420 module = __import__(moduleName) 421 except ImportError as e: 422 self.logPrint('Failure to find language module: '+str(e)) 423 moduleName = 'config.compile.'+language 424 module = __import__(moduleName) 425 components = moduleName.split('.') 426 for component in components[1:]: 427 module = getattr(module, component) 428 module._loadName = moduleName 429 self.languageModule[language] = module 430 return self.languageModule[language] 431 432 def getPreprocessorObject(self, language): 433 if not language in self.preprocessorObject: 434 self.preprocessorObject[language] = self.getLanguageModule(language).Preprocessor(self.argDB) 435 self.preprocessorObject[language].setup() 436 if not self.compilers is None: 437 self.preprocessorObject[language].configCompilers = self.compilers 438 if not self.versionControl is None: 439 self.preprocessorObject[language].versionControl = self.versionControl 440 return self.preprocessorObject[language] 441 442 def setPreprocessorObject(self, language, preprocessor): 443 self.preprocessorObject[language] = preprocessor 444 return self.getPreprocessorObject(language) 445 446 def getCompilerObject(self, language): 447 if not language in self.compilerObject: 448 self.compilerObject[language] = self.getLanguageModule(language).Compiler(self.argDB) 449 self.compilerObject[language].setup() 450 if not self.compilers is None: 451 self.compilerObject[language].configCompilers = self.compilers 452 if not self.versionControl is None: 453 self.compilerObject[language].versionControl = self.versionControl 454 return self.compilerObject[language] 455 456 def setCompilerObject(self, language, compiler): 457 self.compilerObject[language] = compiler 458 return self.getCompilerObject(language) 459 460 def getLinkerObject(self, language): 461 if not language in self.linkerObject: 462 self.linkerObject[language] = self.getLanguageModule(language).Linker(self.argDB) 463 self.linkerObject[language].setup() 464 if not self.compilers is None: 465 self.linkerObject[language].configCompilers = self.compilers 466 if not self.libraries is None: 467 self.linkerObject[language].configLibraries = self.libraries 468 if not self.versionControl is None: 469 self.linkerObject[language].versionControl = self.versionControl 470 return self.linkerObject[language] 471 472 def setLinkerObject(self, language, linker): 473 self.linkerObject[language] = linker 474 return self.getLinkerObject(language) 475 476 def getSharedLinkerObject(self, language): 477 if not language in self.sharedLinkerObject: 478 self.sharedLinkerObject[language] = self.getLanguageModule(language).SharedLinker(self.argDB) 479 self.sharedLinkerObject[language].setup() 480 if not self.compilers is None: 481 self.sharedLinkerObject[language].configCompilers = self.compilers 482 if not self.libraries is None: 483 self.sharedLinkerObject[language].configLibraries = self.libraries 484 if not self.versionControl is None: 485 self.sharedLinkerObject[language].versionControl = self.versionControl 486 return self.sharedLinkerObject[language] 487 488 def setSharedLinkerObject(self, language, linker): 489 self.sharedLinkerObject[language] = linker 490 return self.getSharedLinkerObject(language) 491 492 def getDynamicLinkerObject(self, language): 493 if not language in self.dynamicLinkerObject: 494 self.dynamicLinkerObject[language] = self.getLanguageModule(language).DynamicLinker(self.argDB) 495 self.dynamicLinkerObject[language].setup() 496 if not self.compilers is None: 497 self.dynamicLinkerObject[language].configCompilers = self.compilers 498 if not self.libraries is None: 499 self.dynamicLinkerObject[language].configLibraries = self.libraries 500 if not self.versionControl is None: 501 self.dynamicLinkerObject[language].versionControl = self.versionControl 502 return self.dynamicLinkerObject[language] 503 504 def setDynamicLinkerObject(self, language, linker): 505 self.dynamicLinkerObject[language] = linker 506 return self.getDynamicLinkerObject(language) 507