1from __future__ import print_function 2from __future__ import absolute_import 3import sys 4if not hasattr(sys, 'version_info'): 5 print('*** Python version 1 is not supported. Please get the latest version from www.python.org ***') 6 sys.exit(4) 7 8import pickle 9 10import subprocess 11 12import nargs 13 14# Uses threads to monitor running programs and time them out if they take too long 15useThreads = nargs.Arg.findArgument('useThreads', sys.argv[1:]) 16if useThreads == 'no' or useThreads == '0': 17 useThreads = 0 18elif useThreads == None or useThreads == 'yes' or useThreads == '1': 19 useThreads = 1 20else: 21 raise RuntimeError('Unknown option value for --useThreads ',useThreads) 22 23useSelect = nargs.Arg.findArgument('useSelect', sys.argv[1:]) 24if useSelect == 'no' or useSelect == '0': 25 useSelect = 0 26elif useSelect is None or useSelect == 'yes' or useSelect == '1': 27 useSelect = 1 28else: 29 raise RuntimeError('Unknown option value for --useSelect ',useSelect) 30 31# Run parts of configure in parallel, does not currently work; 32# see config/BuildSystem/config/framework.parallelQueueEvaluation() 33useParallel = nargs.Arg.findArgument('useParallel', sys.argv[1:]) 34if useParallel == 'no' or useParallel == '0': 35 useParallel = 0 36elif useParallel is None or useParallel == 'yes': 37 useParallel = 5 38else: 39 if useParallel == '1': 40 # handle case with --useParallel was used 41 found = 0 42 for i in sys.argv[1:]: 43 if i.startswith('--useParallel='): 44 found = 1 45 break 46 if found: useParallel = int(useParallel) 47 else: useParallel = 5 48useParallel = 0 49 50import logger 51 52class Script(logger.Logger): 53 def __init__(self, clArgs = None, argDB = None, log = None): 54 self.checkPython() 55 logger.Logger.__init__(self, clArgs, argDB, log) 56 self.shell = '/bin/sh' 57 self.showHelp = 1 58 return 59 60 def hasHelpFlag(self): 61 '''Decide whether to display the help message and exit''' 62 import nargs 63 64 if not self.showHelp: 65 return 0 66 if nargs.Arg.findArgument('help', self.clArgs) is None and nargs.Arg.findArgument('h', self.clArgs) is None: 67 return 0 68 return 1 69 70 def hasListFlag(self): 71 '''Decide whether to display the list of download files and exit''' 72 import nargs 73 74 if not self.showHelp: 75 return 0 76 if nargs.Arg.findArgument('with-packages-download-dir', self.clArgs) is None: 77 return 0 78 return 1 79 80 def setupArguments(self, argDB): 81 '''This method now also creates the help and action logs''' 82 import help 83 84 argDB = logger.Logger.setupArguments(self, argDB) 85 86 self.help = help.Help(argDB) 87 self.help.title = 'Script Help' 88 89 self.actions = help.Info(argDB) 90 self.actions.title = 'Script Actions' 91 92 self.setupHelp(self.help) 93 return argDB 94 95 def setupHelp(self, help): 96 '''This method should be overidden to provide help for arguments''' 97 import nargs 98 99 help.addArgument('Script', '-h', nargs.ArgBool(None, 0, 'Print this help message', isTemporary = 1), ignoreDuplicates = 1) 100 help.addArgument('Script', '-help', nargs.ArgBool(None, 0, 'Print this help message', isTemporary = 1), ignoreDuplicates = 1) 101 help.addArgument('Script', '-with-packages-download-dir=<dir>', nargs.ArgDir(None,None, 'Skip network download of package tarballs and locate them in specified dir. If not found in dir, print package URL - so it can be obtained manually.', isTemporary = 1), ignoreDuplicates = 1) 102 return help 103 104 def setup(self): 105 ''' This method checks to see whether help was requested''' 106 if hasattr(self, '_setup'): 107 return 108 logger.Logger.setup(self) 109 self._setup = 1 110 if self.hasHelpFlag(): 111 self.argDB.readonly = True 112 if self.argDB.target == ['default']: 113 sections = None 114 else: 115 sections = self.argDB.target 116 self.help.output(sections = sections) 117 sys.exit() 118 if self.hasListFlag(): 119 self.help.outputDownload() 120 return 121 122 def cleanup(self): 123 '''This method outputs the action log''' 124 self.actions.output(self.log) 125 return 126 127 def checkPython(self): 128 if not hasattr(sys, 'version_info') or sys.version_info < (2,6): 129 raise RuntimeError('BuildSystem requires Python version 2.6 or higher. Get Python at https://www.python.org/') 130 return 131 132 def getModule(root, name): 133 '''Retrieve a specific module from the directory root, bypassing the usual paths''' 134 import imp 135 136 (fp, pathname, description) = imp.find_module(name, [root]) 137 try: 138 return imp.load_module(name, fp, pathname, description) 139 finally: 140 if fp: fp.close() 141 return 142 getModule = staticmethod(getModule) 143 144 def importModule(moduleName): 145 '''Import the named module, and return the module object 146 - Works properly for fully qualified names''' 147 module = __import__(moduleName) 148 components = moduleName.split('.') 149 for comp in components[1:]: 150 module = getattr(module, comp) 151 return module 152 importModule = staticmethod(importModule) 153 154 @staticmethod 155 def runShellCommand(command, log=None, cwd=None): 156 return Script.runShellCommandSeq([command], log=log, cwd=cwd) 157 158 @staticmethod 159 def runShellCommandSeq(commandseq, log=None, cwd=None): 160 Popen = subprocess.Popen 161 PIPE = subprocess.PIPE 162 output = '' 163 error = '' 164 for command in commandseq: 165 useShell = isinstance(command, str) or isinstance(command, bytes) 166 if log: log.write('Executing: %s\n' % (command,)) 167 try: 168 pipe = Popen(command, cwd=cwd, stdin=None, stdout=PIPE, stderr=PIPE, 169 shell=useShell) 170 (out, err) = pipe.communicate() 171 if sys.version_info >= (3,0): 172 out = out.decode(encoding='UTF-8',errors='replace') 173 err = err.decode(encoding='UTF-8',errors='replace') 174 ret = pipe.returncode 175 except Exception as e: 176 if hasattr(e,'message') and hasattr(e,'errno'): 177 return ('', e.message, e.errno) 178 else: 179 return ('', str(e),1) 180 output += out 181 error += err 182 if ret: 183 break 184 return (output, error, ret) 185 186 def defaultCheckCommand(command, status, output, error): 187 '''Raise an error if the exit status is nonzero''' 188 if status: raise RuntimeError('Could not execute "%s":\n%s' % (command,output+error)) 189 defaultCheckCommand = staticmethod(defaultCheckCommand) 190 191 def passCheckCommand(command, status, output, error): 192 '''Does not check the command results''' 193 passCheckCommand = staticmethod(passCheckCommand) 194 195 @staticmethod 196 def executeShellCommand(command, checkCommand = None, timeout = 600.0, log = None, lineLimit = 0, cwd=None, logOutputflg = True, threads = 0): 197 '''Execute a shell command returning the output, and optionally provide a custom error checker 198 - This returns a tuple of the (output, error, statuscode)''' 199 '''The timeout is ignored unless the threads values is nonzero''' 200 return Script.executeShellCommandSeq([command], checkCommand=checkCommand, timeout=timeout, log=log, lineLimit=lineLimit, cwd=cwd,logOutputflg = logOutputflg, threads = threads) 201 202 @staticmethod 203 def executeShellCommandSeq(commandseq, checkCommand = None, timeout = 600.0, log = None, lineLimit = 0, cwd=None, logOutputflg = True, threads = 0): 204 '''Execute a sequence of shell commands (an && chain) returning the output, and optionally provide a custom error checker 205 - This returns a tuple of the (output, error, statuscode)''' 206 if not checkCommand: 207 checkCommand = Script.defaultCheckCommand 208 if log is None: 209 log = logger.Logger.defaultLog 210 def logOutput(log, output, logOutputflg): 211 import re 212 if not logOutputflg: return output 213 # get rid of multiple blank lines 214 output = re.sub('\n+','\n', output).strip() 215 if output: 216 if lineLimit: 217 output = '\n'.join(output.split('\n')[:lineLimit]) 218 if '\n' in output: # multi-line output 219 log.write('stdout:\n'+output+'\n') 220 else: 221 log.write('stdout: '+output+'\n') 222 return output 223 def runInShell(commandseq, log, cwd): 224 if not useThreads: log.write('UseThreads is off\n') 225 if useThreads and threads: 226 import threading 227 log.write('Running Executable with threads to time it out at '+str(timeout)+'\n') 228 class InShell(threading.Thread): 229 def __init__(self): 230 threading.Thread.__init__(self) 231 self.name = 'Shell Command' 232 self.setDaemon(1) 233 def run(self): 234 (self.output, self.error, self.status) = ('', '', -1) # So these fields exist even if command fails with no output 235 (self.output, self.error, self.status) = Script.runShellCommandSeq(commandseq, log, cwd) 236 thread = InShell() 237 thread.start() 238 thread.join(timeout) 239 if thread.is_alive(): 240 error = 'Runaway process exceeded time limit of '+str(timeout)+'\n' 241 log.write(error) 242 return ('', error, -1) 243 else: 244 return (thread.output, thread.error, thread.status) 245 else: 246 log.write('Running Executable WITHOUT threads to time it out\n') 247 return Script.runShellCommandSeq(commandseq, log, cwd) 248 249 (output, error, status) = runInShell(commandseq, log, cwd) 250 output = logOutput(log, output,logOutputflg) 251 checkCommand(commandseq, status, output, error) 252 return (output, error, status) 253 254 def loadConfigure(self, argDB = None): 255 if argDB is None: 256 argDB = self.argDB 257 if not 'configureCache' in argDB: 258 self.logPrint('No cached configure in RDict at '+str(argDB.saveFilename)) 259 return None 260 try: 261 cache = argDB['configureCache'] 262 framework = pickle.loads(cache) 263 framework.framework = framework 264 framework.argDB = argDB 265 self.logPrint('Loaded configure to cache: size '+str(len(cache))) 266 except pickle.UnpicklingError as e: 267 framework = None 268 self.logPrint('Invalid cached configure: '+str(e)) 269 return framework 270 271import args 272 273class LanguageProcessor(args.ArgumentProcessor): 274 def __init__(self, clArgs = None, argDB = None, framework = None, versionControl = None): 275 self.languageModule = {} 276 self.preprocessorObject = {} 277 self.compilerObject = {} 278 self.linkerObject = {} 279 self.sharedLinkerObject = {} 280 self.dynamicLinkerObject = {} 281 self.framework = framework 282 self.versionControl = versionControl 283 args.ArgumentProcessor.__init__(self, clArgs, argDB) 284 self.outputFiles = {} 285 self.modulePath = 'config.compile' 286 return 287 288 def getCompilers(self): 289 if self.framework is None: 290 return 291 return self.framework.require('config.compilers', None) 292 compilers = property(getCompilers, doc = 'The config.compilers configure object') 293 def getLibraries(self): 294 if self.framework is None: 295 return 296 return self.framework.require('config.libraries', None) 297 libraries = property(getLibraries, doc = 'The config.libraries configure object') 298 299 def __getstate__(self, d = None): 300 '''We only want to pickle the language module names and output files. The other objects are set by configure.''' 301 if d is None: 302 d = args.ArgumentProcessor.__getstate__(self) 303 if 'languageModule' in d: 304 d['languageModule'] = dict([(lang,mod._loadName) for lang,mod in d['languageModule'].items()]) 305 for member in ['preprocessorObject', 'compilerObject', 'linkerObject', 'sharedLinkerObject', 'dynamicLinkerObject', 'framework']: 306 if member in d: 307 del d[member] 308 return d 309 310 def __setstate__(self, d): 311 '''We must create the language modules''' 312 args.ArgumentProcessor.__setstate__(self, d) 313 self.__dict__.update(d) 314 [self.getLanguageModule(language, moduleName) for language,moduleName in self.languageModule.items()] 315 self.preprocessorObject = {} 316 self.compilerObject = {} 317 self.linkerObject = {} 318 self.sharedLinkerObject = {} 319 self.dynamicLinkerObject = {} 320 return 321 322 def setArgDB(self, argDB): 323 args.ArgumentProcessor.setArgDB(self, argDB) 324 for obj in self.preprocessorObject.values(): 325 if not hasattr(obj, 'argDB') or not obj.argDB == argDB: 326 obj.argDB = argDB 327 for obj in self.compilerObject.values(): 328 if not hasattr(obj, 'argDB') or not obj.argDB == argDB: 329 obj.argDB = argDB 330 for obj in self.linkerObject.values(): 331 if not hasattr(obj, 'argDB') or not obj.argDB == argDB: 332 obj.argDB = argDB 333 for obj in self.sharedLinkerObject.values(): 334 if not hasattr(obj, 'argDB') or not obj.argDB == argDB: 335 obj.argDB = argDB 336 for obj in self.dynamicLinkerObject.values(): 337 if not hasattr(obj, 'argDB') or not obj.argDB == argDB: 338 obj.argDB = argDB 339 if not self.compilers is None: 340 self.compilers.argDB = argDB 341 for obj in self.preprocessorObject.values(): 342 if hasattr(obj, 'configCompilers'): 343 obj.configCompilers.argDB = argDB 344 for obj in self.compilerObject.values(): 345 if hasattr(obj, 'configCompilers'): 346 obj.configCompilers.argDB = argDB 347 for obj in self.linkerObject.values(): 348 if hasattr(obj, 'configCompilers'): 349 obj.configCompilers.argDB = argDB 350 for obj in self.sharedLinkerObject.values(): 351 if hasattr(obj, 'configCompilers'): 352 obj.configCompilers.argDB = argDB 353 for obj in self.dynamicLinkerObject.values(): 354 if hasattr(obj, 'configCompilers'): 355 obj.configCompilers.argDB = argDB 356 if not self.libraries is None: 357 self.libraries.argDB = argDB 358 for obj in self.linkerObject.values(): 359 if hasattr(obj, 'configLibraries'): 360 obj.configLibraries.argDB = argDB 361 for obj in self.sharedLinkerObject.values(): 362 if hasattr(obj, 'configLibraries'): 363 obj.configLibraries.argDB = argDB 364 for obj in self.dynamicLinkerObject.values(): 365 if hasattr(obj, 'configLibraries'): 366 obj.configLibraries.argDB = argDB 367 return 368 argDB = property(args.ArgumentProcessor.getArgDB, setArgDB, doc = 'The RDict argument database') 369 370 def getLanguageModule(self, language, moduleName = None): 371 '''Return the module associated with operations for a given language 372 - Giving a moduleName explicitly forces a reimport''' 373 if not language in self.languageModule or not moduleName is None: 374 try: 375 if moduleName is None: 376 moduleName = self.modulePath+'.'+language 377 module = __import__(moduleName) 378 except ImportError as e: 379 if not moduleName is None: 380 self.logPrint('Failure to find language module: '+str(e)) 381 try: 382 moduleName = self.modulePath+'.'+language 383 module = __import__(moduleName) 384 except ImportError as e: 385 self.logPrint('Failure to find language module: '+str(e)) 386 moduleName = 'config.compile.'+language 387 module = __import__(moduleName) 388 components = moduleName.split('.') 389 for component in components[1:]: 390 module = getattr(module, component) 391 module._loadName = moduleName 392 self.languageModule[language] = module 393 return self.languageModule[language] 394 395 def getPreprocessorObject(self, language): 396 if not language in self.preprocessorObject: 397 self.preprocessorObject[language] = self.getLanguageModule(language).Preprocessor(self.argDB) 398 self.preprocessorObject[language].setup() 399 if not self.compilers is None: 400 self.preprocessorObject[language].configCompilers = self.compilers 401 if not self.versionControl is None: 402 self.preprocessorObject[language].versionControl = self.versionControl 403 return self.preprocessorObject[language] 404 405 def setPreprocessorObject(self, language, preprocessor): 406 self.preprocessorObject[language] = preprocessor 407 return self.getPreprocessorObject(language) 408 409 def getCompilerObject(self, language): 410 if not language in self.compilerObject: 411 self.compilerObject[language] = self.getLanguageModule(language).Compiler(self.argDB) 412 self.compilerObject[language].setup() 413 if not self.compilers is None: 414 self.compilerObject[language].configCompilers = self.compilers 415 if not self.versionControl is None: 416 self.compilerObject[language].versionControl = self.versionControl 417 return self.compilerObject[language] 418 419 def setCompilerObject(self, language, compiler): 420 self.compilerObject[language] = compiler 421 return self.getCompilerObject(language) 422 423 def getLinkerObject(self, language): 424 if not language in self.linkerObject: 425 self.linkerObject[language] = self.getLanguageModule(language).Linker(self.argDB) 426 self.linkerObject[language].setup() 427 if not self.compilers is None: 428 self.linkerObject[language].configCompilers = self.compilers 429 if not self.libraries is None: 430 self.linkerObject[language].configLibraries = self.libraries 431 if not self.versionControl is None: 432 self.linkerObject[language].versionControl = self.versionControl 433 return self.linkerObject[language] 434 435 def setLinkerObject(self, language, linker): 436 self.linkerObject[language] = linker 437 return self.getLinkerObject(language) 438 439 def getSharedLinkerObject(self, language): 440 if not language in self.sharedLinkerObject: 441 self.sharedLinkerObject[language] = self.getLanguageModule(language).SharedLinker(self.argDB) 442 self.sharedLinkerObject[language].setup() 443 if not self.compilers is None: 444 self.sharedLinkerObject[language].configCompilers = self.compilers 445 if not self.libraries is None: 446 self.sharedLinkerObject[language].configLibraries = self.libraries 447 if not self.versionControl is None: 448 self.sharedLinkerObject[language].versionControl = self.versionControl 449 return self.sharedLinkerObject[language] 450 451 def setSharedLinkerObject(self, language, linker): 452 self.sharedLinkerObject[language] = linker 453 return self.getSharedLinkerObject(language) 454 455 def getDynamicLinkerObject(self, language): 456 if not language in self.dynamicLinkerObject: 457 self.dynamicLinkerObject[language] = self.getLanguageModule(language).DynamicLinker(self.argDB) 458 self.dynamicLinkerObject[language].setup() 459 if not self.compilers is None: 460 self.dynamicLinkerObject[language].configCompilers = self.compilers 461 if not self.libraries is None: 462 self.dynamicLinkerObject[language].configLibraries = self.libraries 463 if not self.versionControl is None: 464 self.dynamicLinkerObject[language].versionControl = self.versionControl 465 return self.dynamicLinkerObject[language] 466 467 def setDynamicLinkerObject(self, language, linker): 468 self.dynamicLinkerObject[language] = linker 469 return self.getDynamicLinkerObject(language) 470