1from __future__ import print_function 2from __future__ import absolute_import 3import sys 4if not hasattr(sys, 'version_info'): 5 print('*** Python version 1 is not supported. Please get the latest version from www.python.org ***') 6 sys.exit(4) 7 8import pickle 9 10import subprocess 11 12import nargs 13 14# Uses threads to monitor running programs and time them out if they take too long 15useThreads = nargs.Arg.findArgument('useThreads', sys.argv[1:]) 16if useThreads == 'no' or useThreads == '0': 17 useThreads = 0 18elif useThreads == None or useThreads == 'yes' or useThreads == '1': 19 useThreads = 1 20else: 21 raise RuntimeError('Unknown option value for --useThreads ',useThreads) 22 23useSelect = nargs.Arg.findArgument('useSelect', sys.argv[1:]) 24if useSelect == 'no' or useSelect == '0': 25 useSelect = 0 26elif useSelect is None or useSelect == 'yes' or useSelect == '1': 27 useSelect = 1 28else: 29 raise RuntimeError('Unknown option value for --useSelect ',useSelect) 30 31# Run parts of configure in parallel, does not currently work; 32# see config/BuildSystem/config/framework.parallelQueueEvaluation() 33useParallel = nargs.Arg.findArgument('useParallel', sys.argv[1:]) 34if useParallel == 'no' or useParallel == '0': 35 useParallel = 0 36elif useParallel is None or useParallel == 'yes': 37 useParallel = 5 38else: 39 if useParallel == '1': 40 # handle case with --useParallel was used 41 found = 0 42 for i in sys.argv[1:]: 43 if i.startswith('--useParallel='): 44 found = 1 45 break 46 if found: useParallel = int(useParallel) 47 else: useParallel = 5 48useParallel = 0 49 50import logger 51 52class Script(logger.Logger): 53 def __init__(self, clArgs = None, argDB = None, log = None): 54 self.checkPython() 55 logger.Logger.__init__(self, clArgs, argDB, log) 56 self.shell = '/bin/sh' 57 self.showHelp = 1 58 return 59 60 def hasHelpFlag(self): 61 '''Decide whether to display the help message and exit''' 62 import nargs 63 64 if not self.showHelp: 65 return 0 66 if nargs.Arg.findArgument('help', self.clArgs) is None and nargs.Arg.findArgument('h', self.clArgs) is None: 67 return 0 68 return 1 69 70 def hasListFlag(self): 71 '''Decide whether to display the list of download files and exit''' 72 import nargs 73 74 if not self.showHelp: 75 return 0 76 if nargs.Arg.findArgument('with-packages-download-dir', self.clArgs) is None: 77 return 0 78 return 1 79 80 def setupArguments(self, argDB): 81 '''This method now also creates the help and action logs''' 82 import help 83 84 argDB = logger.Logger.setupArguments(self, argDB) 85 86 self.help = help.Help(argDB) 87 self.help.title = 'Script Help' 88 89 self.actions = help.Info(argDB) 90 self.actions.title = 'Script Actions' 91 92 self.setupHelp(self.help) 93 return argDB 94 95 def setupHelp(self, help): 96 '''This method should be overridden to provide help for arguments''' 97 import nargs 98 99 help.addArgument('Script', '-h', nargs.ArgBool(None, 0, 'Print this help message', isTemporary = 1), ignoreDuplicates = 1) 100 help.addArgument('Script', '-help', nargs.ArgBool(None, 0, 'Print this help message', isTemporary = 1), ignoreDuplicates = 1) 101 help.addArgument('Script', '-with-packages-download-dir=<dir>', nargs.ArgDir(None,None, 'Skip network download of package tarballs and locate them in specified dir. If not found in dir, print package URL - so it can be obtained manually.', isTemporary = 1), ignoreDuplicates = 1) 102 return help 103 104 def setup(self): 105 ''' This method checks to see whether help was requested''' 106 if hasattr(self, '_setup'): 107 return 108 logger.Logger.setup(self) 109 self._setup = 1 110 if self.hasHelpFlag(): 111 self.argDB.readonly = True 112 if self.argDB.target == ['default']: 113 sections = None 114 else: 115 sections = self.argDB.target 116 self.help.output(sections = sections) 117 sys.exit() 118 if self.hasListFlag(): 119 self.help.outputDownload() 120 return 121 122 def cleanup(self): 123 '''This method outputs the action log''' 124 self.actions.output(self.log) 125 return 126 127 def checkPython(self): 128 if not hasattr(sys, 'version_info') or sys.version_info < (3,4): 129 raise RuntimeError('BuildSystem requires Python version 3.4 or higher. Get Python at https://www.python.org/') 130 return 131 132 @staticmethod 133 def getModule(root, name): 134 '''Retrieve a specific module from the directory root, bypassing the usual paths''' 135 import imp 136 137 (fp, pathname, description) = imp.find_module(name, [root]) 138 try: 139 return imp.load_module(name, fp, pathname, description) 140 finally: 141 if fp: fp.close() 142 143 @staticmethod 144 def importModule(moduleName): 145 '''Import the named module, and return the module object 146 - Works properly for fully qualified names''' 147 module = __import__(moduleName) 148 components = moduleName.split('.') 149 for comp in components[1:]: 150 module = getattr(module, comp) 151 return module 152 153 @staticmethod 154 def runShellCommand(command, log=None, cwd=None, env=None): 155 return Script.runShellCommandSeq([command], log=log, cwd=cwd, env=env) 156 157 @staticmethod 158 def runShellCommandSeq(commandseq, log=None, cwd=None, env=None): 159 Popen = subprocess.Popen 160 PIPE = subprocess.PIPE 161 output = '' 162 error = '' 163 ret = 0 164 for command in commandseq: 165 useShell = isinstance(command, str) or isinstance(command, bytes) 166 if log: log.write('Executing: %s\n' % (command,)) 167 try: 168 pipe = Popen(command, cwd=cwd, env=env, stdin=None, stdout=PIPE, stderr=PIPE, 169 shell=useShell) 170 (out, err) = pipe.communicate() 171 out = out.decode(encoding='UTF-8',errors='replace') 172 err = err.decode(encoding='UTF-8',errors='replace') 173 ret = pipe.returncode 174 except Exception as e: 175 if hasattr(e,'message') and hasattr(e,'errno'): 176 return ('', e.message, e.errno) 177 else: 178 return ('', str(e),1) 179 output += out 180 error += err 181 if ret: 182 break 183 return (output, error, ret) 184 185 @staticmethod 186 def defaultCheckCommand(command, status, output, error): 187 '''Raise an error if the exit status is nonzero 188 Since output and error may be huge and the exception error message may be printed to the 189 screen we cannot print the entire output''' 190 if status: 191 mlen = 512//2 192 if len(output) > 2*mlen: 193 output = output[0:mlen]+'\n .... more output .....\n'+output[len(output)- mlen:] 194 if len(error) > 2*mlen: 195 error = error[0:mlen]+'\n .... more error .....\n'+error[len(error)- mlen:] 196 raise RuntimeError('Could not execute "%s":\n%s' % (command,output+error)) 197 198 @staticmethod 199 def passCheckCommand(command, status, output, error): 200 '''Does not check the command results''' 201 202 @staticmethod 203 def executeShellCommand(command, checkCommand = None, timeout = 600.0, log = None, lineLimit = 0, cwd=None, env=None, logOutputflg = True, threads = 0): 204 '''Execute a shell command returning the output, and optionally provide a custom error checker 205 - This returns a tuple of the (output, error, statuscode)''' 206 '''The timeout is ignored unless the threads values is nonzero''' 207 return Script.executeShellCommandSeq([command], checkCommand=checkCommand, timeout=timeout, log=log, lineLimit=lineLimit, cwd=cwd, env=env, logOutputflg = logOutputflg, threads = threads) 208 209 @staticmethod 210 def executeShellCommandSeq(commandseq, checkCommand = None, timeout = 600.0, log = None, lineLimit = 0, cwd=None, env=None, logOutputflg = True, threads = 0): 211 '''Execute a sequence of shell commands (an && chain) returning the output, and optionally provide a custom error checker 212 - This returns a tuple of the (output, error, statuscode)''' 213 if not checkCommand: 214 checkCommand = Script.defaultCheckCommand 215 if log is None: 216 log = logger.Logger.defaultLog 217 def logOutput(log, output, logOutputflg): 218 import re 219 if not logOutputflg: return output 220 # get rid of multiple blank lines 221 output = re.sub('\n+','\n', output).strip() 222 if output: 223 if lineLimit: 224 output = '\n'.join(output.split('\n')[:lineLimit]) 225 if '\n' in output: # multi-line output 226 log.write('stdout:\n'+output+'\n') 227 else: 228 log.write('stdout: '+output+'\n') 229 return output 230 def runInShell(commandseq, log, cwd, env): 231 if useThreads and threads: 232 import threading 233 log.write('Running Executable with threads to time it out at '+str(timeout)+'\n') 234 class InShell(threading.Thread): 235 def __init__(self): 236 threading.Thread.__init__(self) 237 self.name = 'Shell Command' 238 self.setDaemon(1) 239 def run(self): 240 (self.output, self.error, self.status) = ('', '', -1) # So these fields exist even if command fails with no output 241 (self.output, self.error, self.status) = Script.runShellCommandSeq(commandseq, log, cwd, env) 242 thread = InShell() 243 thread.start() 244 thread.join(timeout) 245 if thread.is_alive(): 246 error = 'Runaway process exceeded time limit of '+str(timeout)+'\n' 247 log.write(error) 248 return ('', error, -1) 249 else: 250 return (thread.output, thread.error, thread.status) 251 else: 252 return Script.runShellCommandSeq(commandseq, log, cwd, env) 253 254 (output, error, status) = runInShell(commandseq, log, cwd, env) 255 output = logOutput(log, output,logOutputflg) 256 logOutput(log, error,logOutputflg) 257 checkCommand(commandseq, status, output, error) 258 return (output, error, status) 259 260 def loadConfigure(self, argDB = None): 261 if argDB is None: 262 argDB = self.argDB 263 if not 'configureCache' in argDB: 264 self.logPrint('No cached configure in RDict at '+str(argDB.saveFilename)) 265 return None 266 try: 267 cache = argDB['configureCache'] 268 framework = pickle.loads(cache) 269 framework.framework = framework 270 framework.argDB = argDB 271 self.logPrint('Loaded configure to cache: size '+str(len(cache))) 272 except pickle.UnpicklingError as e: 273 framework = None 274 self.logPrint('Invalid cached configure: '+str(e)) 275 return framework 276 277import args 278 279class LanguageProcessor(args.ArgumentProcessor): 280 def __init__(self, clArgs = None, argDB = None, framework = None, versionControl = None): 281 self.languageModule = {} 282 self.preprocessorObject = {} 283 self.compilerObject = {} 284 self.linkerObject = {} 285 self.sharedLinkerObject = {} 286 self.dynamicLinkerObject = {} 287 self.framework = framework 288 self.versionControl = versionControl 289 args.ArgumentProcessor.__init__(self, clArgs, argDB) 290 self.outputFiles = {} 291 self.modulePath = 'config.compile' 292 return 293 294 def getCompilers(self): 295 if self.framework is None: 296 return 297 return self.framework.require('config.compilers', None) 298 compilers = property(getCompilers, doc = 'The config.compilers configure object') 299 def getLibraries(self): 300 if self.framework is None: 301 return 302 return self.framework.require('config.libraries', None) 303 libraries = property(getLibraries, doc = 'The config.libraries configure object') 304 305 def __getstate__(self, d = None): 306 '''We only want to pickle the language module names and output files. The other objects are set by configure.''' 307 if d is None: 308 d = args.ArgumentProcessor.__getstate__(self) 309 if 'languageModule' in d: 310 d['languageModule'] = dict([(lang,mod._loadName) for lang,mod in d['languageModule'].items()]) 311 for member in ['preprocessorObject', 'compilerObject', 'linkerObject', 'sharedLinkerObject', 'dynamicLinkerObject', 'framework']: 312 if member in d: 313 del d[member] 314 return d 315 316 def __setstate__(self, d): 317 '''We must create the language modules''' 318 args.ArgumentProcessor.__setstate__(self, d) 319 self.__dict__.update(d) 320 [self.getLanguageModule(language, moduleName) for language,moduleName in self.languageModule.items()] 321 self.preprocessorObject = {} 322 self.compilerObject = {} 323 self.linkerObject = {} 324 self.sharedLinkerObject = {} 325 self.dynamicLinkerObject = {} 326 return 327 328 def setArgDB(self, argDB): 329 args.ArgumentProcessor.setArgDB(self, argDB) 330 for obj in self.preprocessorObject.values(): 331 if not hasattr(obj, 'argDB') or not obj.argDB == argDB: 332 obj.argDB = argDB 333 for obj in self.compilerObject.values(): 334 if not hasattr(obj, 'argDB') or not obj.argDB == argDB: 335 obj.argDB = argDB 336 for obj in self.linkerObject.values(): 337 if not hasattr(obj, 'argDB') or not obj.argDB == argDB: 338 obj.argDB = argDB 339 for obj in self.sharedLinkerObject.values(): 340 if not hasattr(obj, 'argDB') or not obj.argDB == argDB: 341 obj.argDB = argDB 342 for obj in self.dynamicLinkerObject.values(): 343 if not hasattr(obj, 'argDB') or not obj.argDB == argDB: 344 obj.argDB = argDB 345 if not self.compilers is None: 346 self.compilers.argDB = argDB 347 for obj in self.preprocessorObject.values(): 348 if hasattr(obj, 'configCompilers'): 349 obj.configCompilers.argDB = argDB 350 for obj in self.compilerObject.values(): 351 if hasattr(obj, 'configCompilers'): 352 obj.configCompilers.argDB = argDB 353 for obj in self.linkerObject.values(): 354 if hasattr(obj, 'configCompilers'): 355 obj.configCompilers.argDB = argDB 356 for obj in self.sharedLinkerObject.values(): 357 if hasattr(obj, 'configCompilers'): 358 obj.configCompilers.argDB = argDB 359 for obj in self.dynamicLinkerObject.values(): 360 if hasattr(obj, 'configCompilers'): 361 obj.configCompilers.argDB = argDB 362 if not self.libraries is None: 363 self.libraries.argDB = argDB 364 for obj in self.linkerObject.values(): 365 if hasattr(obj, 'configLibraries'): 366 obj.configLibraries.argDB = argDB 367 for obj in self.sharedLinkerObject.values(): 368 if hasattr(obj, 'configLibraries'): 369 obj.configLibraries.argDB = argDB 370 for obj in self.dynamicLinkerObject.values(): 371 if hasattr(obj, 'configLibraries'): 372 obj.configLibraries.argDB = argDB 373 return 374 argDB = property(args.ArgumentProcessor.getArgDB, setArgDB, doc = 'The RDict argument database') 375 376 def getLanguageModule(self, language, moduleName = None): 377 '''Return the module associated with operations for a given language 378 - Giving a moduleName explicitly forces a reimport''' 379 if not language in self.languageModule or not moduleName is None: 380 try: 381 if moduleName is None: 382 moduleName = self.modulePath+'.'+language 383 module = __import__(moduleName) 384 except ImportError as e: 385 if not moduleName is None: 386 self.logPrint('Failure to find language module: '+str(e)) 387 try: 388 moduleName = self.modulePath+'.'+language 389 module = __import__(moduleName) 390 except ImportError as e: 391 self.logPrint('Failure to find language module: '+str(e)) 392 moduleName = 'config.compile.'+language 393 module = __import__(moduleName) 394 components = moduleName.split('.') 395 for component in components[1:]: 396 module = getattr(module, component) 397 module._loadName = moduleName 398 self.languageModule[language] = module 399 return self.languageModule[language] 400 401 def getPreprocessorObject(self, language): 402 if not language in self.preprocessorObject: 403 self.preprocessorObject[language] = self.getLanguageModule(language).Preprocessor(self.argDB) 404 self.preprocessorObject[language].setup() 405 if not self.compilers is None: 406 self.preprocessorObject[language].configCompilers = self.compilers 407 if not self.versionControl is None: 408 self.preprocessorObject[language].versionControl = self.versionControl 409 return self.preprocessorObject[language] 410 411 def setPreprocessorObject(self, language, preprocessor): 412 self.preprocessorObject[language] = preprocessor 413 return self.getPreprocessorObject(language) 414 415 def getCompilerObject(self, language): 416 if not language in self.compilerObject: 417 self.compilerObject[language] = self.getLanguageModule(language).Compiler(self.argDB) 418 self.compilerObject[language].setup() 419 if not self.compilers is None: 420 self.compilerObject[language].configCompilers = self.compilers 421 if not self.versionControl is None: 422 self.compilerObject[language].versionControl = self.versionControl 423 return self.compilerObject[language] 424 425 def setCompilerObject(self, language, compiler): 426 self.compilerObject[language] = compiler 427 return self.getCompilerObject(language) 428 429 def getLinkerObject(self, language): 430 if not language in self.linkerObject: 431 self.linkerObject[language] = self.getLanguageModule(language).Linker(self.argDB) 432 self.linkerObject[language].setup() 433 if not self.compilers is None: 434 self.linkerObject[language].configCompilers = self.compilers 435 if not self.libraries is None: 436 self.linkerObject[language].configLibraries = self.libraries 437 if not self.versionControl is None: 438 self.linkerObject[language].versionControl = self.versionControl 439 return self.linkerObject[language] 440 441 def setLinkerObject(self, language, linker): 442 self.linkerObject[language] = linker 443 return self.getLinkerObject(language) 444 445 def getSharedLinkerObject(self, language): 446 if not language in self.sharedLinkerObject: 447 self.sharedLinkerObject[language] = self.getLanguageModule(language).SharedLinker(self.argDB) 448 self.sharedLinkerObject[language].setup() 449 if not self.compilers is None: 450 self.sharedLinkerObject[language].configCompilers = self.compilers 451 if not self.libraries is None: 452 self.sharedLinkerObject[language].configLibraries = self.libraries 453 if not self.versionControl is None: 454 self.sharedLinkerObject[language].versionControl = self.versionControl 455 return self.sharedLinkerObject[language] 456 457 def setSharedLinkerObject(self, language, linker): 458 self.sharedLinkerObject[language] = linker 459 return self.getSharedLinkerObject(language) 460 461 def getDynamicLinkerObject(self, language): 462 if not language in self.dynamicLinkerObject: 463 self.dynamicLinkerObject[language] = self.getLanguageModule(language).DynamicLinker(self.argDB) 464 self.dynamicLinkerObject[language].setup() 465 if not self.compilers is None: 466 self.dynamicLinkerObject[language].configCompilers = self.compilers 467 if not self.libraries is None: 468 self.dynamicLinkerObject[language].configLibraries = self.libraries 469 if not self.versionControl is None: 470 self.dynamicLinkerObject[language].versionControl = self.versionControl 471 return self.dynamicLinkerObject[language] 472 473 def setDynamicLinkerObject(self, language, linker): 474 self.dynamicLinkerObject[language] = linker 475 return self.getDynamicLinkerObject(language) 476