1from __future__ import print_function 2from __future__ import absolute_import 3import sys 4if not hasattr(sys, 'version_info'): 5 print('*** Python version 1 is not supported. Please get the latest version from www.python.org ***') 6 sys.exit(4) 7 8import pickle 9 10import subprocess 11 12import nargs 13 14# Uses threads to monitor running programs and time them out if they take too long 15useThreads = nargs.Arg.findArgument('useThreads', sys.argv[1:]) 16if useThreads == 'no' or useThreads == '0': 17 useThreads = 0 18elif useThreads == None or useThreads == 'yes' or useThreads == '1': 19 useThreads = 1 20else: 21 raise RuntimeError('Unknown option value for --useThreads ',useThreads) 22 23useSelect = nargs.Arg.findArgument('useSelect', sys.argv[1:]) 24if useSelect == 'no' or useSelect == '0': 25 useSelect = 0 26elif useSelect is None or useSelect == 'yes' or useSelect == '1': 27 useSelect = 1 28else: 29 raise RuntimeError('Unknown option value for --useSelect ',useSelect) 30 31# Run parts of configure in parallel, does not currently work; 32# see config/BuildSystem/config/framework.parallelQueueEvaluation() 33useParallel = nargs.Arg.findArgument('useParallel', sys.argv[1:]) 34if useParallel == 'no' or useParallel == '0': 35 useParallel = 0 36elif useParallel is None or useParallel == 'yes': 37 useParallel = 5 38else: 39 if useParallel == '1': 40 # handle case with --useParallel was used 41 found = 0 42 for i in sys.argv[1:]: 43 if i.startswith('--useParallel='): 44 found = 1 45 break 46 if found: useParallel = int(useParallel) 47 else: useParallel = 5 48useParallel = 0 49 50import logger 51 52class Script(logger.Logger): 53 def __init__(self, clArgs = None, argDB = None, log = None): 54 self.checkPython() 55 logger.Logger.__init__(self, clArgs, argDB, log) 56 self.shell = '/bin/sh' 57 self.showHelp = 1 58 return 59 60 def hasHelpFlag(self): 61 '''Decide whether to display the help message and exit''' 62 import nargs 63 64 if not self.showHelp: 65 return 0 66 if nargs.Arg.findArgument('help', self.clArgs) is None and nargs.Arg.findArgument('h', self.clArgs) is None: 67 return 0 68 return 1 69 70 def hasListFlag(self): 71 '''Decide whether to display the list of download files and exit''' 72 import nargs 73 74 if not self.showHelp: 75 return 0 76 if nargs.Arg.findArgument('with-packages-download-dir', self.clArgs) is None: 77 return 0 78 return 1 79 80 def setupArguments(self, argDB): 81 '''This method now also creates the help and action logs''' 82 import help 83 84 argDB = logger.Logger.setupArguments(self, argDB) 85 86 self.help = help.Help(argDB) 87 self.help.title = 'Script Help' 88 89 self.actions = help.Info(argDB) 90 self.actions.title = 'Script Actions' 91 92 self.setupHelp(self.help) 93 return argDB 94 95 def setupHelp(self, help): 96 '''This method should be overridden to provide help for arguments''' 97 import nargs 98 99 help.addArgument('Script', '-h', nargs.ArgBool(None, 0, 'Print this help message', isTemporary = 1), ignoreDuplicates = 1) 100 help.addArgument('Script', '-help', nargs.ArgBool(None, 0, 'Print this help message', isTemporary = 1), ignoreDuplicates = 1) 101 help.addArgument('Script', '-with-packages-download-dir=<dir>', nargs.ArgDir(None,None, 'Skip network download of package tarballs and locate them in specified dir. If not found in dir, print package URL - so it can be obtained manually.', isTemporary = 1), ignoreDuplicates = 1) 102 return help 103 104 def setup(self): 105 ''' This method checks to see whether help was requested''' 106 if hasattr(self, '_setup'): 107 return 108 logger.Logger.setup(self) 109 self._setup = 1 110 if self.hasHelpFlag(): 111 self.argDB.readonly = True 112 if self.argDB.target == ['default']: 113 sections = None 114 else: 115 sections = self.argDB.target 116 self.help.output(sections = sections) 117 sys.exit() 118 if self.hasListFlag(): 119 self.help.outputDownload() 120 return 121 122 def cleanup(self): 123 '''This method outputs the action log''' 124 self.actions.output(self.log) 125 return 126 127 def checkPython(self): 128 if not hasattr(sys, 'version_info') or sys.version_info < (3,4): 129 raise RuntimeError('BuildSystem requires Python version 3.4 or higher. Get Python at https://www.python.org/') 130 return 131 132 @staticmethod 133 def getModule(root, name): 134 '''Retrieve a specific module from the directory root, bypassing the usual paths''' 135 if sys.version_info < (3,12): 136 import imp 137 (fp, pathname, description) = imp.find_module(name, [root]) 138 try: 139 return imp.load_module(name, fp, pathname, description) 140 finally: 141 if fp: fp.close() 142 else: 143 import importlib.util 144 spec = importlib.util.spec_from_file_location(name, root) 145 module = importlib.util.module_from_spec(spec) # novermin 146 sys.modules[name] = module 147 spec.loader.exec_module(module) 148 149 @staticmethod 150 def importModule(moduleName): 151 '''Import the named module, and return the module object 152 - Works properly for fully qualified names''' 153 module = __import__(moduleName) 154 components = moduleName.split('.') 155 for comp in components[1:]: 156 module = getattr(module, comp) 157 return module 158 159 @staticmethod 160 def runShellCommand(command, log=None, cwd=None, env=None): 161 return Script.runShellCommandSeq([command], log=log, cwd=cwd, env=env) 162 163 @staticmethod 164 def runShellCommandSeq(commandseq, log=None, cwd=None, env=None): 165 Popen = subprocess.Popen 166 PIPE = subprocess.PIPE 167 output = '' 168 error = '' 169 ret = 0 170 for command in commandseq: 171 useShell = isinstance(command, str) or isinstance(command, bytes) 172 if log: log.write('Executing: %s\n' % (command,)) 173 try: 174 pipe = Popen(command, cwd=cwd, env=env, stdin=None, stdout=PIPE, stderr=PIPE, 175 shell=useShell) 176 (out, err) = pipe.communicate() 177 out = out.decode(encoding='UTF-8',errors='replace') 178 err = err.decode(encoding='UTF-8',errors='replace') 179 ret = pipe.returncode 180 except Exception as e: 181 if hasattr(e,'message') and hasattr(e,'errno'): 182 return ('', e.message, e.errno) 183 else: 184 return ('', str(e),1) 185 output += out 186 error += err 187 if ret: 188 break 189 return (output, error, ret) 190 191 @staticmethod 192 def defaultCheckCommand(command, status, output, error): 193 '''Raise an error if the exit status is nonzero 194 Since output and error may be huge and the exception error message may be printed to the 195 screen we cannot print the entire output''' 196 if status: 197 mlen = 512//2 198 if len(output) > 2*mlen: 199 output = output[0:mlen]+'\n .... more output .....\n'+output[len(output)- mlen:] 200 if len(error) > 2*mlen: 201 error = error[0:mlen]+'\n .... more error .....\n'+error[len(error)- mlen:] 202 raise RuntimeError('Could not execute "%s":\n%s' % (command,output+error)) 203 204 @staticmethod 205 def passCheckCommand(command, status, output, error): 206 '''Does not check the command results''' 207 208 @staticmethod 209 def executeShellCommand(command, checkCommand = None, timeout = 600.0, log = None, lineLimit = 0, cwd=None, env=None, logOutputflg = True, threads = 0): 210 '''Execute a shell command returning the output, and optionally provide a custom error checker 211 - This returns a tuple of the (output, error, statuscode)''' 212 '''The timeout is ignored unless the threads values is nonzero''' 213 return Script.executeShellCommandSeq([command], checkCommand=checkCommand, timeout=timeout, log=log, lineLimit=lineLimit, cwd=cwd, env=env, logOutputflg = logOutputflg, threads = threads) 214 215 @staticmethod 216 def executeShellCommandSeq(commandseq, checkCommand = None, timeout = 600.0, log = None, lineLimit = 0, cwd=None, env=None, logOutputflg = True, threads = 0): 217 '''Execute a sequence of shell commands (an && chain) returning the output, and optionally provide a custom error checker 218 - This returns a tuple of the (output, error, statuscode)''' 219 if not checkCommand: 220 checkCommand = Script.defaultCheckCommand 221 if log is None: 222 log = logger.Logger.defaultLog 223 def logOutput(log, output, logOutputflg): 224 import re 225 if not logOutputflg: return output 226 # get rid of multiple blank lines 227 output = re.sub('\n+','\n', output).strip() 228 if output: 229 if lineLimit: 230 output = '\n'.join(output.split('\n')[:lineLimit]) 231 if '\n' in output: # multi-line output 232 log.write('stdout:\n'+output+'\n') 233 else: 234 log.write('stdout: '+output+'\n') 235 return output 236 def runInShell(commandseq, log, cwd, env): 237 if useThreads and threads: 238 import threading 239 log.write('Running Executable with threads to time it out at '+str(timeout)+'\n') 240 class InShell(threading.Thread): 241 def __init__(self): 242 threading.Thread.__init__(self) 243 self.name = 'Shell Command' 244 self.setDaemon(1) 245 def run(self): 246 (self.output, self.error, self.status) = ('', '', -1) # So these fields exist even if command fails with no output 247 (self.output, self.error, self.status) = Script.runShellCommandSeq(commandseq, log, cwd, env) 248 thread = InShell() 249 thread.start() 250 thread.join(timeout) 251 if thread.is_alive(): 252 error = 'Runaway process exceeded time limit of '+str(timeout)+'\n' 253 log.write(error) 254 return ('', error, -1) 255 else: 256 return (thread.output, thread.error, thread.status) 257 else: 258 return Script.runShellCommandSeq(commandseq, log, cwd, env) 259 260 (output, error, status) = runInShell(commandseq, log, cwd, env) 261 output = logOutput(log, output,logOutputflg) 262 logOutput(log, error,logOutputflg) 263 checkCommand(commandseq, status, output, error) 264 return (output, error, status) 265 266 def loadConfigure(self, argDB = None): 267 if argDB is None: 268 argDB = self.argDB 269 if not 'configureCache' in argDB: 270 self.logPrint('No cached configure in RDict at '+str(argDB.saveFilename)) 271 return None 272 try: 273 cache = argDB['configureCache'] 274 framework = pickle.loads(cache) 275 framework.framework = framework 276 framework.argDB = argDB 277 self.logPrint('Loaded configure to cache: size '+str(len(cache))) 278 except pickle.UnpicklingError as e: 279 framework = None 280 self.logPrint('Invalid cached configure: '+str(e)) 281 return framework 282 283import args 284 285class LanguageProcessor(args.ArgumentProcessor): 286 def __init__(self, clArgs = None, argDB = None, framework = None, versionControl = None): 287 self.languageModule = {} 288 self.preprocessorObject = {} 289 self.compilerObject = {} 290 self.linkerObject = {} 291 self.sharedLinkerObject = {} 292 self.dynamicLinkerObject = {} 293 self.framework = framework 294 self.versionControl = versionControl 295 args.ArgumentProcessor.__init__(self, clArgs, argDB) 296 self.outputFiles = {} 297 self.modulePath = 'config.compile' 298 return 299 300 def getCompilers(self): 301 if self.framework is None: 302 return 303 return self.framework.require('config.compilers', None) 304 compilers = property(getCompilers, doc = 'The config.compilers configure object') 305 def getLibraries(self): 306 if self.framework is None: 307 return 308 return self.framework.require('config.libraries', None) 309 libraries = property(getLibraries, doc = 'The config.libraries configure object') 310 311 def __getstate__(self, d = None): 312 '''We only want to pickle the language module names and output files. The other objects are set by configure.''' 313 if d is None: 314 d = args.ArgumentProcessor.__getstate__(self) 315 if 'languageModule' in d: 316 d['languageModule'] = dict([(lang,mod._loadName) for lang,mod in d['languageModule'].items()]) 317 for member in ['preprocessorObject', 'compilerObject', 'linkerObject', 'sharedLinkerObject', 'dynamicLinkerObject', 'framework']: 318 if member in d: 319 del d[member] 320 return d 321 322 def __setstate__(self, d): 323 '''We must create the language modules''' 324 args.ArgumentProcessor.__setstate__(self, d) 325 self.__dict__.update(d) 326 [self.getLanguageModule(language, moduleName) for language,moduleName in self.languageModule.items()] 327 self.preprocessorObject = {} 328 self.compilerObject = {} 329 self.linkerObject = {} 330 self.sharedLinkerObject = {} 331 self.dynamicLinkerObject = {} 332 return 333 334 def setArgDB(self, argDB): 335 args.ArgumentProcessor.setArgDB(self, argDB) 336 for obj in self.preprocessorObject.values(): 337 if not hasattr(obj, 'argDB') or not obj.argDB == argDB: 338 obj.argDB = argDB 339 for obj in self.compilerObject.values(): 340 if not hasattr(obj, 'argDB') or not obj.argDB == argDB: 341 obj.argDB = argDB 342 for obj in self.linkerObject.values(): 343 if not hasattr(obj, 'argDB') or not obj.argDB == argDB: 344 obj.argDB = argDB 345 for obj in self.sharedLinkerObject.values(): 346 if not hasattr(obj, 'argDB') or not obj.argDB == argDB: 347 obj.argDB = argDB 348 for obj in self.dynamicLinkerObject.values(): 349 if not hasattr(obj, 'argDB') or not obj.argDB == argDB: 350 obj.argDB = argDB 351 if not self.compilers is None: 352 self.compilers.argDB = argDB 353 for obj in self.preprocessorObject.values(): 354 if hasattr(obj, 'configCompilers'): 355 obj.configCompilers.argDB = argDB 356 for obj in self.compilerObject.values(): 357 if hasattr(obj, 'configCompilers'): 358 obj.configCompilers.argDB = argDB 359 for obj in self.linkerObject.values(): 360 if hasattr(obj, 'configCompilers'): 361 obj.configCompilers.argDB = argDB 362 for obj in self.sharedLinkerObject.values(): 363 if hasattr(obj, 'configCompilers'): 364 obj.configCompilers.argDB = argDB 365 for obj in self.dynamicLinkerObject.values(): 366 if hasattr(obj, 'configCompilers'): 367 obj.configCompilers.argDB = argDB 368 if not self.libraries is None: 369 self.libraries.argDB = argDB 370 for obj in self.linkerObject.values(): 371 if hasattr(obj, 'configLibraries'): 372 obj.configLibraries.argDB = argDB 373 for obj in self.sharedLinkerObject.values(): 374 if hasattr(obj, 'configLibraries'): 375 obj.configLibraries.argDB = argDB 376 for obj in self.dynamicLinkerObject.values(): 377 if hasattr(obj, 'configLibraries'): 378 obj.configLibraries.argDB = argDB 379 return 380 argDB = property(args.ArgumentProcessor.getArgDB, setArgDB, doc = 'The RDict argument database') 381 382 def getLanguageModule(self, language, moduleName = None): 383 '''Return the module associated with operations for a given language 384 - Giving a moduleName explicitly forces a reimport''' 385 if not language in self.languageModule or not moduleName is None: 386 try: 387 if moduleName is None: 388 moduleName = self.modulePath+'.'+language 389 module = __import__(moduleName) 390 except ImportError as e: 391 if not moduleName is None: 392 self.logPrint('Failure to find language module: '+str(e)) 393 try: 394 moduleName = self.modulePath+'.'+language 395 module = __import__(moduleName) 396 except ImportError as e: 397 self.logPrint('Failure to find language module: '+str(e)) 398 moduleName = 'config.compile.'+language 399 module = __import__(moduleName) 400 components = moduleName.split('.') 401 for component in components[1:]: 402 module = getattr(module, component) 403 module._loadName = moduleName 404 self.languageModule[language] = module 405 return self.languageModule[language] 406 407 def getPreprocessorObject(self, language): 408 if not language in self.preprocessorObject: 409 self.preprocessorObject[language] = self.getLanguageModule(language).Preprocessor(self.argDB) 410 self.preprocessorObject[language].setup() 411 if not self.compilers is None: 412 self.preprocessorObject[language].configCompilers = self.compilers 413 if not self.versionControl is None: 414 self.preprocessorObject[language].versionControl = self.versionControl 415 return self.preprocessorObject[language] 416 417 def setPreprocessorObject(self, language, preprocessor): 418 self.preprocessorObject[language] = preprocessor 419 return self.getPreprocessorObject(language) 420 421 def getCompilerObject(self, language): 422 if not language in self.compilerObject: 423 self.compilerObject[language] = self.getLanguageModule(language).Compiler(self.argDB) 424 self.compilerObject[language].setup() 425 if not self.compilers is None: 426 self.compilerObject[language].configCompilers = self.compilers 427 if not self.versionControl is None: 428 self.compilerObject[language].versionControl = self.versionControl 429 return self.compilerObject[language] 430 431 def setCompilerObject(self, language, compiler): 432 self.compilerObject[language] = compiler 433 return self.getCompilerObject(language) 434 435 def getLinkerObject(self, language): 436 if not language in self.linkerObject: 437 self.linkerObject[language] = self.getLanguageModule(language).Linker(self.argDB) 438 self.linkerObject[language].setup() 439 if not self.compilers is None: 440 self.linkerObject[language].configCompilers = self.compilers 441 if not self.libraries is None: 442 self.linkerObject[language].configLibraries = self.libraries 443 if not self.versionControl is None: 444 self.linkerObject[language].versionControl = self.versionControl 445 return self.linkerObject[language] 446 447 def setLinkerObject(self, language, linker): 448 self.linkerObject[language] = linker 449 return self.getLinkerObject(language) 450 451 def getSharedLinkerObject(self, language): 452 if not language in self.sharedLinkerObject: 453 self.sharedLinkerObject[language] = self.getLanguageModule(language).SharedLinker(self.argDB) 454 self.sharedLinkerObject[language].setup() 455 if not self.compilers is None: 456 self.sharedLinkerObject[language].configCompilers = self.compilers 457 if not self.libraries is None: 458 self.sharedLinkerObject[language].configLibraries = self.libraries 459 if not self.versionControl is None: 460 self.sharedLinkerObject[language].versionControl = self.versionControl 461 return self.sharedLinkerObject[language] 462 463 def setSharedLinkerObject(self, language, linker): 464 self.sharedLinkerObject[language] = linker 465 return self.getSharedLinkerObject(language) 466 467 def getDynamicLinkerObject(self, language): 468 if not language in self.dynamicLinkerObject: 469 self.dynamicLinkerObject[language] = self.getLanguageModule(language).DynamicLinker(self.argDB) 470 self.dynamicLinkerObject[language].setup() 471 if not self.compilers is None: 472 self.dynamicLinkerObject[language].configCompilers = self.compilers 473 if not self.libraries is None: 474 self.dynamicLinkerObject[language].configLibraries = self.libraries 475 if not self.versionControl is None: 476 self.dynamicLinkerObject[language].versionControl = self.versionControl 477 return self.dynamicLinkerObject[language] 478 479 def setDynamicLinkerObject(self, language, linker): 480 self.dynamicLinkerObject[language] = linker 481 return self.getDynamicLinkerObject(language) 482