1179860b2SJed Brown'''A remote dictionary server 2179860b2SJed Brown 3179860b2SJed Brown RDict is a typed, hierarchical, persistent dictionary intended to manage 4179860b2SJed Brown all arguments or options for a program. The interface remains exactly the 5179860b2SJed Brown same as dict, but the storage is more complicated. 6179860b2SJed Brown 7179860b2SJed Brown Argument typing is handled by wrapping all values stored in the dictionary 8179860b2SJed Brown with nargs.Arg or a subclass. A user can call setType() to set the type of 9179860b2SJed Brown an argument without any value being present. Whenever __getitem__() or 10179860b2SJed Brown __setitem__() is called, values are extracted or replaced in the wrapper. 11179860b2SJed Brown These wrappers can be accessed directly using getType(), setType(), and 12179860b2SJed Brown types(). 13179860b2SJed Brown 14179860b2SJed Brown Hierarchy is allowed using a single "parent" dictionary. All operations 15179860b2SJed Brown cascade to the parent. For instance, the length of the dictionary is the 16179860b2SJed Brown number of local keys plus the number of keys in the parent, and its 17179860b2SJed Brown parent, etc. Also, a dictionary need not have a parent. If a key does not 18d5b43468SJose E. Roman appear in the local dictionary, the call if passed to the parent. However, 19179860b2SJed Brown in this case we see that local keys can shadow those in a parent. 20179860b2SJed Brown Communication with the parent is handled using sockets, with the parent 21179860b2SJed Brown being a server and the interactive dictionary a client. 22179860b2SJed Brown 23179860b2SJed Brown The default persistence mechanism is a pickle file, RDict.db, written 24179860b2SJed Brown whenever an argument is changed locally. A timer thread is created after 25179860b2SJed Brown an initial change, so that many rapid changes do not cause many writes. 26179860b2SJed Brown Each dictionary only saves its local entries, so all parents also 27179860b2SJed Brown separately save data in different RDict.db files. Each time a dictionary 28179860b2SJed Brown is created, the current directory is searched for an RDict.db file, and 29179860b2SJed Brown if found the contents are loaded into the dictionary. 30179860b2SJed Brown 31179860b2SJed Brown This script also provides some default actions: 32179860b2SJed Brown 33179860b2SJed Brown - server [parent] 34179860b2SJed Brown Starts a server in the current directory with an optional parent. This 35179860b2SJed Brown server will accept socket connections from other dictionaries and act 36179860b2SJed Brown as a parent. 37179860b2SJed Brown 38179860b2SJed Brown - client [parent] 39179860b2SJed Brown Creates a dictionary in the current directory with an optional parent 40179860b2SJed Brown and lists the contents. Notice that the contents may come from either 41179860b2SJed Brown an RDict.db file in the current directory, or from the parent. 42179860b2SJed Brown 43179860b2SJed Brown - clear [parent] 44179860b2SJed Brown Creates a dictionary in the current directory with an optional parent 45179860b2SJed Brown and clears the contents. Notice that this will also clear the parent. 46179860b2SJed Brown 47179860b2SJed Brown - insert <parent> <key> <value> 48179860b2SJed Brown Creates a dictionary in the current directory with a parent, and inserts 49179860b2SJed Brown the key-value pair. If "parent" is "None", no parent is assigned. 50179860b2SJed Brown 51179860b2SJed Brown - remove <parent> <key> 52179860b2SJed Brown Creates a dictionary in the current directory with a parent, and removes 53179860b2SJed Brown the given key. If "parent" is "None", no parent is assigned. 54179860b2SJed Brown''' 555b6bfdb9SJed Brownfrom __future__ import print_function 565b6bfdb9SJed Brownfrom __future__ import absolute_import 57179860b2SJed Browntry: 58179860b2SJed Brown import project # This is necessary for us to create Project objects on load 59179860b2SJed Brown import build.buildGraph # This is necessary for us to create BuildGraph objects on load 60179860b2SJed Brownexcept ImportError: 61179860b2SJed Brown pass 62179860b2SJed Brownimport nargs 63179860b2SJed Brown 64492432c8SJed Brownimport pickle 65179860b2SJed Brownimport os 66179860b2SJed Brownimport sys 67179860b2SJed BrownuseThreads = nargs.Arg.findArgument('useThreads', sys.argv[1:]) 68179860b2SJed Brownif useThreads is None: 6992626d4aSBarry Smith useThreads = 0 # workaround issue with parallel configure 7092626d4aSBarry Smithelif useThreads == 'no' or useThreads == '0': 7192626d4aSBarry Smith useThreads = 0 7292626d4aSBarry Smithelif useThreads == 'yes' or useThreads == '1': 7331de54b5SMatthew G. Knepley useThreads = 1 74179860b2SJed Brownelse: 7592626d4aSBarry Smith raise RuntimeError('Unknown option value for --useThreads ',useThreads) 76179860b2SJed Brown 77179860b2SJed Brownclass RDict(dict): 78179860b2SJed Brown '''An RDict is a typed dictionary, which may be hierarchically composed. All elements derive from the 79179860b2SJed BrownArg class, which wraps the usual value.''' 80179860b2SJed Brown # The server will self-shutdown after this many seconds 81179860b2SJed Brown shutdownDelay = 60*60*5 82179860b2SJed Brown 83179860b2SJed Brown def __init__(self, parentAddr = None, parentDirectory = None, load = 1, autoShutdown = 1, readonly = False): 84179860b2SJed Brown import atexit 85179860b2SJed Brown import time 86179860b2SJed Brown 87179860b2SJed Brown self.logFile = None 88179860b2SJed Brown self.setupLogFile() 89179860b2SJed Brown self.target = ['default'] 90179860b2SJed Brown self.parent = None 91179860b2SJed Brown self.saveTimer = None 92179860b2SJed Brown self.shutdownTimer = None 93179860b2SJed Brown self.lastAccess = time.time() 94179860b2SJed Brown self.saveFilename = 'RDict.db' 95179860b2SJed Brown self.addrFilename = 'RDict.loc' 96179860b2SJed Brown self.parentAddr = parentAddr 97179860b2SJed Brown self.isServer = 0 98179860b2SJed Brown self.readonly = readonly 99179860b2SJed Brown self.parentDirectory = parentDirectory 100492432c8SJed Brown self.stopCmd = pickle.dumps(('stop',)) 101179860b2SJed Brown self.writeLogLine('Greetings') 102179860b2SJed Brown self.connectParent(self.parentAddr, self.parentDirectory) 103179860b2SJed Brown if load: self.load() 104179860b2SJed Brown if autoShutdown and useThreads: 105179860b2SJed Brown atexit.register(self.shutdown) 106179860b2SJed Brown self.writeLogLine('SERVER: Last access '+str(self.lastAccess)) 107179860b2SJed Brown return 108179860b2SJed Brown 109179860b2SJed Brown def __getstate__(self): 110*9bb2fd42SSatish Balay '''Remove any parent socket object, and the log file from the dictionary before pickling''' 111179860b2SJed Brown self.writeLogLine('Pickling RDict') 112179860b2SJed Brown d = self.__dict__.copy() 113179860b2SJed Brown if 'parent' in d: del d['parent'] 114179860b2SJed Brown if 'saveTimer' in d: del d['saveTimer'] 115179860b2SJed Brown if '_setCommandLine' in d: del d['_setCommandLine'] 116179860b2SJed Brown del d['logFile'] 117179860b2SJed Brown return d 118179860b2SJed Brown 119179860b2SJed Brown def __setstate__(self, d): 120*9bb2fd42SSatish Balay '''Reconnect the parent socket object and reopen the log file after unpickling''' 121c6ef1b5bSJed Brown self.logFile = open('RDict.log', 'a') 122179860b2SJed Brown self.writeLogLine('Unpickling RDict') 123179860b2SJed Brown self.__dict__.update(d) 124179860b2SJed Brown self.connectParent(self.parentAddr, self.parentDirectory) 125179860b2SJed Brown return 126179860b2SJed Brown 127179860b2SJed Brown def setupLogFile(self, filename = 'RDict.log'): 128179860b2SJed Brown if not self.logFile is None: 129179860b2SJed Brown self.logFile.close() 130179860b2SJed Brown if os.path.isfile(filename) and os.stat(filename).st_size > 10*1024*1024: 131179860b2SJed Brown if os.path.isfile(filename+'.bkp'): 132179860b2SJed Brown os.remove(filename+'.bkp') 133179860b2SJed Brown os.rename(filename, filename+'.bkp') 134c6ef1b5bSJed Brown self.logFile = open(filename, 'w') 135179860b2SJed Brown else: 136c6ef1b5bSJed Brown self.logFile = open(filename, 'a') 137179860b2SJed Brown return 138179860b2SJed Brown 139179860b2SJed Brown def writeLogLine(self, message): 140179860b2SJed Brown '''Writes the message to the log along with the current time''' 141179860b2SJed Brown import time 142179860b2SJed Brown self.logFile.write('('+str(os.getpid())+')('+str(id(self))+')'+message+' ['+time.asctime(time.localtime())+']\n') 143179860b2SJed Brown self.logFile.flush() 144179860b2SJed Brown return 145179860b2SJed Brown 146179860b2SJed Brown def __len__(self): 147179860b2SJed Brown '''Returns the length of both the local and parent dictionaries''' 148179860b2SJed Brown length = dict.__len__(self) 149179860b2SJed Brown if not self.parent is None: 150179860b2SJed Brown length = length + self.send() 151179860b2SJed Brown return length 152179860b2SJed Brown 153179860b2SJed Brown def getType(self, key): 154179860b2SJed Brown '''Checks for the key locally, and if not found consults the parent. Returns the Arg object or None if not found.''' 1553b049ba2SJed Brown try: 1563b049ba2SJed Brown value = dict.__getitem__(self, key) 1573b049ba2SJed Brown self.writeLogLine('getType: Getting local type for '+key+' '+str(value)) 1583b049ba2SJed Brown return value 1593b049ba2SJed Brown except KeyError: 1603b049ba2SJed Brown pass 1613b049ba2SJed Brown if self.parent: 162179860b2SJed Brown return self.send(key) 163179860b2SJed Brown return None 164179860b2SJed Brown 1653b049ba2SJed Brown def dict_has_key(self, key): 1663b049ba2SJed Brown """Utility to check whether the key is present in the dictionary without RDict side-effects.""" 1673b049ba2SJed Brown return key in dict(self) 1683b049ba2SJed Brown 169179860b2SJed Brown def __getitem__(self, key): 170179860b2SJed Brown '''Checks for the key locally, and if not found consults the parent. Returns the value of the Arg. 171179860b2SJed Brown - If the value has not been set, the user will be prompted for input''' 1723b049ba2SJed Brown if self.dict_has_key(key): 173179860b2SJed Brown self.writeLogLine('__getitem__: '+key+' has local type') 174179860b2SJed Brown pass 175179860b2SJed Brown elif not self.parent is None: 176179860b2SJed Brown self.writeLogLine('__getitem__: Checking parent value') 177179860b2SJed Brown if self.send(key, operation = 'has_key'): 178179860b2SJed Brown self.writeLogLine('__getitem__: Parent has value') 179179860b2SJed Brown return self.send(key) 180179860b2SJed Brown else: 181179860b2SJed Brown self.writeLogLine('__getitem__: Checking parent type') 182179860b2SJed Brown arg = self.send(key, operation = 'getType') 183179860b2SJed Brown if not arg: 184179860b2SJed Brown self.writeLogLine('__getitem__: Parent has no type') 185179860b2SJed Brown arg = nargs.Arg(key) 186179860b2SJed Brown try: 187179860b2SJed Brown value = arg.getValue() 1885b6bfdb9SJed Brown except AttributeError as e: 189179860b2SJed Brown self.writeLogLine('__getitem__: Parent had invalid entry: '+str(e)) 190179860b2SJed Brown arg = nargs.Arg(key) 191179860b2SJed Brown value = arg.getValue() 192179860b2SJed Brown self.writeLogLine('__getitem__: Setting parent value '+str(value)) 193179860b2SJed Brown self.send(key, value, operation = '__setitem__') 194179860b2SJed Brown return value 195179860b2SJed Brown else: 196179860b2SJed Brown self.writeLogLine('__getitem__: Setting local type for '+key) 197179860b2SJed Brown dict.__setitem__(self, key, nargs.Arg(key)) 19831f4da61SSatish Balay #self.save() 199179860b2SJed Brown self.writeLogLine('__getitem__: Setting local value for '+key) 200179860b2SJed Brown return dict.__getitem__(self, key).getValue() 201179860b2SJed Brown 202179860b2SJed Brown def setType(self, key, value, forceLocal = 0): 203179860b2SJed Brown '''Checks for the key locally, and if not found consults the parent. Sets the type for this key. 204179860b2SJed Brown - If a value for the key already exists, it is converted to the new type''' 205179860b2SJed Brown if not isinstance(value, nargs.Arg): 206179860b2SJed Brown raise TypeError('An argument type must be a subclass of Arg') 207179860b2SJed Brown value.setKey(key) 2083b049ba2SJed Brown if forceLocal or self.parent is None or self.dict_has_key(key): 2093b049ba2SJed Brown if self.dict_has_key(key): 210179860b2SJed Brown v = dict.__getitem__(self, key) 211179860b2SJed Brown if v.isValueSet(): 212179860b2SJed Brown try: 213179860b2SJed Brown value.setValue(v.getValue()) 214a5737712SSatish Balay except TypeError: 2155b6bfdb9SJed Brown print(value.__class__.__name__[3:]) 2165b6bfdb9SJed Brown print('-----------------------------------------------------------------------') 2175b6bfdb9SJed Brown print('Warning! Incorrect argument type specified: -'+str(key)+'='+str(v.getValue())+' - expecting type '+value.__class__.__name__[3:]+'.') 2185b6bfdb9SJed Brown print('-----------------------------------------------------------------------') 219a5737712SSatish Balay pass 220179860b2SJed Brown dict.__setitem__(self, key, value) 22131f4da61SSatish Balay #self.save() 222179860b2SJed Brown else: 223179860b2SJed Brown return self.send(key, value) 224179860b2SJed Brown return 225179860b2SJed Brown 226179860b2SJed Brown def __setitem__(self, key, value): 227179860b2SJed Brown '''Checks for the key locally, and if not found consults the parent. Sets the value of the Arg.''' 2283b049ba2SJed Brown if not self.dict_has_key(key): 229179860b2SJed Brown if not self.parent is None: 230179860b2SJed Brown return self.send(key, value) 231179860b2SJed Brown else: 232179860b2SJed Brown dict.__setitem__(self, key, nargs.Arg(key)) 233179860b2SJed Brown dict.__getitem__(self, key).setValue(value) 234179860b2SJed Brown self.writeLogLine('__setitem__: Set value for '+key+' to '+str(dict.__getitem__(self, key))) 23531f4da61SSatish Balay #self.save() 236179860b2SJed Brown return 237179860b2SJed Brown 238179860b2SJed Brown def __delitem__(self, key): 239179860b2SJed Brown '''Checks for the key locally, and if not found consults the parent. Deletes the Arg completely.''' 2403b049ba2SJed Brown if self.dict_has_key(key): 241179860b2SJed Brown dict.__delitem__(self, key) 24231f4da61SSatish Balay #self.save() 243179860b2SJed Brown elif not self.parent is None: 244179860b2SJed Brown self.send(key) 245179860b2SJed Brown return 246179860b2SJed Brown 247179860b2SJed Brown def clear(self): 248179860b2SJed Brown '''Clears both the local and parent dictionaries''' 249179860b2SJed Brown if dict.__len__(self): 250179860b2SJed Brown dict.clear(self) 25131f4da61SSatish Balay #self.save() 252179860b2SJed Brown if not self.parent is None: 253179860b2SJed Brown self.send() 254179860b2SJed Brown return 255179860b2SJed Brown 256179860b2SJed Brown def __contains__(self, key): 257179860b2SJed Brown '''Checks for the key locally, and if not found consults the parent. Then checks whether the value has been set''' 2583b049ba2SJed Brown if self.dict_has_key(key): 259179860b2SJed Brown if dict.__getitem__(self, key).isValueSet(): 260179860b2SJed Brown self.writeLogLine('has_key: Have value for '+key) 261179860b2SJed Brown else: 262179860b2SJed Brown self.writeLogLine('has_key: Do not have value for '+key) 263179860b2SJed Brown return dict.__getitem__(self, key).isValueSet() 264179860b2SJed Brown elif not self.parent is None: 265179860b2SJed Brown return self.send(key) 266179860b2SJed Brown return 0 267179860b2SJed Brown 268179860b2SJed Brown def get(self, key, default=None): 2695b6bfdb9SJed Brown if key in self: 270179860b2SJed Brown return self.__getitem__(key) 271179860b2SJed Brown else: 272179860b2SJed Brown return default 273179860b2SJed Brown 274179860b2SJed Brown def hasType(self, key): 275179860b2SJed Brown '''Checks for the key locally, and if not found consults the parent. Then checks whether the type has been set''' 2763b049ba2SJed Brown if self.dict_has_key(key): 277179860b2SJed Brown return 1 278179860b2SJed Brown elif not self.parent is None: 279179860b2SJed Brown return self.send(key) 280179860b2SJed Brown return 0 281179860b2SJed Brown 282179860b2SJed Brown def items(self): 283179860b2SJed Brown '''Return a list of all accessible items, as (key, value) pairs.''' 284179860b2SJed Brown l = dict.items(self) 285179860b2SJed Brown if not self.parent is None: 286179860b2SJed Brown l.extend(self.send()) 287179860b2SJed Brown return l 288179860b2SJed Brown 289179860b2SJed Brown def localitems(self): 290179860b2SJed Brown '''Return a list of all the items stored locally, as (key, value) pairs.''' 291179860b2SJed Brown return dict.items(self) 292179860b2SJed Brown 293179860b2SJed Brown def keys(self): 294179860b2SJed Brown '''Returns the list of keys in both the local and parent dictionaries''' 295bb3dd2f6SJed Brown keyList = [key for key in dict.keys(self) if dict.__getitem__(self, key).isValueSet()] 296179860b2SJed Brown if not self.parent is None: 297179860b2SJed Brown keyList.extend(self.send()) 298179860b2SJed Brown return keyList 299179860b2SJed Brown 300179860b2SJed Brown def types(self): 301179860b2SJed Brown '''Returns the list of keys for which types are defined in both the local and parent dictionaries''' 302179860b2SJed Brown keyList = dict.keys(self) 303179860b2SJed Brown if not self.parent is None: 304179860b2SJed Brown keyList.extend(self.send()) 305179860b2SJed Brown return keyList 306179860b2SJed Brown 307179860b2SJed Brown def update(self, d): 308179860b2SJed Brown '''Update the dictionary with the contents of d''' 309179860b2SJed Brown for k in d: 310179860b2SJed Brown self[k] = d[k] 311179860b2SJed Brown return 312179860b2SJed Brown 313179860b2SJed Brown def updateTypes(self, d): 314179860b2SJed Brown '''Update types locally, which is equivalent to the dict.update() method''' 315179860b2SJed Brown return dict.update(self, d) 316179860b2SJed Brown 317179860b2SJed Brown def insertArg(self, key, value, arg): 318179860b2SJed Brown '''Insert a (key, value) pair into the dictionary. If key is None, arg is put into the target list.''' 319179860b2SJed Brown if not key is None: 320179860b2SJed Brown self[key] = value 321179860b2SJed Brown else: 322179860b2SJed Brown if not self.target == ['default']: 323179860b2SJed Brown self.target.append(arg) 324179860b2SJed Brown else: 325179860b2SJed Brown self.target = [arg] 326179860b2SJed Brown return 327179860b2SJed Brown 328179860b2SJed Brown def insertArgs(self, args): 329179860b2SJed Brown '''Insert some text arguments into the dictionary (list and dictionaries are recognized)''' 330179860b2SJed Brown 331179860b2SJed Brown if isinstance(args, list): 332179860b2SJed Brown for arg in args: 333179860b2SJed Brown (key, value) = nargs.Arg.parseArgument(arg) 334179860b2SJed Brown self.insertArg(key, value, arg) 3356998aabbSVaclav Hapla elif hasattr(args, 'keys'): 336179860b2SJed Brown for key in args.keys(): 337179860b2SJed Brown if isinstance(args[key], str): 338179860b2SJed Brown value = nargs.Arg.parseValue(args[key]) 339179860b2SJed Brown else: 340179860b2SJed Brown value = args[key] 341179860b2SJed Brown self.insertArg(key, value, None) 342179860b2SJed Brown elif isinstance(args, str): 343179860b2SJed Brown (key, value) = nargs.Arg.parseArgument(args) 344179860b2SJed Brown self.insertArg(key, value, args) 345179860b2SJed Brown return 346179860b2SJed Brown 347179860b2SJed Brown def hasParent(self): 348179860b2SJed Brown '''Return True if this RDict has a parent dictionary''' 349179860b2SJed Brown return not self.parent is None 350179860b2SJed Brown 351179860b2SJed Brown def getServerAddr(self, dir): 352179860b2SJed Brown '''Read the server socket address (in pickled form) from a file, usually RDict.loc 353179860b2SJed Brown - If we fail to connect to the server specified in the file, we spawn it using startServer()''' 354179860b2SJed Brown filename = os.path.join(dir, self.addrFilename) 355179860b2SJed Brown if not os.path.exists(filename): 356179860b2SJed Brown self.startServer(filename) 357179860b2SJed Brown if not os.path.exists(filename): 358179860b2SJed Brown raise RuntimeError('Server address file does not exist: '+filename) 359179860b2SJed Brown try: 360179860b2SJed Brown f = open(filename, 'r') 361492432c8SJed Brown addr = pickle.load(f) 362179860b2SJed Brown f.close() 363179860b2SJed Brown return addr 3645b6bfdb9SJed Brown except Exception as e: 365179860b2SJed Brown self.writeLogLine('CLIENT: Exception during server address determination: '+str(e.__class__)+': '+str(e)) 366179860b2SJed Brown raise RuntimeError('Could not get server address in '+filename) 367179860b2SJed Brown 368179860b2SJed Brown def writeServerAddr(self, server): 369179860b2SJed Brown '''Write the server socket address (in pickled form) to a file, usually RDict.loc.''' 370c6ef1b5bSJed Brown f = open(self.addrFilename, 'w') 371492432c8SJed Brown pickle.dump(server.server_address, f) 372179860b2SJed Brown f.close() 373179860b2SJed Brown self.writeLogLine('SERVER: Wrote lock file '+os.path.abspath(self.addrFilename)) 374179860b2SJed Brown return 375179860b2SJed Brown 376179860b2SJed Brown def startServer(self, addrFilename): 377179860b2SJed Brown '''Spawn a new RDict server in the parent directory''' 378179860b2SJed Brown import RDict # Need this to locate server script 379179860b2SJed Brown import sys 380179860b2SJed Brown import time 381becf0a19SJed Brown import sysconfig 382179860b2SJed Brown 383179860b2SJed Brown self.writeLogLine('CLIENT: Spawning a new server with lock file '+os.path.abspath(addrFilename)) 384179860b2SJed Brown if os.path.exists(addrFilename): 385179860b2SJed Brown os.remove(addrFilename) 386179860b2SJed Brown oldDir = os.getcwd() 387179860b2SJed Brown source = os.path.join(os.path.dirname(os.path.abspath(sys.modules['RDict'].__file__)), 'RDict.py') 388becf0a19SJed Brown interpreter = os.path.join(sysconfig.get_config_var('BINDIR'), sysconfig.get_config_var('PYTHON')) 389179860b2SJed Brown if not os.path.isfile(interpreter): 390179860b2SJed Brown interpreter = 'python' 391179860b2SJed Brown os.chdir(os.path.dirname(addrFilename)) 392179860b2SJed Brown self.writeLogLine('CLIENT: Executing '+interpreter+' '+source+' server"') 393179860b2SJed Brown try: 394179860b2SJed Brown os.spawnvp(os.P_NOWAIT, interpreter, [interpreter, source, 'server']) 395179860b2SJed Brown except: 396179860b2SJed Brown self.writeLogLine('CLIENT: os.spawnvp failed.\n \ 397179860b2SJed Brown This is a typical problem on CYGWIN systems. If you are using CYGWIN,\n \ 398179860b2SJed Brown you can fix this problem by running /bin/rebaseall. If you do not have\n \ 399179860b2SJed Brown this program, you can install it with the CYGWIN installer in the package\n \ 400179860b2SJed Brown Rebase, under the category System. You must run /bin/rebaseall after\n \ 401179860b2SJed Brown turning off all cygwin services -- in particular sshd, if any such services\n \ 402179860b2SJed Brown are running. For more information about rebase, go to http://www.cygwin.com') 4035b6bfdb9SJed Brown print('\n \ 404179860b2SJed Brown This is a typical problem on CYGWIN systems. If you are using CYGWIN,\n \ 405179860b2SJed Brown you can fix this problem by running /bin/rebaseall. If you do not have\n \ 406179860b2SJed Brown this program, you can install it with the CYGWIN installer in the package\n \ 407179860b2SJed Brown Rebase, under the category System. You must run /bin/rebaseall after\n \ 408179860b2SJed Brown turning off all cygwin services -- in particular sshd, if any such services\n \ 4095b6bfdb9SJed Brown are running. For more information about rebase, go to http://www.cygwin.com\n') 410179860b2SJed Brown raise 411179860b2SJed Brown os.chdir(oldDir) 412179860b2SJed Brown timeout = 1 413179860b2SJed Brown for i in range(10): 414179860b2SJed Brown time.sleep(timeout) 415179860b2SJed Brown timeout *= 2 416179860b2SJed Brown if timeout > 100: timeout = 100 417179860b2SJed Brown if os.path.exists(addrFilename): return 418179860b2SJed Brown self.writeLogLine('CLIENT: Could not start server') 419179860b2SJed Brown return 420179860b2SJed Brown 421179860b2SJed Brown def connectParent(self, addr, dir): 422179860b2SJed Brown '''Try to connect to a parent RDict server 423179860b2SJed Brown - If addr and dir are both None, this operation fails 424179860b2SJed Brown - If addr is None, check for an address file in dir''' 425179860b2SJed Brown if addr is None: 426179860b2SJed Brown if dir is None: return 0 427179860b2SJed Brown addr = self.getServerAddr(dir) 428179860b2SJed Brown 429179860b2SJed Brown import socket 430179860b2SJed Brown import errno 431179860b2SJed Brown connected = 0 432179860b2SJed Brown s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 433179860b2SJed Brown timeout = 1 434179860b2SJed Brown for i in range(10): 435179860b2SJed Brown try: 436179860b2SJed Brown self.writeLogLine('CLIENT: Trying to connect to '+str(addr)) 437179860b2SJed Brown s.connect(addr) 438179860b2SJed Brown connected = 1 439179860b2SJed Brown break 4405b6bfdb9SJed Brown except socket.error as e: 441179860b2SJed Brown self.writeLogLine('CLIENT: Failed to connect: '+str(e)) 442179860b2SJed Brown if e[0] == errno.ECONNREFUSED: 443179860b2SJed Brown try: 444179860b2SJed Brown import time 445179860b2SJed Brown time.sleep(timeout) 446179860b2SJed Brown timeout *= 2 447179860b2SJed Brown if timeout > 100: timeout = 100 448179860b2SJed Brown except KeyboardInterrupt: 449179860b2SJed Brown break 450179860b2SJed Brown # Try to spawn parent 451179860b2SJed Brown if dir: 452179860b2SJed Brown filename = os.path.join(dir, self.addrFilename) 453179860b2SJed Brown if os.path.isfile(filename): 454179860b2SJed Brown os.remove(filename) 455179860b2SJed Brown self.startServer(filename) 4565b6bfdb9SJed Brown except Exception as e: 457179860b2SJed Brown self.writeLogLine('CLIENT: Failed to connect: '+str(e.__class__)+': '+str(e)) 458179860b2SJed Brown if not connected: 459179860b2SJed Brown self.writeLogLine('CLIENT: Failed to connect to parent') 460179860b2SJed Brown return 0 461179860b2SJed Brown self.parent = s 462179860b2SJed Brown self.writeLogLine('CLIENT: Connected to '+str(self.parent)) 463179860b2SJed Brown return 1 464179860b2SJed Brown 465179860b2SJed Brown def sendPacket(self, s, packet, source = 'Unknown', isPickled = 0): 466179860b2SJed Brown '''Pickle the input packet. Send first the size of the pickled string in 32-bit integer, and then the string itself''' 467179860b2SJed Brown self.writeLogLine(source+': Sending packet '+str(packet)) 468179860b2SJed Brown if isPickled: 469179860b2SJed Brown p = packet 470179860b2SJed Brown else: 471492432c8SJed Brown p = pickle.dumps(packet) 472179860b2SJed Brown if hasattr(s, 'write'): 473179860b2SJed Brown s.write(p) 474179860b2SJed Brown else: 475179860b2SJed Brown s.sendall(p) 476179860b2SJed Brown self.writeLogLine(source+': Sent packet') 477179860b2SJed Brown return 478179860b2SJed Brown 479179860b2SJed Brown def recvPacket(self, s, source = 'Unknown'): 480179860b2SJed Brown '''Receive first the size of the pickled string in a 32-bit integer, and then the string itself. Return the unpickled object''' 481179860b2SJed Brown self.writeLogLine(source+': Receiving packet') 482179860b2SJed Brown if hasattr(s, 'read'): 483179860b2SJed Brown s.read(4) 484492432c8SJed Brown value = pickle.load(s) 485179860b2SJed Brown else: 486179860b2SJed Brown objString = '' 487179860b2SJed Brown while len(objString) < length: 488179860b2SJed Brown objString += s.recv(length - len(objString)) 489492432c8SJed Brown value = pickle.loads(objString) 490179860b2SJed Brown self.writeLogLine(source+': Received packet '+str(value)) 491179860b2SJed Brown return value 492179860b2SJed Brown 493179860b2SJed Brown def send(self, key = None, value = None, operation = None): 494179860b2SJed Brown '''Send a request to the parent''' 495179860b2SJed Brown import inspect 496179860b2SJed Brown 497179860b2SJed Brown objString = '' 498179860b2SJed Brown for i in range(3): 499179860b2SJed Brown try: 500179860b2SJed Brown packet = [] 501179860b2SJed Brown if operation is None: 502179860b2SJed Brown operation = inspect.stack()[1][3] 503179860b2SJed Brown packet.append(operation) 504179860b2SJed Brown if not key is None: 505179860b2SJed Brown packet.append(key) 506179860b2SJed Brown if not value is None: 507179860b2SJed Brown packet.append(value) 508179860b2SJed Brown self.sendPacket(self.parent, tuple(packet), source = 'CLIENT') 509179860b2SJed Brown response = self.recvPacket(self.parent, source = 'CLIENT') 510179860b2SJed Brown break 5115b6bfdb9SJed Brown except IOError as e: 512179860b2SJed Brown self.writeLogLine('CLIENT: IOError '+str(e)) 513179860b2SJed Brown if e.errno == 32: 514179860b2SJed Brown self.connectParent(self.parentAddr, self.parentDirectory) 5155b6bfdb9SJed Brown except Exception as e: 516179860b2SJed Brown self.writeLogLine('CLIENT: Exception '+str(e)+' '+str(e.__class__)) 517179860b2SJed Brown try: 518179860b2SJed Brown if isinstance(response, Exception): 519179860b2SJed Brown self.writeLogLine('CLIENT: Got an exception '+str(response)) 520179860b2SJed Brown raise response 521179860b2SJed Brown else: 522179860b2SJed Brown self.writeLogLine('CLIENT: Received value '+str(response)+' '+str(type(response))) 523179860b2SJed Brown except UnboundLocalError: 524179860b2SJed Brown self.writeLogLine('CLIENT: Could not unpickle response') 525179860b2SJed Brown response = None 526179860b2SJed Brown return response 527179860b2SJed Brown 528179860b2SJed Brown def serve(self): 529179860b2SJed Brown '''Start a server''' 530179860b2SJed Brown import socket 5318f450857SSatish Balay import SocketServer # novermin 532179860b2SJed Brown 533179860b2SJed Brown if not useThreads: 534179860b2SJed Brown raise RuntimeError('Cannot run a server if threads are disabled') 535179860b2SJed Brown 536179860b2SJed Brown class ProcessHandler(SocketServer.StreamRequestHandler): 537179860b2SJed Brown def handle(self): 538179860b2SJed Brown import time 539179860b2SJed Brown 540179860b2SJed Brown self.server.rdict.lastAccess = time.time() 541179860b2SJed Brown self.server.rdict.writeLogLine('SERVER: Started new handler') 542179860b2SJed Brown while 1: 543179860b2SJed Brown try: 544179860b2SJed Brown value = self.server.rdict.recvPacket(self.rfile, source = 'SERVER') 5455b6bfdb9SJed Brown except EOFError as e: 546179860b2SJed Brown self.server.rdict.writeLogLine('SERVER: EOFError receiving packet '+str(e)+' '+str(e.__class__)) 547179860b2SJed Brown return 5485b6bfdb9SJed Brown except Exception as e: 549179860b2SJed Brown self.server.rdict.writeLogLine('SERVER: Error receiving packet '+str(e)+' '+str(e.__class__)) 550179860b2SJed Brown self.server.rdict.sendPacket(self.wfile, e, source = 'SERVER') 551179860b2SJed Brown continue 552179860b2SJed Brown if value[0] == 'stop': break 553179860b2SJed Brown try: 554179860b2SJed Brown response = getattr(self.server.rdict, value[0])(*value[1:]) 5555b6bfdb9SJed Brown except Exception as e: 556179860b2SJed Brown self.server.rdict.writeLogLine('SERVER: Error executing operation '+str(e)+' '+str(e.__class__)) 557179860b2SJed Brown self.server.rdict.sendPacket(self.wfile, e, source = 'SERVER') 558179860b2SJed Brown else: 559179860b2SJed Brown self.server.rdict.sendPacket(self.wfile, response, source = 'SERVER') 560179860b2SJed Brown return 561179860b2SJed Brown 562179860b2SJed Brown # check if server is running 563179860b2SJed Brown if os.path.exists(self.addrFilename): 564179860b2SJed Brown rdict = RDict(parentDirectory = '.') 565179860b2SJed Brown hasParent = rdict.hasParent() 566179860b2SJed Brown del rdict 567179860b2SJed Brown if hasParent: 568179860b2SJed Brown self.writeLogLine('SERVER: Another server is already running') 569179860b2SJed Brown raise RuntimeError('Server already running') 570179860b2SJed Brown 571179860b2SJed Brown # Daemonize server 572179860b2SJed Brown self.writeLogLine('SERVER: Daemonizing server') 573179860b2SJed Brown if os.fork(): # Launch child 574179860b2SJed Brown os._exit(0) # Kill off parent, so we are not a process group leader and get a new PID 575179860b2SJed Brown os.setsid() # Set session ID, so that we have no controlling terminal 576179860b2SJed Brown # We choose to leave cwd at RDict.py: os.chdir('/') # Make sure root directory is not on a mounted drive 5775b6bfdb9SJed Brown os.umask(0o77) # Fix creation mask 578179860b2SJed Brown for i in range(3): # Crappy stopgap for closing descriptors 579179860b2SJed Brown try: 580179860b2SJed Brown os.close(i) 5815b6bfdb9SJed Brown except OSError as e: 582179860b2SJed Brown if e.errno != errno.EBADF: 583179860b2SJed Brown raise RuntimeError('Could not close default descriptor '+str(i)) 584179860b2SJed Brown 585179860b2SJed Brown # wish there was a better way to get a usable socket 586179860b2SJed Brown self.writeLogLine('SERVER: Establishing socket server') 587179860b2SJed Brown basePort = 8000 588179860b2SJed Brown flag = 'nosocket' 589179860b2SJed Brown p = 1 590179860b2SJed Brown while p < 1000 and flag == 'nosocket': 591179860b2SJed Brown try: 592179860b2SJed Brown server = SocketServer.ThreadingTCPServer((socket.gethostname(), basePort+p), ProcessHandler) 593179860b2SJed Brown flag = 'socket' 5945b6bfdb9SJed Brown except Exception as e: 595179860b2SJed Brown p = p + 1 596179860b2SJed Brown if flag == 'nosocket': 597179860b2SJed Brown p = 1 598179860b2SJed Brown while p < 1000 and flag == 'nosocket': 599179860b2SJed Brown try: 600179860b2SJed Brown server = SocketServer.ThreadingTCPServer(('localhost', basePort+p), ProcessHandler) 601179860b2SJed Brown flag = 'socket' 6025b6bfdb9SJed Brown except Exception as e: 603179860b2SJed Brown p = p + 1 604179860b2SJed Brown if flag == 'nosocket': 605179860b2SJed Brown self.writeLogLine('SERVER: Could not established socket server on port '+str(basePort+p)) 6065b6bfdb9SJed Brown raise RuntimeError('Cannot get available socket') 607179860b2SJed Brown self.writeLogLine('SERVER: Established socket server on port '+str(basePort+p)) 608179860b2SJed Brown 609179860b2SJed Brown self.isServer = 1 610179860b2SJed Brown self.writeServerAddr(server) 611179860b2SJed Brown self.serverShutdown(os.getpid()) 612179860b2SJed Brown 613179860b2SJed Brown server.rdict = self 614179860b2SJed Brown self.writeLogLine('SERVER: Started server') 615179860b2SJed Brown server.serve_forever() 616179860b2SJed Brown return 617179860b2SJed Brown 618179860b2SJed Brown def load(self): 619179860b2SJed Brown '''Load the saved dictionary''' 620179860b2SJed Brown if not self.parentDirectory is None and os.path.samefile(os.getcwd(), self.parentDirectory): 621179860b2SJed Brown return 622179860b2SJed Brown self.saveFilename = os.path.abspath(self.saveFilename) 623179860b2SJed Brown if os.path.exists(self.saveFilename): 624179860b2SJed Brown try: 625d2acaa84SFrancesco Ballarin dbFile = open(self.saveFilename, 'rb') 626492432c8SJed Brown data = pickle.load(dbFile) 627179860b2SJed Brown self.updateTypes(data) 628179860b2SJed Brown dbFile.close() 629179860b2SJed Brown self.writeLogLine('Loaded dictionary from '+self.saveFilename) 6305b6bfdb9SJed Brown except Exception as e: 631179860b2SJed Brown self.writeLogLine('Problem loading dictionary from '+self.saveFilename+'\n--> '+str(e)) 632179860b2SJed Brown else: 633179860b2SJed Brown self.writeLogLine('No dictionary to load in this file: '+self.saveFilename) 634179860b2SJed Brown return 635179860b2SJed Brown 636603e7a67SSatish Balay def save(self, force = 1): 637179860b2SJed Brown '''Save the dictionary after 5 seconds, ignoring all subsequent calls until the save 638179860b2SJed Brown - Giving force = True will cause an immediate save''' 639179860b2SJed Brown if self.readonly: return 64008eb64ffSMatthew G. Knepley if force: 641179860b2SJed Brown self.saveTimer = None 642179860b2SJed Brown # This should be a critical section 643b8b3d021SJed Brown dbFile = open(self.saveFilename, 'wb') 644bb3dd2f6SJed Brown data = dict([i for i in self.localitems() if not i[1].getTemporary()]) 645492432c8SJed Brown pickle.dump(data, dbFile) 646179860b2SJed Brown dbFile.close() 647179860b2SJed Brown self.writeLogLine('Saved local dictionary to '+os.path.abspath(self.saveFilename)) 648179860b2SJed Brown elif not self.saveTimer: 649179860b2SJed Brown import threading 650179860b2SJed Brown self.saveTimer = threading.Timer(5, self.save, [], {'force': 1}) 651179860b2SJed Brown self.saveTimer.setDaemon(1) 652179860b2SJed Brown self.saveTimer.start() 653179860b2SJed Brown return 654179860b2SJed Brown 655179860b2SJed Brown def shutdown(self): 656179860b2SJed Brown '''Shutdown the dictionary, writing out changes and notifying parent''' 657179860b2SJed Brown if self.saveTimer: 658179860b2SJed Brown self.saveTimer.cancel() 659179860b2SJed Brown self.save(force = 1) 660179860b2SJed Brown if self.isServer and os.path.isfile(self.addrFilename): 661179860b2SJed Brown os.remove(self.addrFilename) 662179860b2SJed Brown if not self.parent is None: 663179860b2SJed Brown self.sendPacket(self.parent, self.stopCmd, isPickled = 1) 664179860b2SJed Brown self.parent.close() 665179860b2SJed Brown self.parent = None 666179860b2SJed Brown self.writeLogLine('Shutting down') 667179860b2SJed Brown self.logFile.close() 668179860b2SJed Brown return 669179860b2SJed Brown 670179860b2SJed Brown def serverShutdown(self, pid, delay = shutdownDelay): 671179860b2SJed Brown if self.shutdownTimer is None: 672179860b2SJed Brown import threading 673179860b2SJed Brown 674179860b2SJed Brown self.shutdownTimer = threading.Timer(delay, self.serverShutdown, [pid], {'delay': 0}) 675179860b2SJed Brown self.shutdownTimer.setDaemon(1) 676179860b2SJed Brown self.shutdownTimer.start() 677179860b2SJed Brown self.writeLogLine('SERVER: Set shutdown timer for process '+str(pid)+' at '+str(delay)+' seconds') 678179860b2SJed Brown else: 679179860b2SJed Brown try: 680179860b2SJed Brown import signal 681179860b2SJed Brown import time 682179860b2SJed Brown 683179860b2SJed Brown idleTime = time.time() - self.lastAccess 684179860b2SJed Brown self.writeLogLine('SERVER: Last access '+str(self.lastAccess)) 685179860b2SJed Brown self.writeLogLine('SERVER: Idle time '+str(idleTime)) 686179860b2SJed Brown if idleTime < RDict.shutdownDelay: 687179860b2SJed Brown self.writeLogLine('SERVER: Extending shutdown timer for '+str(pid)+' by '+str(RDict.shutdownDelay - idleTime)+' seconds') 688179860b2SJed Brown self.shutdownTimer = None 689179860b2SJed Brown self.serverShutdown(pid, RDict.shutdownDelay - idleTime) 690179860b2SJed Brown else: 691179860b2SJed Brown self.writeLogLine('SERVER: Killing server '+str(pid)) 692179860b2SJed Brown os.kill(pid, signal.SIGTERM) 6935b6bfdb9SJed Brown except Exception as e: 694179860b2SJed Brown self.writeLogLine('SERVER: Exception killing server: '+str(e)) 695179860b2SJed Brown return 696179860b2SJed Brown 697179860b2SJed Brownif __name__ == '__main__': 698179860b2SJed Brown import sys 699179860b2SJed Brown try: 700179860b2SJed Brown if len(sys.argv) < 2: 7015b6bfdb9SJed Brown print('RDict.py [server | client | clear | insert | remove] [parent]') 702179860b2SJed Brown else: 703179860b2SJed Brown action = sys.argv[1] 704179860b2SJed Brown parent = None 705179860b2SJed Brown if len(sys.argv) > 2: 706179860b2SJed Brown if not sys.argv[2] == 'None': parent = sys.argv[2] 707179860b2SJed Brown if action == 'server': 708179860b2SJed Brown RDict(parentDirectory = parent).serve() 709179860b2SJed Brown elif action == 'client': 7105b6bfdb9SJed Brown print('Entries in server dictionary') 711179860b2SJed Brown rdict = RDict(parentDirectory = parent) 712179860b2SJed Brown for key in rdict.types(): 713179860b2SJed Brown if not key.startswith('cacheKey') and not key.startswith('stamp-'): 7145b6bfdb9SJed Brown print(str(key)+' '+str(rdict.getType(key))) 715179860b2SJed Brown elif action == 'cacheClient': 7165b6bfdb9SJed Brown print('Cache entries in server dictionary') 717179860b2SJed Brown rdict = RDict(parentDirectory = parent) 718179860b2SJed Brown for key in rdict.types(): 719179860b2SJed Brown if key.startswith('cacheKey'): 7205b6bfdb9SJed Brown print(str(key)+' '+str(rdict.getType(key))) 721179860b2SJed Brown elif action == 'stampClient': 7225b6bfdb9SJed Brown print('Stamp entries in server dictionary') 723179860b2SJed Brown rdict = RDict(parentDirectory = parent) 724179860b2SJed Brown for key in rdict.types(): 725179860b2SJed Brown if key.startswith('stamp-'): 7265b6bfdb9SJed Brown print(str(key)+' '+str(rdict.getType(key))) 727179860b2SJed Brown elif action == 'clear': 7285b6bfdb9SJed Brown print('Clearing all dictionaries') 729179860b2SJed Brown RDict(parentDirectory = parent).clear() 730179860b2SJed Brown elif action == 'insert': 731179860b2SJed Brown rdict = RDict(parentDirectory = parent) 732179860b2SJed Brown rdict[sys.argv[3]] = sys.argv[4] 733179860b2SJed Brown elif action == 'remove': 734179860b2SJed Brown rdict = RDict(parentDirectory = parent) 735179860b2SJed Brown del rdict[sys.argv[3]] 736179860b2SJed Brown else: 737179860b2SJed Brown sys.exit('Unknown action: '+action) 7385b6bfdb9SJed Brown except Exception as e: 739179860b2SJed Brown import traceback 7405b6bfdb9SJed Brown print(traceback.print_tb(sys.exc_info()[2])) 741179860b2SJed Brown sys.exit(str(e)) 742179860b2SJed Brown sys.exit(0) 743