1#!/usr/bin/env python 2'''A remote dictionary server 3 4 RDict is a typed, hierarchical, persistent dictionary intended to manage 5 all arguments or options for a program. The interface remains exactly the 6 same as dict, but the storage is more complicated. 7 8 Argument typing is handled by wrapping all values stored in the dictionary 9 with nargs.Arg or a subclass. A user can call setType() to set the type of 10 an argument without any value being present. Whenever __getitem__() or 11 __setitem__() is called, values are extracted or replaced in the wrapper. 12 These wrappers can be accessed directly using getType(), setType(), and 13 types(). 14 15 Hierarchy is allowed using a single "parent" dictionary. All operations 16 cascade to the parent. For instance, the length of the dictionary is the 17 number of local keys plus the number of keys in the parent, and its 18 parent, etc. Also, a dictionary need not have a parent. If a key does not 19 appear in the local dicitonary, the call if passed to the parent. However, 20 in this case we see that local keys can shadow those in a parent. 21 Communication with the parent is handled using sockets, with the parent 22 being a server and the interactive dictionary a client. 23 24 The default persistence mechanism is a pickle file, RDict.db, written 25 whenever an argument is changed locally. A timer thread is created after 26 an initial change, so that many rapid changes do not cause many writes. 27 Each dictionary only saves its local entries, so all parents also 28 separately save data in different RDict.db files. Each time a dictionary 29 is created, the current directory is searched for an RDict.db file, and 30 if found the contents are loaded into the dictionary. 31 32 This script also provides some default actions: 33 34 - server [parent] 35 Starts a server in the current directory with an optional parent. This 36 server will accept socket connections from other dictionaries and act 37 as a parent. 38 39 - client [parent] 40 Creates a dictionary in the current directory with an optional parent 41 and lists the contents. Notice that the contents may come from either 42 an RDict.db file in the current directory, or from the parent. 43 44 - clear [parent] 45 Creates a dictionary in the current directory with an optional parent 46 and clears the contents. Notice that this will also clear the parent. 47 48 - insert <parent> <key> <value> 49 Creates a dictionary in the current directory with a parent, and inserts 50 the key-value pair. If "parent" is "None", no parent is assigned. 51 52 - remove <parent> <key> 53 Creates a dictionary in the current directory with a parent, and removes 54 the given key. If "parent" is "None", no parent is assigned. 55''' 56from __future__ import print_function 57from __future__ import absolute_import 58try: 59 import project # This is necessary for us to create Project objects on load 60 import build.buildGraph # This is necessary for us to create BuildGraph objects on load 61except ImportError: 62 pass 63import nargs 64 65import cPickle 66import os 67import sys 68useThreads = nargs.Arg.findArgument('useThreads', sys.argv[1:]) 69if useThreads is None: 70 useThreads = 0 # workaround issue with parallel configure 71elif useThreads == 'no' or useThreads == '0': 72 useThreads = 0 73elif useThreads == 'yes' or useThreads == '1': 74 useThreads = 1 75else: 76 raise RuntimeError('Unknown option value for --useThreads ',useThreads) 77 78class RDict(dict): 79 '''An RDict is a typed dictionary, which may be hierarchically composed. All elements derive from the 80Arg class, which wraps the usual value.''' 81 # The server will self-shutdown after this many seconds 82 shutdownDelay = 60*60*5 83 84 def __init__(self, parentAddr = None, parentDirectory = None, load = 1, autoShutdown = 1, readonly = False): 85 import atexit 86 import time 87 import xdrlib 88 89 self.logFile = None 90 self.setupLogFile() 91 self.target = ['default'] 92 self.parent = None 93 self.saveTimer = None 94 self.shutdownTimer = None 95 self.lastAccess = time.time() 96 self.saveFilename = 'RDict.db' 97 self.addrFilename = 'RDict.loc' 98 self.parentAddr = parentAddr 99 self.isServer = 0 100 self.readonly = readonly 101 self.parentDirectory = parentDirectory 102 self.packer = xdrlib.Packer() 103 self.unpacker = xdrlib.Unpacker('') 104 self.stopCmd = cPickle.dumps(('stop',)) 105 self.writeLogLine('Greetings') 106 self.connectParent(self.parentAddr, self.parentDirectory) 107 if load: self.load() 108 if autoShutdown and useThreads: 109 atexit.register(self.shutdown) 110 self.writeLogLine('SERVER: Last access '+str(self.lastAccess)) 111 return 112 113 def __getstate__(self): 114 '''Remove any parent socket object, the XDR translators, and the log file from the dictionary before pickling''' 115 self.writeLogLine('Pickling RDict') 116 d = self.__dict__.copy() 117 if 'parent' in d: del d['parent'] 118 if 'saveTimer' in d: del d['saveTimer'] 119 if '_setCommandLine' in d: del d['_setCommandLine'] 120 del d['packer'] 121 del d['unpacker'] 122 del d['logFile'] 123 return d 124 125 def __setstate__(self, d): 126 '''Reconnect the parent socket object, recreate the XDR translators and reopen the log file after unpickling''' 127 self.logFile = file('RDict.log', 'a') 128 self.writeLogLine('Unpickling RDict') 129 self.__dict__.update(d) 130 import xdrlib 131 self.packer = xdrlib.Packer() 132 self.unpacker = xdrlib.Unpacker('') 133 self.connectParent(self.parentAddr, self.parentDirectory) 134 return 135 136 def setupLogFile(self, filename = 'RDict.log'): 137 if not self.logFile is None: 138 self.logFile.close() 139 if os.path.isfile(filename) and os.stat(filename).st_size > 10*1024*1024: 140 if os.path.isfile(filename+'.bkp'): 141 os.remove(filename+'.bkp') 142 os.rename(filename, filename+'.bkp') 143 self.logFile = file(filename, 'w') 144 else: 145 self.logFile = file(filename, 'a') 146 return 147 148 def writeLogLine(self, message): 149 '''Writes the message to the log along with the current time''' 150 import time 151 self.logFile.write('('+str(os.getpid())+')('+str(id(self))+')'+message+' ['+time.asctime(time.localtime())+']\n') 152 self.logFile.flush() 153 return 154 155 def __len__(self): 156 '''Returns the length of both the local and parent dictionaries''' 157 length = dict.__len__(self) 158 if not self.parent is None: 159 length = length + self.send() 160 return length 161 162 def getType(self, key): 163 '''Checks for the key locally, and if not found consults the parent. Returns the Arg object or None if not found.''' 164 if dict.has_key(self, key): 165 self.writeLogLine('getType: Getting local type for '+key+' '+str(dict.__getitem__(self, key))) 166 return dict.__getitem__(self, key) 167 elif not self.parent is None: 168 return self.send(key) 169 return None 170 171 def __getitem__(self, key): 172 '''Checks for the key locally, and if not found consults the parent. Returns the value of the Arg. 173 - If the value has not been set, the user will be prompted for input''' 174 if dict.has_key(self, key): 175 self.writeLogLine('__getitem__: '+key+' has local type') 176 pass 177 elif not self.parent is None: 178 self.writeLogLine('__getitem__: Checking parent value') 179 if self.send(key, operation = 'has_key'): 180 self.writeLogLine('__getitem__: Parent has value') 181 return self.send(key) 182 else: 183 self.writeLogLine('__getitem__: Checking parent type') 184 arg = self.send(key, operation = 'getType') 185 if not arg: 186 self.writeLogLine('__getitem__: Parent has no type') 187 arg = nargs.Arg(key) 188 try: 189 value = arg.getValue() 190 except AttributeError as e: 191 self.writeLogLine('__getitem__: Parent had invalid entry: '+str(e)) 192 arg = nargs.Arg(key) 193 value = arg.getValue() 194 self.writeLogLine('__getitem__: Setting parent value '+str(value)) 195 self.send(key, value, operation = '__setitem__') 196 return value 197 else: 198 self.writeLogLine('__getitem__: Setting local type for '+key) 199 dict.__setitem__(self, key, nargs.Arg(key)) 200 #self.save() 201 self.writeLogLine('__getitem__: Setting local value for '+key) 202 return dict.__getitem__(self, key).getValue() 203 204 def setType(self, key, value, forceLocal = 0): 205 '''Checks for the key locally, and if not found consults the parent. Sets the type for this key. 206 - If a value for the key already exists, it is converted to the new type''' 207 if not isinstance(value, nargs.Arg): 208 raise TypeError('An argument type must be a subclass of Arg') 209 value.setKey(key) 210 if forceLocal or self.parent is None or dict.has_key(self, key): 211 if dict.has_key(self, key): 212 v = dict.__getitem__(self, key) 213 if v.isValueSet(): 214 try: 215 value.setValue(v.getValue()) 216 except TypeError: 217 print(value.__class__.__name__[3:]) 218 print('-----------------------------------------------------------------------') 219 print('Warning! Incorrect argument type specified: -'+str(key)+'='+str(v.getValue())+' - expecting type '+value.__class__.__name__[3:]+'.') 220 print('-----------------------------------------------------------------------') 221 pass 222 dict.__setitem__(self, key, value) 223 #self.save() 224 else: 225 return self.send(key, value) 226 return 227 228 def __setitem__(self, key, value): 229 '''Checks for the key locally, and if not found consults the parent. Sets the value of the Arg.''' 230 if not dict.has_key(self, key): 231 if not self.parent is None: 232 return self.send(key, value) 233 else: 234 dict.__setitem__(self, key, nargs.Arg(key)) 235 dict.__getitem__(self, key).setValue(value) 236 self.writeLogLine('__setitem__: Set value for '+key+' to '+str(dict.__getitem__(self, key))) 237 #self.save() 238 return 239 240 def __delitem__(self, key): 241 '''Checks for the key locally, and if not found consults the parent. Deletes the Arg completely.''' 242 if dict.has_key(self, key): 243 dict.__delitem__(self, key) 244 #self.save() 245 elif not self.parent is None: 246 self.send(key) 247 return 248 249 def clear(self): 250 '''Clears both the local and parent dictionaries''' 251 if dict.__len__(self): 252 dict.clear(self) 253 #self.save() 254 if not self.parent is None: 255 self.send() 256 return 257 258 def __contains__(self, key): 259 '''Checks for the key locally, and if not found consults the parent. Then checks whether the value has been set''' 260 if dict.has_key(self, key): 261 if dict.__getitem__(self, key).isValueSet(): 262 self.writeLogLine('has_key: Have value for '+key) 263 else: 264 self.writeLogLine('has_key: Do not have value for '+key) 265 return dict.__getitem__(self, key).isValueSet() 266 elif not self.parent is None: 267 return self.send(key) 268 return 0 269 270 def get(self, key, default=None): 271 if key in self: 272 return self.__getitem__(key) 273 else: 274 return default 275 276 def hasType(self, key): 277 '''Checks for the key locally, and if not found consults the parent. Then checks whether the type has been set''' 278 if dict.has_key(self, key): 279 return 1 280 elif not self.parent is None: 281 return self.send(key) 282 return 0 283 284 def items(self): 285 '''Return a list of all accessible items, as (key, value) pairs.''' 286 l = dict.items(self) 287 if not self.parent is None: 288 l.extend(self.send()) 289 return l 290 291 def localitems(self): 292 '''Return a list of all the items stored locally, as (key, value) pairs.''' 293 return dict.items(self) 294 295 def keys(self): 296 '''Returns the list of keys in both the local and parent dictionaries''' 297 keyList = filter(lambda key: dict.__getitem__(self, key).isValueSet(), dict.keys(self)) 298 if not self.parent is None: 299 keyList.extend(self.send()) 300 return keyList 301 302 def types(self): 303 '''Returns the list of keys for which types are defined in both the local and parent dictionaries''' 304 keyList = dict.keys(self) 305 if not self.parent is None: 306 keyList.extend(self.send()) 307 return keyList 308 309 def update(self, d): 310 '''Update the dictionary with the contents of d''' 311 for k in d: 312 self[k] = d[k] 313 return 314 315 def updateTypes(self, d): 316 '''Update types locally, which is equivalent to the dict.update() method''' 317 return dict.update(self, d) 318 319 def insertArg(self, key, value, arg): 320 '''Insert a (key, value) pair into the dictionary. If key is None, arg is put into the target list.''' 321 if not key is None: 322 self[key] = value 323 else: 324 if not self.target == ['default']: 325 self.target.append(arg) 326 else: 327 self.target = [arg] 328 return 329 330 def insertArgs(self, args): 331 '''Insert some text arguments into the dictionary (list and dictionaries are recognized)''' 332 import UserDict 333 334 if isinstance(args, list): 335 for arg in args: 336 (key, value) = nargs.Arg.parseArgument(arg) 337 self.insertArg(key, value, arg) 338 # Necessary since os.environ is a UserDict 339 elif isinstance(args, dict) or isinstance(args, UserDict.UserDict): 340 for key in args.keys(): 341 if isinstance(args[key], str): 342 value = nargs.Arg.parseValue(args[key]) 343 else: 344 value = args[key] 345 self.insertArg(key, value, None) 346 elif isinstance(args, str): 347 (key, value) = nargs.Arg.parseArgument(args) 348 self.insertArg(key, value, args) 349 return 350 351 def hasParent(self): 352 '''Return True if this RDict has a parent dictionary''' 353 return not self.parent is None 354 355 def getServerAddr(self, dir): 356 '''Read the server socket address (in pickled form) from a file, usually RDict.loc 357 - If we fail to connect to the server specified in the file, we spawn it using startServer()''' 358 filename = os.path.join(dir, self.addrFilename) 359 if not os.path.exists(filename): 360 self.startServer(filename) 361 if not os.path.exists(filename): 362 raise RuntimeError('Server address file does not exist: '+filename) 363 try: 364 f = open(filename, 'r') 365 addr = cPickle.load(f) 366 f.close() 367 return addr 368 except Exception as e: 369 self.writeLogLine('CLIENT: Exception during server address determination: '+str(e.__class__)+': '+str(e)) 370 raise RuntimeError('Could not get server address in '+filename) 371 372 def writeServerAddr(self, server): 373 '''Write the server socket address (in pickled form) to a file, usually RDict.loc.''' 374 f = file(self.addrFilename, 'w') 375 cPickle.dump(server.server_address, f) 376 f.close() 377 self.writeLogLine('SERVER: Wrote lock file '+os.path.abspath(self.addrFilename)) 378 return 379 380 def startServer(self, addrFilename): 381 '''Spawn a new RDict server in the parent directory''' 382 import RDict # Need this to locate server script 383 import sys 384 import time 385 import distutils.sysconfig 386 387 self.writeLogLine('CLIENT: Spawning a new server with lock file '+os.path.abspath(addrFilename)) 388 if os.path.exists(addrFilename): 389 os.remove(addrFilename) 390 oldDir = os.getcwd() 391 source = os.path.join(os.path.dirname(os.path.abspath(sys.modules['RDict'].__file__)), 'RDict.py') 392 interpreter = os.path.join(distutils.sysconfig.get_config_var('BINDIR'), distutils.sysconfig.get_config_var('PYTHON')) 393 if not os.path.isfile(interpreter): 394 interpreter = 'python' 395 os.chdir(os.path.dirname(addrFilename)) 396 self.writeLogLine('CLIENT: Executing '+interpreter+' '+source+' server"') 397 try: 398 os.spawnvp(os.P_NOWAIT, interpreter, [interpreter, source, 'server']) 399 except: 400 self.writeLogLine('CLIENT: os.spawnvp failed.\n \ 401 This is a typical problem on CYGWIN systems. If you are using CYGWIN,\n \ 402 you can fix this problem by running /bin/rebaseall. If you do not have\n \ 403 this program, you can install it with the CYGWIN installer in the package\n \ 404 Rebase, under the category System. You must run /bin/rebaseall after\n \ 405 turning off all cygwin services -- in particular sshd, if any such services\n \ 406 are running. For more information about rebase, go to http://www.cygwin.com') 407 print('\n \ 408 This is a typical problem on CYGWIN systems. If you are using CYGWIN,\n \ 409 you can fix this problem by running /bin/rebaseall. If you do not have\n \ 410 this program, you can install it with the CYGWIN installer in the package\n \ 411 Rebase, under the category System. You must run /bin/rebaseall after\n \ 412 turning off all cygwin services -- in particular sshd, if any such services\n \ 413 are running. For more information about rebase, go to http://www.cygwin.com\n') 414 raise 415 os.chdir(oldDir) 416 timeout = 1 417 for i in range(10): 418 time.sleep(timeout) 419 timeout *= 2 420 if timeout > 100: timeout = 100 421 if os.path.exists(addrFilename): return 422 self.writeLogLine('CLIENT: Could not start server') 423 return 424 425 def connectParent(self, addr, dir): 426 '''Try to connect to a parent RDict server 427 - If addr and dir are both None, this operation fails 428 - If addr is None, check for an address file in dir''' 429 if addr is None: 430 if dir is None: return 0 431 addr = self.getServerAddr(dir) 432 433 import socket 434 import errno 435 connected = 0 436 s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 437 timeout = 1 438 for i in range(10): 439 try: 440 self.writeLogLine('CLIENT: Trying to connect to '+str(addr)) 441 s.connect(addr) 442 connected = 1 443 break 444 except socket.error as e: 445 self.writeLogLine('CLIENT: Failed to connect: '+str(e)) 446 if e[0] == errno.ECONNREFUSED: 447 try: 448 import time 449 time.sleep(timeout) 450 timeout *= 2 451 if timeout > 100: timeout = 100 452 except KeyboardInterrupt: 453 break 454 # Try to spawn parent 455 if dir: 456 filename = os.path.join(dir, self.addrFilename) 457 if os.path.isfile(filename): 458 os.remove(filename) 459 self.startServer(filename) 460 except Exception as e: 461 self.writeLogLine('CLIENT: Failed to connect: '+str(e.__class__)+': '+str(e)) 462 if not connected: 463 self.writeLogLine('CLIENT: Failed to connect to parent') 464 return 0 465 self.parent = s 466 self.writeLogLine('CLIENT: Connected to '+str(self.parent)) 467 return 1 468 469 def sendPacket(self, s, packet, source = 'Unknown', isPickled = 0): 470 '''Pickle the input packet. Send first the size of the pickled string in 32-bit integer, and then the string itself''' 471 self.writeLogLine(source+': Sending packet '+str(packet)) 472 if isPickled: 473 p = packet 474 else: 475 p = cPickle.dumps(packet) 476 self.packer.reset() 477 self.packer.pack_uint(len(p)) 478 if hasattr(s, 'write'): 479 s.write(self.packer.get_buffer()) 480 s.write(p) 481 else: 482 s.sendall(self.packer.get_buffer()) 483 s.sendall(p) 484 self.writeLogLine(source+': Sent packet') 485 return 486 487 def recvPacket(self, s, source = 'Unknown'): 488 '''Receive first the size of the pickled string in a 32-bit integer, and then the string itself. Return the unpickled object''' 489 self.writeLogLine(source+': Receiving packet') 490 if hasattr(s, 'read'): 491 s.read(4) 492 value = cPickle.load(s) 493 else: 494 # I probably need to check that it actually read these 4 bytes 495 self.unpacker.reset(s.recv(4)) 496 length = self.unpacker.unpack_uint() 497 objString = '' 498 while len(objString) < length: 499 objString += s.recv(length - len(objString)) 500 value = cPickle.loads(objString) 501 self.writeLogLine(source+': Received packet '+str(value)) 502 return value 503 504 def send(self, key = None, value = None, operation = None): 505 '''Send a request to the parent''' 506 import inspect 507 508 objString = '' 509 for i in range(3): 510 try: 511 packet = [] 512 if operation is None: 513 operation = inspect.stack()[1][3] 514 packet.append(operation) 515 if not key is None: 516 packet.append(key) 517 if not value is None: 518 packet.append(value) 519 self.sendPacket(self.parent, tuple(packet), source = 'CLIENT') 520 response = self.recvPacket(self.parent, source = 'CLIENT') 521 break 522 except IOError as e: 523 self.writeLogLine('CLIENT: IOError '+str(e)) 524 if e.errno == 32: 525 self.connectParent(self.parentAddr, self.parentDirectory) 526 except Exception as e: 527 self.writeLogLine('CLIENT: Exception '+str(e)+' '+str(e.__class__)) 528 try: 529 if isinstance(response, Exception): 530 self.writeLogLine('CLIENT: Got an exception '+str(response)) 531 raise response 532 else: 533 self.writeLogLine('CLIENT: Received value '+str(response)+' '+str(type(response))) 534 except UnboundLocalError: 535 self.writeLogLine('CLIENT: Could not unpickle response') 536 response = None 537 return response 538 539 def serve(self): 540 '''Start a server''' 541 import socket 542 import SocketServer 543 544 if not useThreads: 545 raise RuntimeError('Cannot run a server if threads are disabled') 546 547 class ProcessHandler(SocketServer.StreamRequestHandler): 548 def handle(self): 549 import time 550 551 self.server.rdict.lastAccess = time.time() 552 self.server.rdict.writeLogLine('SERVER: Started new handler') 553 while 1: 554 try: 555 value = self.server.rdict.recvPacket(self.rfile, source = 'SERVER') 556 except EOFError as e: 557 self.server.rdict.writeLogLine('SERVER: EOFError receiving packet '+str(e)+' '+str(e.__class__)) 558 return 559 except Exception as e: 560 self.server.rdict.writeLogLine('SERVER: Error receiving packet '+str(e)+' '+str(e.__class__)) 561 self.server.rdict.sendPacket(self.wfile, e, source = 'SERVER') 562 continue 563 if value[0] == 'stop': break 564 try: 565 response = getattr(self.server.rdict, value[0])(*value[1:]) 566 except Exception as e: 567 self.server.rdict.writeLogLine('SERVER: Error executing operation '+str(e)+' '+str(e.__class__)) 568 self.server.rdict.sendPacket(self.wfile, e, source = 'SERVER') 569 else: 570 self.server.rdict.sendPacket(self.wfile, response, source = 'SERVER') 571 return 572 573 # check if server is running 574 if os.path.exists(self.addrFilename): 575 rdict = RDict(parentDirectory = '.') 576 hasParent = rdict.hasParent() 577 del rdict 578 if hasParent: 579 self.writeLogLine('SERVER: Another server is already running') 580 raise RuntimeError('Server already running') 581 582 # Daemonize server 583 self.writeLogLine('SERVER: Daemonizing server') 584 if os.fork(): # Launch child 585 os._exit(0) # Kill off parent, so we are not a process group leader and get a new PID 586 os.setsid() # Set session ID, so that we have no controlling terminal 587 # We choose to leave cwd at RDict.py: os.chdir('/') # Make sure root directory is not on a mounted drive 588 os.umask(0o77) # Fix creation mask 589 for i in range(3): # Crappy stopgap for closing descriptors 590 try: 591 os.close(i) 592 except OSError as e: 593 if e.errno != errno.EBADF: 594 raise RuntimeError('Could not close default descriptor '+str(i)) 595 596 # wish there was a better way to get a usable socket 597 self.writeLogLine('SERVER: Establishing socket server') 598 basePort = 8000 599 flag = 'nosocket' 600 p = 1 601 while p < 1000 and flag == 'nosocket': 602 try: 603 server = SocketServer.ThreadingTCPServer((socket.gethostname(), basePort+p), ProcessHandler) 604 flag = 'socket' 605 except Exception as e: 606 p = p + 1 607 if flag == 'nosocket': 608 p = 1 609 while p < 1000 and flag == 'nosocket': 610 try: 611 server = SocketServer.ThreadingTCPServer(('localhost', basePort+p), ProcessHandler) 612 flag = 'socket' 613 except Exception as e: 614 p = p + 1 615 if flag == 'nosocket': 616 self.writeLogLine('SERVER: Could not established socket server on port '+str(basePort+p)) 617 raise RuntimeError('Cannot get available socket') 618 self.writeLogLine('SERVER: Established socket server on port '+str(basePort+p)) 619 620 self.isServer = 1 621 self.writeServerAddr(server) 622 self.serverShutdown(os.getpid()) 623 624 server.rdict = self 625 self.writeLogLine('SERVER: Started server') 626 server.serve_forever() 627 return 628 629 def load(self): 630 '''Load the saved dictionary''' 631 if not self.parentDirectory is None and os.path.samefile(os.getcwd(), self.parentDirectory): 632 return 633 self.saveFilename = os.path.abspath(self.saveFilename) 634 if os.path.exists(self.saveFilename): 635 try: 636 dbFile = file(self.saveFilename) 637 data = cPickle.load(dbFile) 638 self.updateTypes(data) 639 dbFile.close() 640 self.writeLogLine('Loaded dictionary from '+self.saveFilename) 641 except Exception as e: 642 self.writeLogLine('Problem loading dictionary from '+self.saveFilename+'\n--> '+str(e)) 643 else: 644 self.writeLogLine('No dictionary to load in this file: '+self.saveFilename) 645 return 646 647 def save(self, force = 1): 648 '''Save the dictionary after 5 seconds, ignoring all subsequent calls until the save 649 - Giving force = True will cause an immediate save''' 650 if self.readonly: return 651 if force: 652 self.saveTimer = None 653 # This should be a critical section 654 dbFile = file(self.saveFilename, 'w') 655 data = dict(filter(lambda i: not i[1].getTemporary(), self.localitems())) 656 cPickle.dump(data, dbFile) 657 dbFile.close() 658 self.writeLogLine('Saved local dictionary to '+os.path.abspath(self.saveFilename)) 659 elif not self.saveTimer: 660 import threading 661 self.saveTimer = threading.Timer(5, self.save, [], {'force': 1}) 662 self.saveTimer.setDaemon(1) 663 self.saveTimer.start() 664 return 665 666 def shutdown(self): 667 '''Shutdown the dictionary, writing out changes and notifying parent''' 668 if self.saveTimer: 669 self.saveTimer.cancel() 670 self.save(force = 1) 671 if self.isServer and os.path.isfile(self.addrFilename): 672 os.remove(self.addrFilename) 673 if not self.parent is None: 674 self.sendPacket(self.parent, self.stopCmd, isPickled = 1) 675 self.parent.close() 676 self.parent = None 677 self.writeLogLine('Shutting down') 678 self.logFile.close() 679 return 680 681 def serverShutdown(self, pid, delay = shutdownDelay): 682 if self.shutdownTimer is None: 683 import threading 684 685 self.shutdownTimer = threading.Timer(delay, self.serverShutdown, [pid], {'delay': 0}) 686 self.shutdownTimer.setDaemon(1) 687 self.shutdownTimer.start() 688 self.writeLogLine('SERVER: Set shutdown timer for process '+str(pid)+' at '+str(delay)+' seconds') 689 else: 690 try: 691 import signal 692 import time 693 694 idleTime = time.time() - self.lastAccess 695 self.writeLogLine('SERVER: Last access '+str(self.lastAccess)) 696 self.writeLogLine('SERVER: Idle time '+str(idleTime)) 697 if idleTime < RDict.shutdownDelay: 698 self.writeLogLine('SERVER: Extending shutdown timer for '+str(pid)+' by '+str(RDict.shutdownDelay - idleTime)+' seconds') 699 self.shutdownTimer = None 700 self.serverShutdown(pid, RDict.shutdownDelay - idleTime) 701 else: 702 self.writeLogLine('SERVER: Killing server '+str(pid)) 703 os.kill(pid, signal.SIGTERM) 704 except Exception as e: 705 self.writeLogLine('SERVER: Exception killing server: '+str(e)) 706 return 707 708if __name__ == '__main__': 709 import sys 710 try: 711 if len(sys.argv) < 2: 712 print('RDict.py [server | client | clear | insert | remove] [parent]') 713 else: 714 action = sys.argv[1] 715 parent = None 716 if len(sys.argv) > 2: 717 if not sys.argv[2] == 'None': parent = sys.argv[2] 718 if action == 'server': 719 RDict(parentDirectory = parent).serve() 720 elif action == 'client': 721 print('Entries in server dictionary') 722 rdict = RDict(parentDirectory = parent) 723 for key in rdict.types(): 724 if not key.startswith('cacheKey') and not key.startswith('stamp-'): 725 print(str(key)+' '+str(rdict.getType(key))) 726 elif action == 'cacheClient': 727 print('Cache entries in server dictionary') 728 rdict = RDict(parentDirectory = parent) 729 for key in rdict.types(): 730 if key.startswith('cacheKey'): 731 print(str(key)+' '+str(rdict.getType(key))) 732 elif action == 'stampClient': 733 print('Stamp entries in server dictionary') 734 rdict = RDict(parentDirectory = parent) 735 for key in rdict.types(): 736 if key.startswith('stamp-'): 737 print(str(key)+' '+str(rdict.getType(key))) 738 elif action == 'clear': 739 print('Clearing all dictionaries') 740 RDict(parentDirectory = parent).clear() 741 elif action == 'insert': 742 rdict = RDict(parentDirectory = parent) 743 rdict[sys.argv[3]] = sys.argv[4] 744 elif action == 'remove': 745 rdict = RDict(parentDirectory = parent) 746 del rdict[sys.argv[3]] 747 else: 748 sys.exit('Unknown action: '+action) 749 except Exception as e: 750 import traceback 751 print(traceback.print_tb(sys.exc_info()[2])) 752 sys.exit(str(e)) 753 sys.exit(0) 754