1'''A remote dictionary server 2 3 RDict is a typed, hierarchical, persistent dictionary intended to manage 4 all arguments or options for a program. The interface remains exactly the 5 same as dict, but the storage is more complicated. 6 7 Argument typing is handled by wrapping all values stored in the dictionary 8 with nargs.Arg or a subclass. A user can call setType() to set the type of 9 an argument without any value being present. Whenever __getitem__() or 10 __setitem__() is called, values are extracted or replaced in the wrapper. 11 These wrappers can be accessed directly using getType(), setType(), and 12 types(). 13 14 Hierarchy is allowed using a single "parent" dictionary. All operations 15 cascade to the parent. For instance, the length of the dictionary is the 16 number of local keys plus the number of keys in the parent, and its 17 parent, etc. Also, a dictionary need not have a parent. If a key does not 18 appear in the local dictionary, the call if passed to the parent. However, 19 in this case we see that local keys can shadow those in a parent. 20 Communication with the parent is handled using sockets, with the parent 21 being a server and the interactive dictionary a client. 22 23 The default persistence mechanism is a pickle file, RDict.db, written 24 whenever an argument is changed locally. A timer thread is created after 25 an initial change, so that many rapid changes do not cause many writes. 26 Each dictionary only saves its local entries, so all parents also 27 separately save data in different RDict.db files. Each time a dictionary 28 is created, the current directory is searched for an RDict.db file, and 29 if found the contents are loaded into the dictionary. 30 31 This script also provides some default actions: 32 33 - server [parent] 34 Starts a server in the current directory with an optional parent. This 35 server will accept socket connections from other dictionaries and act 36 as a parent. 37 38 - client [parent] 39 Creates a dictionary in the current directory with an optional parent 40 and lists the contents. Notice that the contents may come from either 41 an RDict.db file in the current directory, or from the parent. 42 43 - clear [parent] 44 Creates a dictionary in the current directory with an optional parent 45 and clears the contents. Notice that this will also clear the parent. 46 47 - insert <parent> <key> <value> 48 Creates a dictionary in the current directory with a parent, and inserts 49 the key-value pair. If "parent" is "None", no parent is assigned. 50 51 - remove <parent> <key> 52 Creates a dictionary in the current directory with a parent, and removes 53 the given key. If "parent" is "None", no parent is assigned. 54''' 55from __future__ import print_function 56from __future__ import absolute_import 57try: 58 import project # This is necessary for us to create Project objects on load 59 import build.buildGraph # This is necessary for us to create BuildGraph objects on load 60except ImportError: 61 pass 62import nargs 63 64import pickle 65import os 66import sys 67useThreads = nargs.Arg.findArgument('useThreads', sys.argv[1:]) 68if useThreads is None: 69 useThreads = 0 # workaround issue with parallel configure 70elif useThreads == 'no' or useThreads == '0': 71 useThreads = 0 72elif useThreads == 'yes' or useThreads == '1': 73 useThreads = 1 74else: 75 raise RuntimeError('Unknown option value for --useThreads ',useThreads) 76 77class RDict(dict): 78 '''An RDict is a typed dictionary, which may be hierarchically composed. All elements derive from the 79Arg class, which wraps the usual value.''' 80 # The server will self-shutdown after this many seconds 81 shutdownDelay = 60*60*5 82 83 def __init__(self, parentAddr = None, parentDirectory = None, load = 1, autoShutdown = 1, readonly = False): 84 import atexit 85 import time 86 import xdrlib 87 88 self.logFile = None 89 self.setupLogFile() 90 self.target = ['default'] 91 self.parent = None 92 self.saveTimer = None 93 self.shutdownTimer = None 94 self.lastAccess = time.time() 95 self.saveFilename = 'RDict.db' 96 self.addrFilename = 'RDict.loc' 97 self.parentAddr = parentAddr 98 self.isServer = 0 99 self.readonly = readonly 100 self.parentDirectory = parentDirectory 101 self.packer = xdrlib.Packer() 102 self.unpacker = xdrlib.Unpacker('') 103 self.stopCmd = pickle.dumps(('stop',)) 104 self.writeLogLine('Greetings') 105 self.connectParent(self.parentAddr, self.parentDirectory) 106 if load: self.load() 107 if autoShutdown and useThreads: 108 atexit.register(self.shutdown) 109 self.writeLogLine('SERVER: Last access '+str(self.lastAccess)) 110 return 111 112 def __getstate__(self): 113 '''Remove any parent socket object, the XDR translators, and the log file from the dictionary before pickling''' 114 self.writeLogLine('Pickling RDict') 115 d = self.__dict__.copy() 116 if 'parent' in d: del d['parent'] 117 if 'saveTimer' in d: del d['saveTimer'] 118 if '_setCommandLine' in d: del d['_setCommandLine'] 119 del d['packer'] 120 del d['unpacker'] 121 del d['logFile'] 122 return d 123 124 def __setstate__(self, d): 125 '''Reconnect the parent socket object, recreate the XDR translators and reopen the log file after unpickling''' 126 self.logFile = open('RDict.log', 'a') 127 self.writeLogLine('Unpickling RDict') 128 self.__dict__.update(d) 129 import xdrlib 130 self.packer = xdrlib.Packer() 131 self.unpacker = xdrlib.Unpacker('') 132 self.connectParent(self.parentAddr, self.parentDirectory) 133 return 134 135 def setupLogFile(self, filename = 'RDict.log'): 136 if not self.logFile is None: 137 self.logFile.close() 138 if os.path.isfile(filename) and os.stat(filename).st_size > 10*1024*1024: 139 if os.path.isfile(filename+'.bkp'): 140 os.remove(filename+'.bkp') 141 os.rename(filename, filename+'.bkp') 142 self.logFile = open(filename, 'w') 143 else: 144 self.logFile = open(filename, 'a') 145 return 146 147 def writeLogLine(self, message): 148 '''Writes the message to the log along with the current time''' 149 import time 150 self.logFile.write('('+str(os.getpid())+')('+str(id(self))+')'+message+' ['+time.asctime(time.localtime())+']\n') 151 self.logFile.flush() 152 return 153 154 def __len__(self): 155 '''Returns the length of both the local and parent dictionaries''' 156 length = dict.__len__(self) 157 if not self.parent is None: 158 length = length + self.send() 159 return length 160 161 def getType(self, key): 162 '''Checks for the key locally, and if not found consults the parent. Returns the Arg object or None if not found.''' 163 try: 164 value = dict.__getitem__(self, key) 165 self.writeLogLine('getType: Getting local type for '+key+' '+str(value)) 166 return value 167 except KeyError: 168 pass 169 if self.parent: 170 return self.send(key) 171 return None 172 173 def dict_has_key(self, key): 174 """Utility to check whether the key is present in the dictionary without RDict side-effects.""" 175 return key in dict(self) 176 177 def __getitem__(self, key): 178 '''Checks for the key locally, and if not found consults the parent. Returns the value of the Arg. 179 - If the value has not been set, the user will be prompted for input''' 180 if self.dict_has_key(key): 181 self.writeLogLine('__getitem__: '+key+' has local type') 182 pass 183 elif not self.parent is None: 184 self.writeLogLine('__getitem__: Checking parent value') 185 if self.send(key, operation = 'has_key'): 186 self.writeLogLine('__getitem__: Parent has value') 187 return self.send(key) 188 else: 189 self.writeLogLine('__getitem__: Checking parent type') 190 arg = self.send(key, operation = 'getType') 191 if not arg: 192 self.writeLogLine('__getitem__: Parent has no type') 193 arg = nargs.Arg(key) 194 try: 195 value = arg.getValue() 196 except AttributeError as e: 197 self.writeLogLine('__getitem__: Parent had invalid entry: '+str(e)) 198 arg = nargs.Arg(key) 199 value = arg.getValue() 200 self.writeLogLine('__getitem__: Setting parent value '+str(value)) 201 self.send(key, value, operation = '__setitem__') 202 return value 203 else: 204 self.writeLogLine('__getitem__: Setting local type for '+key) 205 dict.__setitem__(self, key, nargs.Arg(key)) 206 #self.save() 207 self.writeLogLine('__getitem__: Setting local value for '+key) 208 return dict.__getitem__(self, key).getValue() 209 210 def setType(self, key, value, forceLocal = 0): 211 '''Checks for the key locally, and if not found consults the parent. Sets the type for this key. 212 - If a value for the key already exists, it is converted to the new type''' 213 if not isinstance(value, nargs.Arg): 214 raise TypeError('An argument type must be a subclass of Arg') 215 value.setKey(key) 216 if forceLocal or self.parent is None or self.dict_has_key(key): 217 if self.dict_has_key(key): 218 v = dict.__getitem__(self, key) 219 if v.isValueSet(): 220 try: 221 value.setValue(v.getValue()) 222 except TypeError: 223 print(value.__class__.__name__[3:]) 224 print('-----------------------------------------------------------------------') 225 print('Warning! Incorrect argument type specified: -'+str(key)+'='+str(v.getValue())+' - expecting type '+value.__class__.__name__[3:]+'.') 226 print('-----------------------------------------------------------------------') 227 pass 228 dict.__setitem__(self, key, value) 229 #self.save() 230 else: 231 return self.send(key, value) 232 return 233 234 def __setitem__(self, key, value): 235 '''Checks for the key locally, and if not found consults the parent. Sets the value of the Arg.''' 236 if not self.dict_has_key(key): 237 if not self.parent is None: 238 return self.send(key, value) 239 else: 240 dict.__setitem__(self, key, nargs.Arg(key)) 241 dict.__getitem__(self, key).setValue(value) 242 self.writeLogLine('__setitem__: Set value for '+key+' to '+str(dict.__getitem__(self, key))) 243 #self.save() 244 return 245 246 def __delitem__(self, key): 247 '''Checks for the key locally, and if not found consults the parent. Deletes the Arg completely.''' 248 if self.dict_has_key(key): 249 dict.__delitem__(self, key) 250 #self.save() 251 elif not self.parent is None: 252 self.send(key) 253 return 254 255 def clear(self): 256 '''Clears both the local and parent dictionaries''' 257 if dict.__len__(self): 258 dict.clear(self) 259 #self.save() 260 if not self.parent is None: 261 self.send() 262 return 263 264 def __contains__(self, key): 265 '''Checks for the key locally, and if not found consults the parent. Then checks whether the value has been set''' 266 if self.dict_has_key(key): 267 if dict.__getitem__(self, key).isValueSet(): 268 self.writeLogLine('has_key: Have value for '+key) 269 else: 270 self.writeLogLine('has_key: Do not have value for '+key) 271 return dict.__getitem__(self, key).isValueSet() 272 elif not self.parent is None: 273 return self.send(key) 274 return 0 275 276 def get(self, key, default=None): 277 if key in self: 278 return self.__getitem__(key) 279 else: 280 return default 281 282 def hasType(self, key): 283 '''Checks for the key locally, and if not found consults the parent. Then checks whether the type has been set''' 284 if self.dict_has_key(key): 285 return 1 286 elif not self.parent is None: 287 return self.send(key) 288 return 0 289 290 def items(self): 291 '''Return a list of all accessible items, as (key, value) pairs.''' 292 l = dict.items(self) 293 if not self.parent is None: 294 l.extend(self.send()) 295 return l 296 297 def localitems(self): 298 '''Return a list of all the items stored locally, as (key, value) pairs.''' 299 return dict.items(self) 300 301 def keys(self): 302 '''Returns the list of keys in both the local and parent dictionaries''' 303 keyList = [key for key in dict.keys(self) if dict.__getitem__(self, key).isValueSet()] 304 if not self.parent is None: 305 keyList.extend(self.send()) 306 return keyList 307 308 def types(self): 309 '''Returns the list of keys for which types are defined in both the local and parent dictionaries''' 310 keyList = dict.keys(self) 311 if not self.parent is None: 312 keyList.extend(self.send()) 313 return keyList 314 315 def update(self, d): 316 '''Update the dictionary with the contents of d''' 317 for k in d: 318 self[k] = d[k] 319 return 320 321 def updateTypes(self, d): 322 '''Update types locally, which is equivalent to the dict.update() method''' 323 return dict.update(self, d) 324 325 def insertArg(self, key, value, arg): 326 '''Insert a (key, value) pair into the dictionary. If key is None, arg is put into the target list.''' 327 if not key is None: 328 self[key] = value 329 else: 330 if not self.target == ['default']: 331 self.target.append(arg) 332 else: 333 self.target = [arg] 334 return 335 336 def insertArgs(self, args): 337 '''Insert some text arguments into the dictionary (list and dictionaries are recognized)''' 338 339 if isinstance(args, list): 340 for arg in args: 341 (key, value) = nargs.Arg.parseArgument(arg) 342 self.insertArg(key, value, arg) 343 elif hasattr(args, 'keys'): 344 for key in args.keys(): 345 if isinstance(args[key], str): 346 value = nargs.Arg.parseValue(args[key]) 347 else: 348 value = args[key] 349 self.insertArg(key, value, None) 350 elif isinstance(args, str): 351 (key, value) = nargs.Arg.parseArgument(args) 352 self.insertArg(key, value, args) 353 return 354 355 def hasParent(self): 356 '''Return True if this RDict has a parent dictionary''' 357 return not self.parent is None 358 359 def getServerAddr(self, dir): 360 '''Read the server socket address (in pickled form) from a file, usually RDict.loc 361 - If we fail to connect to the server specified in the file, we spawn it using startServer()''' 362 filename = os.path.join(dir, self.addrFilename) 363 if not os.path.exists(filename): 364 self.startServer(filename) 365 if not os.path.exists(filename): 366 raise RuntimeError('Server address file does not exist: '+filename) 367 try: 368 f = open(filename, 'r') 369 addr = pickle.load(f) 370 f.close() 371 return addr 372 except Exception as e: 373 self.writeLogLine('CLIENT: Exception during server address determination: '+str(e.__class__)+': '+str(e)) 374 raise RuntimeError('Could not get server address in '+filename) 375 376 def writeServerAddr(self, server): 377 '''Write the server socket address (in pickled form) to a file, usually RDict.loc.''' 378 f = open(self.addrFilename, 'w') 379 pickle.dump(server.server_address, f) 380 f.close() 381 self.writeLogLine('SERVER: Wrote lock file '+os.path.abspath(self.addrFilename)) 382 return 383 384 def startServer(self, addrFilename): 385 '''Spawn a new RDict server in the parent directory''' 386 import RDict # Need this to locate server script 387 import sys 388 import time 389 import sysconfig 390 391 self.writeLogLine('CLIENT: Spawning a new server with lock file '+os.path.abspath(addrFilename)) 392 if os.path.exists(addrFilename): 393 os.remove(addrFilename) 394 oldDir = os.getcwd() 395 source = os.path.join(os.path.dirname(os.path.abspath(sys.modules['RDict'].__file__)), 'RDict.py') 396 interpreter = os.path.join(sysconfig.get_config_var('BINDIR'), sysconfig.get_config_var('PYTHON')) 397 if not os.path.isfile(interpreter): 398 interpreter = 'python' 399 os.chdir(os.path.dirname(addrFilename)) 400 self.writeLogLine('CLIENT: Executing '+interpreter+' '+source+' server"') 401 try: 402 os.spawnvp(os.P_NOWAIT, interpreter, [interpreter, source, 'server']) 403 except: 404 self.writeLogLine('CLIENT: os.spawnvp failed.\n \ 405 This is a typical problem on CYGWIN systems. If you are using CYGWIN,\n \ 406 you can fix this problem by running /bin/rebaseall. If you do not have\n \ 407 this program, you can install it with the CYGWIN installer in the package\n \ 408 Rebase, under the category System. You must run /bin/rebaseall after\n \ 409 turning off all cygwin services -- in particular sshd, if any such services\n \ 410 are running. For more information about rebase, go to http://www.cygwin.com') 411 print('\n \ 412 This is a typical problem on CYGWIN systems. If you are using CYGWIN,\n \ 413 you can fix this problem by running /bin/rebaseall. If you do not have\n \ 414 this program, you can install it with the CYGWIN installer in the package\n \ 415 Rebase, under the category System. You must run /bin/rebaseall after\n \ 416 turning off all cygwin services -- in particular sshd, if any such services\n \ 417 are running. For more information about rebase, go to http://www.cygwin.com\n') 418 raise 419 os.chdir(oldDir) 420 timeout = 1 421 for i in range(10): 422 time.sleep(timeout) 423 timeout *= 2 424 if timeout > 100: timeout = 100 425 if os.path.exists(addrFilename): return 426 self.writeLogLine('CLIENT: Could not start server') 427 return 428 429 def connectParent(self, addr, dir): 430 '''Try to connect to a parent RDict server 431 - If addr and dir are both None, this operation fails 432 - If addr is None, check for an address file in dir''' 433 if addr is None: 434 if dir is None: return 0 435 addr = self.getServerAddr(dir) 436 437 import socket 438 import errno 439 connected = 0 440 s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 441 timeout = 1 442 for i in range(10): 443 try: 444 self.writeLogLine('CLIENT: Trying to connect to '+str(addr)) 445 s.connect(addr) 446 connected = 1 447 break 448 except socket.error as e: 449 self.writeLogLine('CLIENT: Failed to connect: '+str(e)) 450 if e[0] == errno.ECONNREFUSED: 451 try: 452 import time 453 time.sleep(timeout) 454 timeout *= 2 455 if timeout > 100: timeout = 100 456 except KeyboardInterrupt: 457 break 458 # Try to spawn parent 459 if dir: 460 filename = os.path.join(dir, self.addrFilename) 461 if os.path.isfile(filename): 462 os.remove(filename) 463 self.startServer(filename) 464 except Exception as e: 465 self.writeLogLine('CLIENT: Failed to connect: '+str(e.__class__)+': '+str(e)) 466 if not connected: 467 self.writeLogLine('CLIENT: Failed to connect to parent') 468 return 0 469 self.parent = s 470 self.writeLogLine('CLIENT: Connected to '+str(self.parent)) 471 return 1 472 473 def sendPacket(self, s, packet, source = 'Unknown', isPickled = 0): 474 '''Pickle the input packet. Send first the size of the pickled string in 32-bit integer, and then the string itself''' 475 self.writeLogLine(source+': Sending packet '+str(packet)) 476 if isPickled: 477 p = packet 478 else: 479 p = pickle.dumps(packet) 480 self.packer.reset() 481 self.packer.pack_uint(len(p)) 482 if hasattr(s, 'write'): 483 s.write(self.packer.get_buffer()) 484 s.write(p) 485 else: 486 s.sendall(self.packer.get_buffer()) 487 s.sendall(p) 488 self.writeLogLine(source+': Sent packet') 489 return 490 491 def recvPacket(self, s, source = 'Unknown'): 492 '''Receive first the size of the pickled string in a 32-bit integer, and then the string itself. Return the unpickled object''' 493 self.writeLogLine(source+': Receiving packet') 494 if hasattr(s, 'read'): 495 s.read(4) 496 value = pickle.load(s) 497 else: 498 # I probably need to check that it actually read these 4 bytes 499 self.unpacker.reset(s.recv(4)) 500 length = self.unpacker.unpack_uint() 501 objString = '' 502 while len(objString) < length: 503 objString += s.recv(length - len(objString)) 504 value = pickle.loads(objString) 505 self.writeLogLine(source+': Received packet '+str(value)) 506 return value 507 508 def send(self, key = None, value = None, operation = None): 509 '''Send a request to the parent''' 510 import inspect 511 512 objString = '' 513 for i in range(3): 514 try: 515 packet = [] 516 if operation is None: 517 operation = inspect.stack()[1][3] 518 packet.append(operation) 519 if not key is None: 520 packet.append(key) 521 if not value is None: 522 packet.append(value) 523 self.sendPacket(self.parent, tuple(packet), source = 'CLIENT') 524 response = self.recvPacket(self.parent, source = 'CLIENT') 525 break 526 except IOError as e: 527 self.writeLogLine('CLIENT: IOError '+str(e)) 528 if e.errno == 32: 529 self.connectParent(self.parentAddr, self.parentDirectory) 530 except Exception as e: 531 self.writeLogLine('CLIENT: Exception '+str(e)+' '+str(e.__class__)) 532 try: 533 if isinstance(response, Exception): 534 self.writeLogLine('CLIENT: Got an exception '+str(response)) 535 raise response 536 else: 537 self.writeLogLine('CLIENT: Received value '+str(response)+' '+str(type(response))) 538 except UnboundLocalError: 539 self.writeLogLine('CLIENT: Could not unpickle response') 540 response = None 541 return response 542 543 def serve(self): 544 '''Start a server''' 545 import socket 546 import SocketServer # novermin 547 548 if not useThreads: 549 raise RuntimeError('Cannot run a server if threads are disabled') 550 551 class ProcessHandler(SocketServer.StreamRequestHandler): 552 def handle(self): 553 import time 554 555 self.server.rdict.lastAccess = time.time() 556 self.server.rdict.writeLogLine('SERVER: Started new handler') 557 while 1: 558 try: 559 value = self.server.rdict.recvPacket(self.rfile, source = 'SERVER') 560 except EOFError as e: 561 self.server.rdict.writeLogLine('SERVER: EOFError receiving packet '+str(e)+' '+str(e.__class__)) 562 return 563 except Exception as e: 564 self.server.rdict.writeLogLine('SERVER: Error receiving packet '+str(e)+' '+str(e.__class__)) 565 self.server.rdict.sendPacket(self.wfile, e, source = 'SERVER') 566 continue 567 if value[0] == 'stop': break 568 try: 569 response = getattr(self.server.rdict, value[0])(*value[1:]) 570 except Exception as e: 571 self.server.rdict.writeLogLine('SERVER: Error executing operation '+str(e)+' '+str(e.__class__)) 572 self.server.rdict.sendPacket(self.wfile, e, source = 'SERVER') 573 else: 574 self.server.rdict.sendPacket(self.wfile, response, source = 'SERVER') 575 return 576 577 # check if server is running 578 if os.path.exists(self.addrFilename): 579 rdict = RDict(parentDirectory = '.') 580 hasParent = rdict.hasParent() 581 del rdict 582 if hasParent: 583 self.writeLogLine('SERVER: Another server is already running') 584 raise RuntimeError('Server already running') 585 586 # Daemonize server 587 self.writeLogLine('SERVER: Daemonizing server') 588 if os.fork(): # Launch child 589 os._exit(0) # Kill off parent, so we are not a process group leader and get a new PID 590 os.setsid() # Set session ID, so that we have no controlling terminal 591 # We choose to leave cwd at RDict.py: os.chdir('/') # Make sure root directory is not on a mounted drive 592 os.umask(0o77) # Fix creation mask 593 for i in range(3): # Crappy stopgap for closing descriptors 594 try: 595 os.close(i) 596 except OSError as e: 597 if e.errno != errno.EBADF: 598 raise RuntimeError('Could not close default descriptor '+str(i)) 599 600 # wish there was a better way to get a usable socket 601 self.writeLogLine('SERVER: Establishing socket server') 602 basePort = 8000 603 flag = 'nosocket' 604 p = 1 605 while p < 1000 and flag == 'nosocket': 606 try: 607 server = SocketServer.ThreadingTCPServer((socket.gethostname(), basePort+p), ProcessHandler) 608 flag = 'socket' 609 except Exception as e: 610 p = p + 1 611 if flag == 'nosocket': 612 p = 1 613 while p < 1000 and flag == 'nosocket': 614 try: 615 server = SocketServer.ThreadingTCPServer(('localhost', basePort+p), ProcessHandler) 616 flag = 'socket' 617 except Exception as e: 618 p = p + 1 619 if flag == 'nosocket': 620 self.writeLogLine('SERVER: Could not established socket server on port '+str(basePort+p)) 621 raise RuntimeError('Cannot get available socket') 622 self.writeLogLine('SERVER: Established socket server on port '+str(basePort+p)) 623 624 self.isServer = 1 625 self.writeServerAddr(server) 626 self.serverShutdown(os.getpid()) 627 628 server.rdict = self 629 self.writeLogLine('SERVER: Started server') 630 server.serve_forever() 631 return 632 633 def load(self): 634 '''Load the saved dictionary''' 635 if not self.parentDirectory is None and os.path.samefile(os.getcwd(), self.parentDirectory): 636 return 637 self.saveFilename = os.path.abspath(self.saveFilename) 638 if os.path.exists(self.saveFilename): 639 try: 640 dbFile = open(self.saveFilename, 'rb') 641 data = pickle.load(dbFile) 642 self.updateTypes(data) 643 dbFile.close() 644 self.writeLogLine('Loaded dictionary from '+self.saveFilename) 645 except Exception as e: 646 self.writeLogLine('Problem loading dictionary from '+self.saveFilename+'\n--> '+str(e)) 647 else: 648 self.writeLogLine('No dictionary to load in this file: '+self.saveFilename) 649 return 650 651 def save(self, force = 1): 652 '''Save the dictionary after 5 seconds, ignoring all subsequent calls until the save 653 - Giving force = True will cause an immediate save''' 654 if self.readonly: return 655 if force: 656 self.saveTimer = None 657 # This should be a critical section 658 dbFile = open(self.saveFilename, 'wb') 659 data = dict([i for i in self.localitems() if not i[1].getTemporary()]) 660 pickle.dump(data, dbFile) 661 dbFile.close() 662 self.writeLogLine('Saved local dictionary to '+os.path.abspath(self.saveFilename)) 663 elif not self.saveTimer: 664 import threading 665 self.saveTimer = threading.Timer(5, self.save, [], {'force': 1}) 666 self.saveTimer.setDaemon(1) 667 self.saveTimer.start() 668 return 669 670 def shutdown(self): 671 '''Shutdown the dictionary, writing out changes and notifying parent''' 672 if self.saveTimer: 673 self.saveTimer.cancel() 674 self.save(force = 1) 675 if self.isServer and os.path.isfile(self.addrFilename): 676 os.remove(self.addrFilename) 677 if not self.parent is None: 678 self.sendPacket(self.parent, self.stopCmd, isPickled = 1) 679 self.parent.close() 680 self.parent = None 681 self.writeLogLine('Shutting down') 682 self.logFile.close() 683 return 684 685 def serverShutdown(self, pid, delay = shutdownDelay): 686 if self.shutdownTimer is None: 687 import threading 688 689 self.shutdownTimer = threading.Timer(delay, self.serverShutdown, [pid], {'delay': 0}) 690 self.shutdownTimer.setDaemon(1) 691 self.shutdownTimer.start() 692 self.writeLogLine('SERVER: Set shutdown timer for process '+str(pid)+' at '+str(delay)+' seconds') 693 else: 694 try: 695 import signal 696 import time 697 698 idleTime = time.time() - self.lastAccess 699 self.writeLogLine('SERVER: Last access '+str(self.lastAccess)) 700 self.writeLogLine('SERVER: Idle time '+str(idleTime)) 701 if idleTime < RDict.shutdownDelay: 702 self.writeLogLine('SERVER: Extending shutdown timer for '+str(pid)+' by '+str(RDict.shutdownDelay - idleTime)+' seconds') 703 self.shutdownTimer = None 704 self.serverShutdown(pid, RDict.shutdownDelay - idleTime) 705 else: 706 self.writeLogLine('SERVER: Killing server '+str(pid)) 707 os.kill(pid, signal.SIGTERM) 708 except Exception as e: 709 self.writeLogLine('SERVER: Exception killing server: '+str(e)) 710 return 711 712if __name__ == '__main__': 713 import sys 714 try: 715 if len(sys.argv) < 2: 716 print('RDict.py [server | client | clear | insert | remove] [parent]') 717 else: 718 action = sys.argv[1] 719 parent = None 720 if len(sys.argv) > 2: 721 if not sys.argv[2] == 'None': parent = sys.argv[2] 722 if action == 'server': 723 RDict(parentDirectory = parent).serve() 724 elif action == 'client': 725 print('Entries in server dictionary') 726 rdict = RDict(parentDirectory = parent) 727 for key in rdict.types(): 728 if not key.startswith('cacheKey') and not key.startswith('stamp-'): 729 print(str(key)+' '+str(rdict.getType(key))) 730 elif action == 'cacheClient': 731 print('Cache entries in server dictionary') 732 rdict = RDict(parentDirectory = parent) 733 for key in rdict.types(): 734 if key.startswith('cacheKey'): 735 print(str(key)+' '+str(rdict.getType(key))) 736 elif action == 'stampClient': 737 print('Stamp entries in server dictionary') 738 rdict = RDict(parentDirectory = parent) 739 for key in rdict.types(): 740 if key.startswith('stamp-'): 741 print(str(key)+' '+str(rdict.getType(key))) 742 elif action == 'clear': 743 print('Clearing all dictionaries') 744 RDict(parentDirectory = parent).clear() 745 elif action == 'insert': 746 rdict = RDict(parentDirectory = parent) 747 rdict[sys.argv[3]] = sys.argv[4] 748 elif action == 'remove': 749 rdict = RDict(parentDirectory = parent) 750 del rdict[sys.argv[3]] 751 else: 752 sys.exit('Unknown action: '+action) 753 except Exception as e: 754 import traceback 755 print(traceback.print_tb(sys.exc_info()[2])) 756 sys.exit(str(e)) 757 sys.exit(0) 758